feat(analytics): configure vector for docker logs instead of syslog d…

…river (#2061)

* fix: update vector config for docker logs

* feat: replace vector syslog driver with docker logs

* chore: add todo for rootless mode

* chore: remove deprecated port

* chore: fix build

* chore: warn about analytics in docker rootless mode

* chore: remove dead code

internal/db/start/start.go

@@ -100,14 +100,14 @@ EOF
 
 func NewHostConfig() container.HostConfig {
 	hostPort := strconv.FormatUint(uint64(utils.Config.Db.Port), 10)
-	hostConfig := WithSyslogConfig(container.HostConfig{
+	hostConfig := container.HostConfig{
 		PortBindings:  nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},
 		RestartPolicy: container.RestartPolicy{Name: "always"},
 		Binds: []string{
 			utils.DbId + ":/var/lib/postgresql/data",
 			utils.ConfigId + ":/etc/postgresql-custom",
 		},
-	})
+	}
 	return hostConfig
 }
 
@@ -184,17 +184,6 @@ func IsUnhealthyError(err error) bool {
 	return ok
 }
 
-func WithSyslogConfig(hostConfig container.HostConfig) container.HostConfig {
-	if utils.Config.Analytics.Enabled {
-		hostConfig.LogConfig.Type = "syslog"
-		hostConfig.LogConfig.Config = map[string]string{
-			"syslog-address": fmt.Sprintf("tcp://%s:%d", utils.Config.Hostname, utils.Config.Analytics.VectorPort),
-			"tag":            "{{.Name}}",
-		}
-	}
-	return hostConfig
-}
-
 func initCurrentBranch(fsys afero.Fs) error {
 	// Create _current_branch file to avoid breaking db branch commands
 	if _, err := fsys.Stat(utils.CurrBranchPath); err == nil {

internal/functions/deploy/deploy.go

@@ -18,7 +18,6 @@ import (
 	"github.com/go-errors/errors"
 	"github.com/spf13/afero"
 	"github.com/spf13/viper"
-	"github.com/supabase/cli/internal/db/start"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/pkg/api"
 )
@@ -138,9 +137,9 @@ func bundleFunction(ctx context.Context, slug, hostImportMapPath string, fsys af
 			Env:   []string{},
 			Cmd:   cmd,
 		},
-		start.WithSyslogConfig(container.HostConfig{
+		container.HostConfig{
 			Binds: binds,
-		}),
+		},
 		network.NetworkingConfig{},
 		"",
 		os.Stdout,

internal/functions/download/download.go

@@ -15,7 +15,6 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/go-errors/errors"
 	"github.com/spf13/afero"
-	"github.com/supabase/cli/internal/db/start"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/pkg/api"
 )
@@ -185,9 +184,9 @@ func extractOne(ctx context.Context, slug, eszipPath string) error {
 			Image: utils.EdgeRuntimeImage,
 			Cmd:   []string{"unbundle", "--eszip", dockerEszipPath, "--output", utils.DockerDenoDir},
 		},
-		start.WithSyslogConfig(container.HostConfig{
+		container.HostConfig{
 			Binds: binds,
-		}),
+		},
 		network.NetworkingConfig{},
 		"",
 		os.Stdout,

internal/functions/serve/serve.go

@@ -15,7 +15,6 @@ import (
 	"github.com/go-errors/errors"
 	"github.com/spf13/afero"
 	"github.com/spf13/viper"
-	"github.com/supabase/cli/internal/db/start"
 	"github.com/supabase/cli/internal/functions/deploy"
 	"github.com/supabase/cli/internal/secrets/set"
 	"github.com/supabase/cli/internal/utils"
@@ -177,10 +176,10 @@ EOF
 			WorkingDir:   utils.DockerDenoDir,
 			// No tcp health check because edge runtime logs them as client connection error
 		},
-		start.WithSyslogConfig(container.HostConfig{
+		container.HostConfig{
 			Binds:        binds,
 			PortBindings: portBindings,
-		}),
+		},
 		network.NetworkingConfig{
 			EndpointsConfig: map[string]*network.EndpointSettings{
 				utils.NetId: {

internal/start/start.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 	"github.com/go-errors/errors"
 	"github.com/jackc/pgconn"
@@ -118,6 +119,7 @@ var (
 
 type vectorConfig struct {
 	ApiKey        string
+	VectorId      string
 	LogflareId    string
 	KongId        string
 	GotrueId      string
@@ -164,6 +166,7 @@ func run(p utils.Program, ctx context.Context, fsys afero.Fs, excludedContainers
 		var vectorConfigBuf bytes.Buffer
 		if err := vectorConfigTemplate.Execute(&vectorConfigBuf, vectorConfig{
 			ApiKey:        utils.Config.Analytics.ApiKey,
+			VectorId:      utils.VectorId,
 			LogflareId:    utils.LogflareId,
 			KongId:        utils.KongId,
 			GotrueId:      utils.GotrueId,
@@ -175,14 +178,25 @@ func run(p utils.Program, ctx context.Context, fsys afero.Fs, excludedContainers
 		}); err != nil {
 			return errors.Errorf("failed to exec template: %w", err)
 		}
-		p.Send(utils.StatusMsg("Starting syslog driver..."))
+		p.Send(utils.StatusMsg("Starting vector..."))
+		var binds []string
+		env := []string{
+			"VECTOR_CONFIG=/etc/vector/vector.yaml",
+		}
+		host := utils.Docker.DaemonHost()
+		if parsed, err := client.ParseHostURL(host); err == nil {
+			if parsed.Scheme == "tcp" {
+				// Special case for GitLab pipeline
+				env = append(env, "DOCKER_HOST="+host)
+			} else {
+				binds = append(binds, parsed.Host+":/var/run/docker.sock:ro")
+			}
+		}
 		if _, err := utils.DockerStart(
 			ctx,
 			container.Config{
 				Image: utils.VectorImage,
-				Env: []string{
-					"VECTOR_CONFIG=/etc/vector/vector.yaml",
-				},
+				Env:   env,
 				Entrypoint: []string{"sh", "-c", `cat <<'EOF' > /etc/vector/vector.yaml && vector
 ` + vectorConfigBuf.String() + `
 EOF
@@ -195,10 +209,9 @@ EOF
 					Timeout:  2 * time.Second,
 					Retries:  3,
 				},
-				ExposedPorts: nat.PortSet{"9000/tcp": {}},
 			},
 			container.HostConfig{
-				PortBindings:  nat.PortMap{"9000/tcp": []nat.PortBinding{{HostPort: strconv.FormatUint(uint64(utils.Config.Analytics.VectorPort), 10)}}},
+				Binds:         binds,
 				RestartPolicy: container.RestartPolicy{Name: "always"},
 			},
 			network.NetworkingConfig{
@@ -370,11 +383,11 @@ EOF
 EOF
 `},
 			},
-			start.WithSyslogConfig(container.HostConfig{
+			container.HostConfig{
 				Binds:         binds,
 				PortBindings:  nat.PortMap{"8000/tcp": []nat.PortBinding{{HostPort: strconv.FormatUint(uint64(utils.Config.Api.Port), 10)}}},
 				RestartPolicy: container.RestartPolicy{Name: "always"},
-			}),
+			},
 			network.NetworkingConfig{
 				EndpointsConfig: map[string]*network.EndpointSettings{
 					utils.NetId: {
@@ -606,9 +619,9 @@ EOF
 					Retries:  3,
 				},
 			},
-			start.WithSyslogConfig(container.HostConfig{
+			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
-			}),
+			},
 			network.NetworkingConfig{
 				EndpointsConfig: map[string]*network.EndpointSettings{
 					utils.NetId: {
@@ -699,9 +712,9 @@ EOF
 					Retries:  3,
 				},
 			},
-			start.WithSyslogConfig(container.HostConfig{
+			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
-			}),
+			},
 			network.NetworkingConfig{
 				EndpointsConfig: map[string]*network.EndpointSettings{
 					utils.NetId: {
@@ -733,9 +746,9 @@ EOF
 				},
 				// PostgREST does not expose a shell for health check
 			},
-			start.WithSyslogConfig(container.HostConfig{
+			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
-			}),
+			},
 			network.NetworkingConfig{
 				EndpointsConfig: map[string]*network.EndpointSettings{
 					utils.NetId: {
@@ -789,10 +802,10 @@ EOF
 					Retries:  3,
 				},
 			},
-			start.WithSyslogConfig(container.HostConfig{
+			container.HostConfig{
 				RestartPolicy: container.RestartPolicy{Name: "always"},
 				Binds:         []string{utils.StorageId + ":" + dockerStoragePath},
-			}),
+			},
 			network.NetworkingConfig{
 				EndpointsConfig: map[string]*network.EndpointSettings{
 					utils.NetId: {

internal/start/templates/vector.yaml

@@ -1,28 +1,30 @@
 api:
   enabled: true
-  address: "0.0.0.0:9001"
+  address: 0.0.0.0:9001
 
 sources:
-  docker_syslog:
-    type: "syslog"
-    address: "0.0.0.0:9000"
-    mode: "tcp"
-    path: "/tmp/socket"
+  docker_host:
+    type: docker_logs
+    exclude_containers:
+      - "{{ .VectorId }}"
 
 transforms:
   project_logs:
     type: remap
     inputs:
-      - docker_syslog
+      - docker_host
     source: |-
       .project = "default"
       .event_message = del(.message)
-      del(.procid)
-      del(.source_id)
+      .appname = del(.container_name)
+      del(.container_created_at)
+      del(.container_id)
       del(.source_type)
-      del(.facility)
+      del(.stream)
+      del(.label)
+      del(.image)
       del(.host)
-      del(.id)
+      del(.stream)
   router:
     type: route
     inputs:
@@ -35,7 +37,7 @@ transforms:
       storage: '.appname == "{{ .StorageId }}"'
       functions: '.appname == "{{ .EdgeRuntimeId }}"'
       db: '.appname == "{{ .DbId }}"'
-  # Kong logs only include api requests
+  # Ignores non nginx errors since they are related with kong booting up
   kong_logs:
     type: remap
     inputs:
@@ -52,7 +54,10 @@ transforms:
           .metadata.request.protocol = req.protocol
           .metadata.response.status_code = req.status
       }
-  # TODO: create a separate page and filter for kong error logs
+      if err != null {
+        abort
+      }
+  # Ignores non nginx errors since they are related with kong booting up
   kong_err:
     type: remap
     inputs:
@@ -73,6 +78,9 @@ transforms:
               .metadata.request.protocol = url[2]
           }
       }
+      if err != null {
+        abort
+      }
   # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.
   auth_logs:
     type: remap
@@ -126,19 +134,27 @@ transforms:
           .metadata.context[0].pid = parsed.pid
       }
   # Postgres logs some messages to stderr which we map to warning severity level
-  # TODO: parse raw postgres logs via regex
   db_logs:
     type: remap
     inputs:
       - router.db
     source: |-
       .metadata.host = "db-default"
       .metadata.parsed.timestamp = .timestamp
-      .metadata.parsed.error_severity = replace!(.severity, r'^err$', "warning")
+
+      parsed, err = parse_regex(.event_message, r'.*(?P<level>INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true)
+
+      if err != null || parsed == null {
+        .metadata.parsed.error_severity = "info"
+      }
+      if parsed != null {
+       .metadata.parsed.error_severity = parsed.level
+      }
       if .metadata.parsed.error_severity == "info" {
           .metadata.parsed.error_severity = "log"
       }
-      .metadata.parsed.error_severity = upcase(.metadata.parsed.error_severity)
+      .metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity)
+
 sinks:
   logflare_auth:
     type: "http"
@@ -179,34 +195,35 @@ sinks:
     method: "post"
     request:
       retry_max_duration_secs: 10
-    # We must route the sink through kong because ingesting logs before logflare is fully initialized will
+    # We must route the sink through kong because ingesting logs before logflare is fully initialised will
     # lead to broken queries from studio. This works by the assumption that containers are started in the
     # following order: vector > db > logflare > kong
     uri: "http://{{ .KongId }}:8000/analytics/v1/api/logs?source_name=postgres.logs&api_key={{ .ApiKey }}"
-  logflare_storage:
+  logflare_functions:
     type: "http"
     inputs:
-      - storage_logs
+      - router.functions
     encoding:
       codec: "json"
     method: "post"
     request:
       retry_max_duration_secs: 10
-    uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=storage.logs.prod.2&api_key={{ .ApiKey }}"
-  logflare_functions:
+    uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=deno-relay-logs&api_key={{ .ApiKey }}"
+  logflare_storage:
     type: "http"
     inputs:
-      - router.functions
+      - storage_logs
     encoding:
       codec: "json"
     method: "post"
     request:
       retry_max_duration_secs: 10
-    uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=deno-relay-logs&api_key={{ .ApiKey }}"
+    uri: "http://{{ .LogflareId }}:4000/api/logs?source_name=storage.logs.prod.2&api_key={{ .ApiKey }}"
   logflare_kong:
     type: "http"
     inputs:
       - kong_logs
+      - kong_err
     encoding:
       codec: "json"
     method: "post"