From adad7e63c7774bb16f793614b62ae98494e9251a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 10:26:13 +0000 Subject: [PATCH 01/10] fix(cli): skip pg_dump in db pull when using pg-delta diff engine Initial pulls under --diff-engine pg-delta were dumping the remote schema via pg_dump before running pg-delta. The dump-then-restore round-trip strips ownership information for objects the local postgres role cannot assume, so platform-managed objects (FDWs, wasm wrappers, system-owned ACLs) leak into the migration file and break \`supabase db reset\` (see CLI-1469, CLI-1470). pg-delta speaks pg_catalog directly via extractCatalog and the supabase integration filters platform objects by owner. Diffing against an empty shadow on initial pull yields a clean initial migration on its own, so dumpRemoteSchema is unnecessary on this path. --- apps/cli-go/internal/db/pull/pull.go | 23 +++++++++++++++---- apps/cli-go/internal/db/pull/pull_test.go | 28 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/apps/cli-go/internal/db/pull/pull.go b/apps/cli-go/internal/db/pull/pull.go index 210dcdfc9b..758cb74810 100644 --- a/apps/cli-go/internal/db/pull/pull.go +++ b/apps/cli-go/internal/db/pull/pull.go @@ -114,11 +114,20 @@ func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, useP config := conn.Config().Config // 1. Assert `supabase/migrations` and `schema_migrations` are in sync. if err := assertRemoteInSync(ctx, conn, fsys); errors.Is(err, errMissing) { - // Ignore schemas flag when working on the initial pull - if err = dumpRemoteSchema(ctx, path, config, fsys); err != nil { - return err + // pg_dump strips ownership when restored as a non-superuser, so platform + // objects (FDWs, wasm wrappers, system-owned ACLs) leak into the migration + // and later break `supabase db reset`. pg-delta speaks pg_catalog directly + // and the supabase integration filter drops these by owner, so the diff + // against an empty shadow yields a clean initial migration on its own. + if !usePgDeltaDiff { + // Ignore schemas flag when working on the initial pull + if err = dumpRemoteSchema(ctx, path, config, fsys); err != nil { + return err + } } - // Run a second pass to pull in changes from default privileges and managed schemas + // For the legacy path this is a second pass that captures changes + // pg_dump cannot emit (default privileges, managed schemas). For the + // pg-delta path this is the only pass and produces the full schema. if err = diffRemoteSchema(ctx, nil, path, config, usePgDeltaDiff, differ, fsys); errors.Is(err, errInSync) { err = nil } @@ -153,7 +162,11 @@ func diffRemoteSchema(ctx context.Context, schema []string, path string, config if trimmed := strings.TrimSpace(output); len(trimmed) == 0 { return errors.New(errInSync) } - // Append to existing migration file since we run this after dump + if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(path)); err != nil { + return err + } + // Append to existing migration file when we run this after dumpRemoteSchema; + // for the pg-delta path this is the only writer and creates the file fresh. f, err := fsys.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { return errors.Errorf("failed to open migration file: %w", err) diff --git a/apps/cli-go/internal/db/pull/pull_test.go b/apps/cli-go/internal/db/pull/pull_test.go index 964a40dbe6..19a1161850 100644 --- a/apps/cli-go/internal/db/pull/pull_test.go +++ b/apps/cli-go/internal/db/pull/pull_test.go @@ -84,6 +84,34 @@ func TestPullSchema(t *testing.T) { assert.Equal(t, []byte("test"), contents) }) + t.Run("skips pg_dump for pg-delta diff engine on initial pull", func(t *testing.T) { + errNetwork := errors.New("network error") + // Setup in-memory fs + fsys := afero.NewMemMapFs() + // Setup mock docker. Only mock the image inspect call that + // CreateShadowDatabase makes; do NOT mock the pg_dump container so + // the test fails loudly if pg_dump is still invoked. + require.NoError(t, apitest.MockDocker(utils.Docker)) + defer gock.OffAll() + gock.New(utils.Docker.DaemonHost()). + Get("/v" + utils.Docker.ClientVersion() + "/images/" + utils.GetRegistryImageUrl(utils.Config.Db.Image) + "/json"). + ReplyError(errNetwork) + // Setup mock postgres (no local migrations -> initial pull path) + conn := pgtest.NewConn() + defer conn.Close(t) + conn.Query(migration.LIST_MIGRATION_VERSION). + Reply("SELECT 0") + // Run test with usePgDeltaDiff=true + err := run(context.Background(), nil, "0_test.sql", conn.MockClient(t), true, diff.DiffPgDelta, fsys) + // Failure must come from shadow-creation image inspect (proving we + // reached the diff step), not from pg_dump. + assert.ErrorIs(t, err, errNetwork) + assert.Empty(t, apitest.ListUnmatchedRequests()) + exists, err := afero.Exists(fsys, "0_test.sql") + assert.NoError(t, err) + assert.False(t, exists, "pg_dump should be skipped for pg-delta diff engine") + }) + t.Run("throws error on diff failure", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() From a51a612f2754db8fd75fb125bdd465cbbb6c7b80 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 20 May 2026 13:34:44 +0200 Subject: [PATCH 02/10] wip --- .repos/effect | 2 +- .repos/effect-v3 | 2 +- .repos/lalph | 2 +- .repos/process-compose | 2 +- .repos/t3code | 2 +- apps/cli-go/cmd/db.go | 9 +- apps/cli-go/cmd/db_pull_routing_test.go | 29 +++++ apps/cli-go/docs/supabase/db/pull.md | 28 ++++- apps/cli-go/internal/db/declarative/debug.go | 32 +++-- apps/cli-go/internal/db/diff/diff.go | 44 +++++-- apps/cli-go/internal/db/diff/diff_test.go | 16 +-- apps/cli-go/internal/db/diff/pgdelta.go | 87 ++++++++----- apps/cli-go/internal/db/diff/pgdelta_debug.go | 85 +++++++++++++ .../internal/db/diff/pgdelta_debug_test.go | 50 ++++++++ .../internal/db/diff/templates/pgdelta.ts | 20 ++- apps/cli-go/internal/db/pgcache/cache.go | 7 +- .../internal/db/pull/pgdelta_pull_debug.go | 113 +++++++++++++++++ .../db/pull/pgdelta_pull_debug_test.go | 96 +++++++++++++++ apps/cli-go/internal/db/pull/pull.go | 44 +++++-- apps/cli-go/internal/db/pull/pull_test.go | 37 ++++++ .../cli-go/internal/gen/types/pgdelta_conn.go | 114 ++++++++++++++++++ .../internal/gen/types/pgdelta_conn_test.go | 53 ++++++++ apps/cli/package.json | 3 +- 23 files changed, 799 insertions(+), 78 deletions(-) create mode 100644 apps/cli-go/cmd/db_pull_routing_test.go create mode 100644 apps/cli-go/internal/db/diff/pgdelta_debug.go create mode 100644 apps/cli-go/internal/db/diff/pgdelta_debug_test.go create mode 100644 apps/cli-go/internal/db/pull/pgdelta_pull_debug.go create mode 100644 apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go create mode 100644 apps/cli-go/internal/gen/types/pgdelta_conn.go create mode 100644 apps/cli-go/internal/gen/types/pgdelta_conn_test.go diff --git a/.repos/effect b/.repos/effect index 49b5a569ea..aae8797b9c 160000 --- a/.repos/effect +++ b/.repos/effect @@ -1 +1 @@ -Subproject commit 49b5a569ea6c9d459f3db9cb2f150ca9d04b3cd0 +Subproject commit aae8797b9cb383be0c182dd58d03d787c354238b diff --git a/.repos/effect-v3 b/.repos/effect-v3 index e71ba68273..70ce155cd7 160000 --- a/.repos/effect-v3 +++ b/.repos/effect-v3 @@ -1 +1 @@ -Subproject commit e71ba68273026a1a2c1ace7218bdb206b0d3386d +Subproject commit 70ce155cd73a3b4cd723fe955454b5837b428f76 diff --git a/.repos/lalph b/.repos/lalph index 0fddf20a7d..203f1ec28f 160000 --- a/.repos/lalph +++ b/.repos/lalph @@ -1 +1 @@ -Subproject commit 0fddf20a7d70391bb583fb0abadc4223b618ec4d +Subproject commit 203f1ec28f26d3a4f18c0f3e092eae3695de1842 diff --git a/.repos/process-compose b/.repos/process-compose index a4038d6698..cd7f6af235 160000 --- a/.repos/process-compose +++ b/.repos/process-compose @@ -1 +1 @@ -Subproject commit a4038d669818c35fc68fc7fc240b39e371ce0e7a +Subproject commit cd7f6af235149a075385f3b8b54d635e83dc0f52 diff --git a/.repos/t3code b/.repos/t3code index d1e85c4e8f..91a03e0747 160000 --- a/.repos/t3code +++ b/.repos/t3code @@ -1 +1 @@ -Subproject commit d1e85c4e8fdef82fbaded9539532b754080419e0 +Subproject commit 91a03e074751e9dc732d0dddcd7b3a291caba34f diff --git a/apps/cli-go/cmd/db.go b/apps/cli-go/cmd/db.go index 99d1f81343..e4487deb3e 100644 --- a/apps/cli-go/cmd/db.go +++ b/apps/cli-go/cmd/db.go @@ -181,7 +181,7 @@ var ( if usePgDeltaDiff { pullDiffer = diff.DiffPgDelta } - useDeclarativePgDelta := shouldUsePgDelta() + useDeclarativePgDelta := shouldUseDeclarativePgDeltaPull(usePgDeltaDiff) return pull.Run(cmd.Context(), schema, flags.DbConfig, name, useDeclarativePgDelta, usePgDeltaDiff, pullDiffer, afero.NewOsFs()) }, PostRun: func(cmd *cobra.Command, args []string) { @@ -358,6 +358,13 @@ func shouldUsePgDelta() bool { return utils.IsPgDeltaEnabled() || usePgDelta || viper.GetBool("EXPERIMENTAL_PG_DELTA") } +func shouldUseDeclarativePgDeltaPull(usePgDeltaDiff bool) bool { + if usePgDeltaDiff { + return false + } + return shouldUsePgDelta() || usePgDelta +} + func init() { // Build branch command dbBranchCmd.AddCommand(dbBranchCreateCmd) diff --git a/apps/cli-go/cmd/db_pull_routing_test.go b/apps/cli-go/cmd/db_pull_routing_test.go new file mode 100644 index 0000000000..8a01e782f8 --- /dev/null +++ b/apps/cli-go/cmd/db_pull_routing_test.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShouldUseDeclarativePgDeltaPull(t *testing.T) { + t.Run("migration pg-delta wins over experimental config", func(t *testing.T) { + usePgDelta = false + t.Cleanup(func() { usePgDelta = false }) + assert.False(t, shouldUseDeclarativePgDeltaPull(true)) + }) + + t.Run("experimental config without diff-engine uses declarative", func(t *testing.T) { + usePgDelta = false + t.Cleanup(func() { usePgDelta = false }) + // Simulate config enabled via shouldUsePgDelta's IsPgDeltaEnabled path indirectly: + // when neither flag nor config is set, declarative is off. + assert.False(t, shouldUseDeclarativePgDeltaPull(false)) + }) + + t.Run("use-pg-delta flag forces declarative", func(t *testing.T) { + usePgDelta = true + t.Cleanup(func() { usePgDelta = false }) + assert.True(t, shouldUseDeclarativePgDeltaPull(false)) + }) +} diff --git a/apps/cli-go/docs/supabase/db/pull.md b/apps/cli-go/docs/supabase/db/pull.md index b137aaa42a..93f964b9a9 100644 --- a/apps/cli-go/docs/supabase/db/pull.md +++ b/apps/cli-go/docs/supabase/db/pull.md @@ -8,6 +8,30 @@ Requires your local project to be linked to a remote database by running `supaba Optionally, a new row can be inserted into the migration history table to reflect the current state of the remote database. -If no entries exist in the migration history table, `pg_dump` will be used to capture all contents of the remote schemas you have created. Otherwise, this command will only diff schema changes against the remote database, similar to running `db diff --linked`. +If no entries exist in the migration history table, the default diff engine uses `pg_dump` to capture all contents of the remote schemas you have created. Otherwise, this command will only diff schema changes against the remote database, similar to running `db diff --linked`. -Pass `--diff-engine pg-delta` to keep the migration-file `db pull` workflow while using pg-delta for the shadow diff step. Pass `--use-pg-delta` to switch to the declarative pg-delta export workflow instead. +Pass `--diff-engine pg-delta` to keep the migration-file `db pull` workflow while using pg-delta for the shadow diff step. On initial pull, pg-delta replaces `pg_dump` and produces the full migration from the shadow diff alone. Pass `--use-pg-delta` to switch to the declarative pg-delta export workflow instead. + +When `[experimental.pgdelta] enabled = true` is set in `config.toml`, `db pull` defaults to the declarative export path. Explicit `--diff-engine pg-delta` still selects the migration-file workflow. + +When pulling from a remote database with `--db-url`, prefer a direct connection (`db..supabase.co:5432`) over the connection pooler so pg-delta can introspect the full catalog reliably. + +## Debugging empty pg-delta pulls + +If `db pull --diff-engine pg-delta` reports `No schema changes found` but you expect schema output, set `PGDELTA_DEBUG=1` before running the command. Unlike `--debug`, this keeps SSL enabled for remote Supabase connections. + +```sh +PGDELTA_DEBUG=1 supabase db pull --db-url "$DATABASE_URL" --diff-engine pg-delta +``` + +When pg-delta returns zero statements, the CLI writes a debug bundle under `supabase/.temp/pgdelta/debug//`: + +- `source-catalog.json` — shadow database baseline pg-delta extracted +- `target-catalog.json` — remote database pg-delta extracted +- `pgdelta-stderr.txt` — pg-delta script diagnostics (statement count, schemas) +- `connection.txt` — redacted connection metadata +- `error.txt` — error summary + +Catalog files are not written during normal `db pull` runs. The `.temp/pgdelta` directory is also used by migration catalog caching (`db push`, local `db start`) when `[experimental.pgdelta] enabled = true`. + +For TLS tracing without disabling SSL, use `SUPABASE_SSL_DEBUG=true` alongside `PGDELTA_DEBUG=1`. diff --git a/apps/cli-go/internal/db/declarative/debug.go b/apps/cli-go/internal/db/declarative/debug.go index f274ed39dd..779514c525 100644 --- a/apps/cli-go/internal/db/declarative/debug.go +++ b/apps/cli-go/internal/db/declarative/debug.go @@ -18,12 +18,16 @@ const ( // DebugBundle collects diagnostic artifacts when a declarative operation fails. type DebugBundle struct { - ID string // timestamp-based unique ID (e.g. "20240414-044403") - SourceRef string // path to source catalog - TargetRef string // path to target catalog - MigrationSQL string // generated migration (if available) - Error error // the error that occurred - Migrations []string // list of local migration files + ID string // timestamp-based unique ID (e.g. "20240414-044403") + SourceRef string // path to source catalog + TargetRef string // path to target catalog + SourceCatalog string // inline source catalog JSON (optional) + TargetCatalog string // inline target catalog JSON (optional) + MigrationSQL string // generated migration (if available) + PgDeltaStderr string // edge-runtime stderr from pg-delta scripts + ConnectionInfo string // redacted connection metadata + Error error // the error that occurred + Migrations []string // list of local migration files } // SaveDebugBundle writes diagnostic artifacts to .temp/pgdelta/debug// and @@ -38,14 +42,18 @@ func SaveDebugBundle(bundle DebugBundle, fsys afero.Fs) (string, error) { } // Copy source catalog if available - if len(bundle.SourceRef) > 0 { + if len(bundle.SourceCatalog) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "source-catalog.json"), []byte(bundle.SourceCatalog), fsys) + } else if len(bundle.SourceRef) > 0 { if data, err := afero.ReadFile(fsys, bundle.SourceRef); err == nil { _ = utils.WriteFile(filepath.Join(debugDir, "source-catalog.json"), data, fsys) } } // Copy target catalog if available - if len(bundle.TargetRef) > 0 { + if len(bundle.TargetCatalog) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "target-catalog.json"), []byte(bundle.TargetCatalog), fsys) + } else if len(bundle.TargetRef) > 0 { if data, err := afero.ReadFile(fsys, bundle.TargetRef); err == nil { _ = utils.WriteFile(filepath.Join(debugDir, "target-catalog.json"), data, fsys) } @@ -61,6 +69,14 @@ func SaveDebugBundle(bundle DebugBundle, fsys afero.Fs) (string, error) { _ = utils.WriteFile(filepath.Join(debugDir, "error.txt"), []byte(bundle.Error.Error()), fsys) } + if len(bundle.PgDeltaStderr) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "pgdelta-stderr.txt"), []byte(bundle.PgDeltaStderr), fsys) + } + + if len(bundle.ConnectionInfo) > 0 { + _ = utils.WriteFile(filepath.Join(debugDir, "connection.txt"), []byte(bundle.ConnectionInfo), fsys) + } + // Copy migration files if len(bundle.Migrations) > 0 { migrationsDir := filepath.Join(debugDir, "migrations") diff --git a/apps/cli-go/internal/db/diff/diff.go b/apps/cli-go/internal/db/diff/diff.go index 05991423d6..1bca9bf39e 100644 --- a/apps/cli-go/internal/db/diff/diff.go +++ b/apps/cli-go/internal/db/diff/diff.go @@ -33,10 +33,11 @@ import ( type DiffFunc func(context.Context, pgconn.Config, pgconn.Config, []string, ...func(*pgx.ConnConfig)) (string, error) func Run(ctx context.Context, schema []string, file string, config pgconn.Config, differ DiffFunc, usePgDelta bool, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (err error) { - out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDelta, options...) + result, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDelta, options...) if err != nil { return err } + out := result.SQL branch := utils.GetGitBranch(fsys) fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n") if err := SaveDiff(out, file, fsys); err != nil { @@ -161,18 +162,18 @@ func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, return migration.ApplyMigrations(ctx, migrations, conn, afero.NewIOFS(fsys)) } -func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ DiffFunc, usePgDelta bool, options ...func(*pgx.ConnConfig)) (string, error) { +func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ DiffFunc, usePgDelta bool, options ...func(*pgx.ConnConfig)) (DatabaseDiff, error) { fmt.Fprintln(w, "Creating shadow database...") shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort) if err != nil { - return "", err + return DatabaseDiff{}, err } defer utils.DockerRemove(shadow) if err := start.WaitForHealthyService(ctx, utils.Config.Db.HealthTimeout, shadow); err != nil { - return "", err + return DatabaseDiff{}, err } if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil { - return "", err + return DatabaseDiff{}, err } shadowConfig := pgconn.Config{ Host: utils.Config.Hostname, @@ -189,20 +190,20 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w declDir := utils.GetDeclarativeDir() if exists, _ := afero.DirExists(fsys, declDir); exists { if err := pgdelta.ApplyDeclarative(ctx, config, fsys); err != nil { - return "", err + return DatabaseDiff{}, err } } else { if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil { - return "", err + return DatabaseDiff{}, err } } } else { if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil { - return "", err + return DatabaseDiff{}, err } } } else if err != nil { - return "", err + return DatabaseDiff{}, err } } // Load all user defined schemas @@ -211,7 +212,30 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w } else { fmt.Fprintln(w, "Diffing schemas...") } - return differ(ctx, shadowConfig, config, schema, options...) + var debugCapture *PgDeltaDebugCapture + if IsPgDeltaDebugEnabled() && usePgDelta { + if snapshot, exportErr := exportCatalogPgDelta(ctx, utils.ToPostgresURL(shadowConfig), "postgres", options...); exportErr == nil { + debugCapture = &PgDeltaDebugCapture{SourceCatalog: snapshot} + } else { + fmt.Fprintf(w, "Warning: failed to export shadow pg-delta catalog: %v\n", exportErr) + } + } + if IsPgDeltaDebugEnabled() && usePgDelta { + result, err := DiffPgDeltaRefDetailed(ctx, utils.ToPostgresURL(shadowConfig), utils.ToPostgresURL(config), schema, pgDeltaFormatOptions(), options...) + if err != nil { + return DatabaseDiff{}, err + } + if debugCapture == nil { + debugCapture = &PgDeltaDebugCapture{} + } + debugCapture.Stderr = result.Stderr + return DatabaseDiff{SQL: result.SQL, Debug: debugCapture}, nil + } + output, err := differ(ctx, shadowConfig, config, schema, options...) + if err != nil { + return DatabaseDiff{}, err + } + return DatabaseDiff{SQL: output, Debug: debugCapture}, nil } func migrateBaseDatabase(ctx context.Context, config pgconn.Config, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { diff --git a/apps/cli-go/internal/db/diff/diff_test.go b/apps/cli-go/internal/db/diff/diff_test.go index 363d7b3f5b..f14c6a9f73 100644 --- a/apps/cli-go/internal/db/diff/diff_test.go +++ b/apps/cli-go/internal/db/diff/diff_test.go @@ -203,9 +203,9 @@ func TestDiffDatabase(t *testing.T) { Get("/v" + utils.Docker.ClientVersion() + "/images/" + utils.GetRegistryImageUrl(utils.Config.Db.Image) + "/json"). ReplyError(errNetwork) // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorIs(t, err, errNetwork) assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -234,9 +234,9 @@ func TestDiffDatabase(t *testing.T) { Delete("/v" + utils.Docker.ClientVersion() + "/containers/test-shadow-db"). Reply(http.StatusOK) // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorContains(t, err, "test-shadow-db container is not running: exited") assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -266,9 +266,9 @@ func TestDiffDatabase(t *testing.T) { conn.Query(utils.GlobalsSql). ReplyError(pgerrcode.DuplicateSchema, `schema "public" already exists`) // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, conn.Intercept) + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, conn.Intercept) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorContains(t, err, `ERROR: schema "public" already exists (SQLSTATE 42P06) At statement: 0 create schema public`) @@ -321,7 +321,7 @@ create schema public`) Query(migration.INSERT_MIGRATION_VERSION, "0", "test", []string{sql}). Reply("INSERT 0 1") // Run test - diff, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, func(cc *pgx.ConnConfig) { + result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, func(cc *pgx.ConnConfig) { if cc.Host == dbConfig.Host { // Fake a SSL error when connecting to target database cc.LookupFunc = func(ctx context.Context, host string) (addrs []string, err error) { @@ -333,7 +333,7 @@ create schema public`) } }) // Check error - assert.Empty(t, diff) + assert.Empty(t, result.SQL) assert.ErrorContains(t, err, "error diffing schema") assert.Empty(t, apitest.ListUnmatchedRequests()) }) diff --git a/apps/cli-go/internal/db/diff/pgdelta.go b/apps/cli-go/internal/db/diff/pgdelta.go index dd6af84924..787e56cb33 100644 --- a/apps/cli-go/internal/db/diff/pgdelta.go +++ b/apps/cli-go/internal/db/diff/pgdelta.go @@ -68,6 +68,22 @@ func pgDeltaFormatOptions() string { return strings.TrimSpace(utils.Config.Experimental.PgDelta.FormatOptions) } +func appendPgDeltaPostgresEnv( + ctx context.Context, + env []string, + name string, + ref string, + sslRootCertEnv string, + options ...func(*pgx.ConnConfig), +) (string, []string, error) { + preparedRef, sslEnv, err := types.PreparePgDeltaPostgresRef(ctx, ref, sslRootCertEnv, options...) + if err != nil { + return "", nil, err + } + env = append(env, name+"="+containerRef(preparedRef)) + return preparedRef, append(env, sslEnv...), nil +} + // DiffPgDelta diffs source and target Postgres configs via pg-delta. // // This wrapper preserves the old config-based interface while delegating to @@ -81,17 +97,25 @@ func DiffPgDelta(ctx context.Context, source, target pgconn.Config, schema []str // on-disk catalog references used by declarative sync commands. formatOptions // is passed through as FORMAT_OPTIONS to the pg-delta script when non-empty. func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (string, error) { - env := []string{ - "TARGET=" + containerRef(targetRef), + result, err := DiffPgDeltaRefDetailed(ctx, sourceRef, targetRef, schema, formatOptions, options...) + if err != nil { + return "", err } - if len(sourceRef) > 0 { - env = append(env, "SOURCE="+containerRef(sourceRef)) + return result.SQL, nil +} + +// DiffPgDeltaRefDetailed is like DiffPgDeltaRef but also returns edge-runtime stderr. +func DiffPgDeltaRefDetailed(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (PgDeltaDiffResult, error) { + var env []string + var err error + targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { + return PgDeltaDiffResult{}, err } - if isPostgresURL(targetRef) { - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { - return "", err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) + if len(sourceRef) > 0 { + sourceRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) + if err != nil { + return PgDeltaDiffResult{}, err } } if len(schema) > 0 { @@ -100,6 +124,9 @@ func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []s if len(strings.TrimSpace(formatOptions)) > 0 { env = append(env, "FORMAT_OPTIONS="+formatOptions) } + if IsPgDeltaDebugEnabled() { + env = append(env, "PGDELTA_DEBUG=1") + } binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} if cwd, err := os.Getwd(); err == nil { binds = append(binds, cwd+":/workspace") @@ -107,11 +134,17 @@ func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []s var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaScript) if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error diffing schema", &stdout, &stderr); err != nil { - return "", err + return PgDeltaDiffResult{}, err } - return stdout.String(), nil + return PgDeltaDiffResult{ + SQL: stdout.String(), + Stderr: stderr.String(), + }, nil } +// exportCatalogPgDelta is overridden in tests to mock catalog export. +var exportCatalogPgDelta = ExportCatalogPgDelta + // DeclarativeExportPgDelta exports target schema as declarative file payloads // while keeping a config-based API for existing call sites. func DeclarativeExportPgDelta(ctx context.Context, source, target pgconn.Config, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (DeclarativeOutput, error) { @@ -121,17 +154,16 @@ func DeclarativeExportPgDelta(ctx context.Context, source, target pgconn.Config, // DeclarativeExportPgDeltaRef exports declarative file payloads using either // live URLs or catalog references as source/target inputs. func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (DeclarativeOutput, error) { - env := []string{ - "TARGET=" + containerRef(targetRef), + var env []string + var err error + targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { + return DeclarativeOutput{}, err } if len(sourceRef) > 0 { - env = append(env, "SOURCE="+containerRef(sourceRef)) - } - if isPostgresURL(targetRef) { - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { + sourceRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) + if err != nil { return DeclarativeOutput{}, err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) } } if len(schema) > 0 { @@ -140,6 +172,9 @@ func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef strin if len(strings.TrimSpace(formatOptions)) > 0 { env = append(env, "FORMAT_OPTIONS="+formatOptions) } + if IsPgDeltaDebugEnabled() { + env = append(env, "PGDELTA_DEBUG=1") + } binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} if cwd, err := os.Getwd(); err == nil { binds = append(binds, cwd+":/workspace") @@ -162,19 +197,15 @@ func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef strin // ExportCatalogPgDelta snapshots a database/catalog into serialized pg-delta // catalog JSON so later operations can diff without reconnecting. func ExportCatalogPgDelta(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { - env := []string{ - "TARGET=" + targetRef, + var env []string + var err error + targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { + return "", err } if len(role) > 0 { env = append(env, "ROLE="+role) } - if isPostgresURL(targetRef) { - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { - return "", err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) - } - } binds := []string{ utils.EdgeRuntimeId + ":/root/.cache/deno:rw", } diff --git a/apps/cli-go/internal/db/diff/pgdelta_debug.go b/apps/cli-go/internal/db/diff/pgdelta_debug.go new file mode 100644 index 0000000000..b901edd5f7 --- /dev/null +++ b/apps/cli-go/internal/db/diff/pgdelta_debug.go @@ -0,0 +1,85 @@ +package diff + +import ( + "encoding/json" + "os" + "strings" +) + +// IsPgDeltaDebugEnabled reports whether pg-delta diagnostic output is requested. +// Unlike --debug, this does not disable SSL for remote Postgres connections. +func IsPgDeltaDebugEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("PGDELTA_DEBUG"))) { + case "1", "true", "yes": + return true + default: + return false + } +} + +// PgDeltaDiffResult holds pg-delta diff output and edge-runtime stderr. +type PgDeltaDiffResult struct { + SQL string + Stderr string +} + +// PgDeltaDebugCapture holds artifacts collected during a pg-delta shadow diff. +type PgDeltaDebugCapture struct { + SourceCatalog string + Stderr string +} + +// DatabaseDiff is the result of diffing a target database against a shadow baseline. +type DatabaseDiff struct { + SQL string + Debug *PgDeltaDebugCapture +} + +// CatalogSummary summarizes object counts extracted from a pg-delta catalog JSON blob. +type CatalogSummary struct { + TotalObjects int + BySchema map[string]int +} + +// SummarizeCatalogJSON best-effort counts catalog objects grouped by schema name. +func SummarizeCatalogJSON(catalogJSON string) CatalogSummary { + summary := CatalogSummary{BySchema: map[string]int{}} + if len(strings.TrimSpace(catalogJSON)) == 0 { + return summary + } + var root any + if err := json.Unmarshal([]byte(catalogJSON), &root); err != nil { + return summary + } + walkCatalogObjects(root, summary.BySchema, &summary.TotalObjects) + return summary +} + +func walkCatalogObjects(node any, bySchema map[string]int, total *int) { + switch value := node.(type) { + case map[string]any: + if schema, ok := schemaNameFromCatalogNode(value); ok { + *total++ + bySchema[schema]++ + } + for _, child := range value { + walkCatalogObjects(child, bySchema, total) + } + case []any: + for _, child := range value { + walkCatalogObjects(child, bySchema, total) + } + } +} + +func schemaNameFromCatalogNode(node map[string]any) (string, bool) { + if schema, ok := node["schema"].(string); ok && len(schema) > 0 { + return schema, true + } + if schemaObj, ok := node["schema"].(map[string]any); ok { + if name, ok := schemaObj["name"].(string); ok && len(name) > 0 { + return name, true + } + } + return "", false +} diff --git a/apps/cli-go/internal/db/diff/pgdelta_debug_test.go b/apps/cli-go/internal/db/diff/pgdelta_debug_test.go new file mode 100644 index 0000000000..a8c0d47763 --- /dev/null +++ b/apps/cli-go/internal/db/diff/pgdelta_debug_test.go @@ -0,0 +1,50 @@ +package diff + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsPgDeltaDebugEnabled(t *testing.T) { + t.Run("disabled by default", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "") + assert.False(t, IsPgDeltaDebugEnabled()) + }) + + t.Run("enabled for 1", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "1") + assert.True(t, IsPgDeltaDebugEnabled()) + }) + + t.Run("enabled for true", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "true") + assert.True(t, IsPgDeltaDebugEnabled()) + }) + + t.Run("enabled for yes", func(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "YES") + assert.True(t, IsPgDeltaDebugEnabled()) + }) +} + +func TestSummarizeCatalogJSON(t *testing.T) { + t.Run("counts schema objects", func(t *testing.T) { + catalog := `{ + "schemas": [ + {"schema": "public", "tables": [{"schema": "public", "name": "airports"}]}, + {"schema": "auth", "tables": [{"schema": "auth", "name": "users"}]} + ] + }` + summary := SummarizeCatalogJSON(catalog) + assert.Equal(t, 4, summary.TotalObjects) + assert.Equal(t, 2, summary.BySchema["public"]) + assert.Equal(t, 2, summary.BySchema["auth"]) + }) + + t.Run("returns empty summary for invalid json", func(t *testing.T) { + summary := SummarizeCatalogJSON("{not-json") + assert.Equal(t, 0, summary.TotalObjects) + assert.Empty(t, summary.BySchema) + }) +} diff --git a/apps/cli-go/internal/db/diff/templates/pgdelta.ts b/apps/cli-go/internal/db/diff/templates/pgdelta.ts index cb5359566b..cccd1bc95b 100644 --- a/apps/cli-go/internal/db/diff/templates/pgdelta.ts +++ b/apps/cli-go/internal/db/diff/templates/pgdelta.ts @@ -2,8 +2,8 @@ import { createPlan, deserializeCatalog, formatSqlStatements, -} from "npm:@supabase/pg-delta@1.0.0-alpha.20"; -import { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase"; +} from "npm:@supabase/pg-delta@1.0.0-alpha.25"; +import { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.25/integrations/supabase"; async function resolveInput(ref: string | undefined) { if (!ref) { @@ -41,12 +41,26 @@ try { const result = await createPlan( await resolveInput(source), await resolveInput(target), - supabase, + { + ...supabase, + skipDefaultPrivilegeSubtraction: true, + }, ); let statements = result?.plan.statements ?? []; if (formatOptions != null) { statements = formatSqlStatements(statements, formatOptions); } + if (Deno.env.get("PGDELTA_DEBUG")) { + console.error( + JSON.stringify({ + statementCount: statements.length, + source: source ? "connected" : "null", + target: target ? "connected" : "null", + includedSchemas: includedSchemas ?? null, + skipDefaultPrivilegeSubtraction: true, + }), + ); + } for (const sql of statements) { console.log(`${sql};`); } diff --git a/apps/cli-go/internal/db/pgcache/cache.go b/apps/cli-go/internal/db/pgcache/cache.go index aeb1ebfbce..ea9bb202aa 100644 --- a/apps/cli-go/internal/db/pgcache/cache.go +++ b/apps/cli-go/internal/db/pgcache/cache.go @@ -246,12 +246,11 @@ func pgDeltaTempPath() string { } func exportCatalog(ctx context.Context, targetRef string, options ...func(*pgx.ConnConfig)) (string, error) { - env := []string{"TARGET=" + targetRef, "ROLE=postgres"} - if ca, err := types.GetRootCA(ctx, targetRef, options...); err != nil { + preparedRef, sslEnv, err := types.PreparePgDeltaPostgresRef(ctx, targetRef, types.PgDeltaTargetSSLRootCert, options...) + if err != nil { return "", err - } else if len(ca) > 0 { - env = append(env, "PGDELTA_TARGET_SSLROOTCERT="+ca) } + env := append([]string{"TARGET=" + preparedRef, "ROLE=postgres"}, sslEnv...) binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaCatalogExportTS) diff --git a/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go b/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go new file mode 100644 index 0000000000..10881cc939 --- /dev/null +++ b/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go @@ -0,0 +1,113 @@ +package pull + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + + "github.com/go-errors/errors" + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/spf13/afero" + "github.com/supabase/cli/internal/db/declarative" + "github.com/supabase/cli/internal/db/diff" + "github.com/supabase/cli/internal/utils" +) + +var exportCatalogPgDelta = diff.ExportCatalogPgDelta + +func saveEmptyPgDeltaPullDebug( + ctx context.Context, + config pgconn.Config, + capture *diff.PgDeltaDebugCapture, + fsys afero.Fs, + options ...func(*pgx.ConnConfig), +) (string, error) { + if capture == nil { + capture = &diff.PgDeltaDebugCapture{} + } + targetCatalog, err := exportCatalogPgDelta(ctx, utils.ToPostgresURL(config), "postgres", options...) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to export remote pg-delta catalog: %v\n", err) + } + bundle := declarative.DebugBundle{ + SourceCatalog: capture.SourceCatalog, + TargetCatalog: targetCatalog, + PgDeltaStderr: capture.Stderr, + ConnectionInfo: formatConnectionInfo(config), + Error: errors.New(errInSync), + } + debugDir, err := declarative.SaveDebugBundle(bundle, fsys) + if err != nil { + return "", err + } + printEmptyPgDeltaPullSummary(debugDir, capture.SourceCatalog, targetCatalog) + declarative.PrintDebugBundleMessage(debugDir) + return debugDir, nil +} + +func printEmptyPgDeltaPullSummary(debugDir, sourceCatalog, targetCatalog string) { + fmt.Fprintln(os.Stderr, "pg-delta returned 0 statements.") + fmt.Fprintln(os.Stderr, "Debug bundle saved to "+utils.Bold(debugDir)) + if len(strings.TrimSpace(sourceCatalog)) > 0 { + fmt.Fprintln(os.Stderr, formatCatalogSummary("Shadow", diff.SummarizeCatalogJSON(sourceCatalog))+ + fmt.Sprintf(" (%s)", formatByteSize(len(sourceCatalog)))) + } + if len(strings.TrimSpace(targetCatalog)) > 0 { + fmt.Fprintln(os.Stderr, formatCatalogSummary("Remote", diff.SummarizeCatalogJSON(targetCatalog))+ + fmt.Sprintf(" (%s)", formatByteSize(len(targetCatalog)))) + } else { + fmt.Fprintln(os.Stderr, "Remote catalog: export failed or empty (inspect connection.txt and pgdelta-stderr.txt)") + } +} + +func formatConnectionInfo(config pgconn.Config) string { + return fmt.Sprintf( + "host=%s port=%d user=%s database=%s url=%s", + config.Host, + config.Port, + config.User, + config.Database, + redactPostgresURL(utils.ToPostgresURL(config)), + ) +} + +func redactPostgresURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + if parsed.User != nil { + username := parsed.User.Username() + if username == "" { + parsed.User = url.UserPassword("redacted", "xxxxx") + } else { + parsed.User = url.UserPassword(username, "xxxxx") + } + } + return parsed.String() +} + +func formatCatalogSummary(label string, summary diff.CatalogSummary) string { + if summary.TotalObjects == 0 { + return label + " catalog: no objects detected" + } + parts := make([]string, 0, len(summary.BySchema)) + for schema, count := range summary.BySchema { + parts = append(parts, fmt.Sprintf("%s=%d", schema, count)) + } + return fmt.Sprintf("%s catalog: %d objects (%s)", label, summary.TotalObjects, strings.Join(parts, ", ")) +} + +func formatByteSize(size int) string { + switch { + case size >= 1<<20: + return fmt.Sprintf("%.1f MB", float64(size)/(1<<20)) + case size >= 1<<10: + return fmt.Sprintf("%.1f KB", float64(size)/(1<<10)) + default: + return fmt.Sprintf("%d B", size) + } +} diff --git a/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go b/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go new file mode 100644 index 0000000000..6a25edb1a4 --- /dev/null +++ b/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go @@ -0,0 +1,96 @@ +package pull + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/internal/db/diff" + "github.com/supabase/cli/internal/utils" +) + +func TestSaveEmptyPgDeltaPullDebug(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "1") + fsys := afero.NewMemMapFs() + original := exportCatalogPgDelta + t.Cleanup(func() { + exportCatalogPgDelta = original + }) + exportCatalogPgDelta = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { + return `{"schema":"public","name":"airports"}`, nil + } + config := pgconn.Config{ + Host: "db.example.supabase.co", + Port: 5432, + User: "postgres", + Password: "secret", + Database: "postgres", + } + capture := &diff.PgDeltaDebugCapture{ + SourceCatalog: `{"schema":"public","name":"roles"}`, + Stderr: `{"statementCount":0}`, + } + debugDir, err := saveEmptyPgDeltaPullDebug(context.Background(), config, capture, fsys) + require.NoError(t, err) + require.NotEmpty(t, debugDir) + + sourcePath := filepath.Join(debugDir, "source-catalog.json") + targetPath := filepath.Join(debugDir, "target-catalog.json") + stderrPath := filepath.Join(debugDir, "pgdelta-stderr.txt") + connectionPath := filepath.Join(debugDir, "connection.txt") + errorPath := filepath.Join(debugDir, "error.txt") + + source, err := afero.ReadFile(fsys, sourcePath) + require.NoError(t, err) + assert.Contains(t, string(source), `"roles"`) + + target, err := afero.ReadFile(fsys, targetPath) + require.NoError(t, err) + assert.Contains(t, string(target), `"airports"`) + + stderr, err := afero.ReadFile(fsys, stderrPath) + require.NoError(t, err) + assert.Contains(t, string(stderr), `"statementCount":0`) + + connection, err := afero.ReadFile(fsys, connectionPath) + require.NoError(t, err) + assert.Contains(t, string(connection), "db.example.supabase.co") + assert.NotContains(t, string(connection), "secret") + + errorText, err := afero.ReadFile(fsys, errorPath) + require.NoError(t, err) + assert.Contains(t, string(errorText), "No schema changes found") +} + +func TestSaveEmptyPgDeltaPullDebugUsesTempDir(t *testing.T) { + fsys := afero.NewMemMapFs() + original := exportCatalogPgDelta + t.Cleanup(func() { + exportCatalogPgDelta = original + }) + exportCatalogPgDelta = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { + return `{}`, nil + } + debugDir, err := saveEmptyPgDeltaPullDebug(context.Background(), pgconn.Config{}, &diff.PgDeltaDebugCapture{}, fsys) + require.NoError(t, err) + assert.Contains(t, debugDir, filepath.Join(utils.TempDir, "pgdelta", "debug")) +} + +func TestDiffRemoteSchemaEmptyWithoutDebug(t *testing.T) { + t.Setenv("PGDELTA_DEBUG", "") + fsys := afero.NewMemMapFs() + existsBefore, err := afero.Exists(fsys, filepath.Join(utils.TempDir, "pgdelta")) + require.NoError(t, err) + assert.False(t, existsBefore) + + // saveEmptyPgDeltaPullDebug should not run when env is unset; verify gate directly. + assert.False(t, diff.IsPgDeltaDebugEnabled()) + _, err = os.Stat(filepath.Join(utils.TempDir, "pgdelta", "debug")) + assert.Error(t, err) +} diff --git a/apps/cli-go/internal/db/pull/pull.go b/apps/cli-go/internal/db/pull/pull.go index 758cb74810..e536c0e990 100644 --- a/apps/cli-go/internal/db/pull/pull.go +++ b/apps/cli-go/internal/db/pull/pull.go @@ -60,7 +60,10 @@ func Run(ctx context.Context, schema []string, config pgconn.Config, name string // 2. Pull schema timestamp := utils.GetCurrentTimestamp() path := new.GetMigrationPath(timestamp, name) - if err := run(ctx, schema, path, conn, usePgDeltaDiff, differ, fsys); err != nil { + if err := run(ctx, schema, path, conn, usePgDeltaDiff, differ, fsys, options...); err != nil { + return err + } + if err := ensureMigrationWritten(fsys, path); err != nil { return err } // 3. Insert a row to `schema_migrations` @@ -110,7 +113,7 @@ func pullDeclarativePgDelta(ctx context.Context, schema []string, config pgconn. return nil } -func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs) error { +func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { config := conn.Config().Config // 1. Assert `supabase/migrations` and `schema_migrations` are in sync. if err := assertRemoteInSync(ctx, conn, fsys); errors.Is(err, errMissing) { @@ -128,15 +131,13 @@ func run(ctx context.Context, schema []string, path string, conn *pgx.Conn, useP // For the legacy path this is a second pass that captures changes // pg_dump cannot emit (default privileges, managed schemas). For the // pg-delta path this is the only pass and produces the full schema. - if err = diffRemoteSchema(ctx, nil, path, config, usePgDeltaDiff, differ, fsys); errors.Is(err, errInSync) { - err = nil - } + err = swallowInitialInSync(diffRemoteSchema(ctx, nil, path, config, usePgDeltaDiff, differ, fsys, options...), fsys, path) return err } else if err != nil { return err } // 2. Fetch remote schema changes - return diffRemoteSchema(ctx, schema, path, config, usePgDeltaDiff, differ, fsys) + return diffRemoteSchema(ctx, schema, path, config, usePgDeltaDiff, differ, fsys, options...) } func dumpRemoteSchema(ctx context.Context, path string, config pgconn.Config, fsys afero.Fs) error { @@ -153,13 +154,21 @@ func dumpRemoteSchema(ctx context.Context, path string, config pgconn.Config, fs return migration.DumpSchema(ctx, config, f, dump.DockerExec) } -func diffRemoteSchema(ctx context.Context, schema []string, path string, config pgconn.Config, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs) error { +func diffRemoteSchema(ctx context.Context, schema []string, path string, config pgconn.Config, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { // Diff remote db (source) & shadow db (target) and write it as a new migration. - output, err := diff.DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDeltaDiff) + result, err := diff.DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDeltaDiff, options...) if err != nil { return err } + output := result.SQL if trimmed := strings.TrimSpace(output); len(trimmed) == 0 { + if usePgDeltaDiff && diff.IsPgDeltaDebugEnabled() { + if debugDir, debugErr := saveEmptyPgDeltaPullDebug(ctx, config, result.Debug, fsys, options...); debugErr != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to save pg-delta debug bundle: %v\n", debugErr) + } else if len(debugDir) > 0 { + return errors.Errorf("%w (debug bundle: %s)", errInSync, debugDir) + } + } return errors.New(errInSync) } if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(path)); err != nil { @@ -227,6 +236,25 @@ func assertRemoteInSync(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) erro return nil } +func hasMigrationContent(fsys afero.Fs, path string) bool { + info, err := fsys.Stat(path) + return err == nil && info.Size() > 0 +} + +func swallowInitialInSync(err error, fsys afero.Fs, path string) error { + if errors.Is(err, errInSync) && hasMigrationContent(fsys, path) { + return nil + } + return err +} + +func ensureMigrationWritten(fsys afero.Fs, path string) error { + if hasMigrationContent(fsys, path) { + return nil + } + return errors.New(errInSync) +} + func suggestMigrationRepair(extraRemote, extraLocal []string) string { result := fmt.Sprintln("\nMake sure your local git repo is up-to-date. If the error persists, try repairing the migration history table:") for _, version := range extraRemote { diff --git a/apps/cli-go/internal/db/pull/pull_test.go b/apps/cli-go/internal/db/pull/pull_test.go index 19a1161850..e9fbae9198 100644 --- a/apps/cli-go/internal/db/pull/pull_test.go +++ b/apps/cli-go/internal/db/pull/pull_test.go @@ -136,6 +136,43 @@ func TestPullSchema(t *testing.T) { }) } +func TestInitialPullInSync(t *testing.T) { + fsys := afero.NewMemMapFs() + path := "0_test.sql" + + t.Run("swallows errInSync when pg_dump already wrote migration content", func(t *testing.T) { + require.NoError(t, afero.WriteFile(fsys, path, []byte("create table t(id int);"), 0644)) + err := swallowInitialInSync(errInSync, fsys, path) + assert.NoError(t, err) + }) + + t.Run("returns errInSync for pg-delta initial pull with no migration file", func(t *testing.T) { + err := swallowInitialInSync(errInSync, fsys, "missing.sql") + assert.ErrorIs(t, err, errInSync) + }) + + t.Run("returns errInSync when migration file is empty", func(t *testing.T) { + require.NoError(t, afero.WriteFile(fsys, "empty.sql", []byte{}, 0644)) + err := swallowInitialInSync(errInSync, fsys, "empty.sql") + assert.ErrorIs(t, err, errInSync) + }) +} + +func TestEnsureMigrationWritten(t *testing.T) { + fsys := afero.NewMemMapFs() + + t.Run("passes when migration file has content", func(t *testing.T) { + path := "0_test.sql" + require.NoError(t, afero.WriteFile(fsys, path, []byte("create table t(id int);"), 0644)) + assert.NoError(t, ensureMigrationWritten(fsys, path)) + }) + + t.Run("returns errInSync when migration file is missing", func(t *testing.T) { + err := ensureMigrationWritten(fsys, "missing.sql") + assert.ErrorIs(t, err, errInSync) + }) +} + func TestSyncRemote(t *testing.T) { t.Run("throws error on permission denied", func(t *testing.T) { // Setup in-memory fs diff --git a/apps/cli-go/internal/gen/types/pgdelta_conn.go b/apps/cli-go/internal/gen/types/pgdelta_conn.go new file mode 100644 index 0000000000..994eecc3b9 --- /dev/null +++ b/apps/cli-go/internal/gen/types/pgdelta_conn.go @@ -0,0 +1,114 @@ +package types + +import ( + "context" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/jackc/pgx/v4" +) + +const ( + PgDeltaSourceSSLRootCert = "PGDELTA_SOURCE_SSLROOTCERT" + PgDeltaTargetSSLRootCert = "PGDELTA_TARGET_SSLROOTCERT" + pgDeltaCABundleRelPath = "supabase/.temp/pgdelta/supabase-ca-bundle.crt" +) + +func isPostgresURL(ref string) bool { + return strings.HasPrefix(ref, "postgres://") || strings.HasPrefix(ref, "postgresql://") +} + +func isSupabaseHostedPostgresURL(dbURL string) bool { + parsed, err := url.Parse(dbURL) + if err != nil { + return false + } + host := strings.ToLower(parsed.Hostname()) + return strings.HasSuffix(host, ".supabase.co") || + strings.Contains(host, "pooler.supabase.com") +} + +// pgDeltaRootCA returns the CA bundle pg-delta should use for a Postgres URL. +// Supabase-hosted databases always receive the embedded bundle even when the +// SSL probe is skipped (for example in --debug mode). +func pgDeltaRootCA(ctx context.Context, dbURL string, options ...func(*pgx.ConnConfig)) (string, error) { + ca, err := GetRootCA(ctx, dbURL, options...) + if err != nil { + return "", err + } + if len(ca) > 0 { + return ca, nil + } + if isSupabaseHostedPostgresURL(dbURL) { + return caStaging + caProd + caSnap, nil + } + return "", nil +} + +// PreparePgDeltaPostgresRef configures a Postgres URL and env vars for pg-delta. +// +// pg-delta disables TLS when sslmode is absent and only reads PGDELTA_*_SSLROOTCERT +// for verify-ca/verify-full. Remote Supabase databases require verify-ca plus a +// CA bundle written into the workspace so edge-runtime can read it from disk. +func PreparePgDeltaPostgresRef( + ctx context.Context, + ref string, + sslRootCertEnv string, + options ...func(*pgx.ConnConfig), +) (string, []string, error) { + if !isPostgresURL(ref) { + return ref, nil, nil + } + ca, err := pgDeltaRootCA(ctx, ref, options...) + if err != nil { + return "", nil, err + } + if len(ca) == 0 { + return ref, nil, nil + } + containerCertPath, err := writePgDeltaCABundleFile(ca) + if err != nil { + return "", nil, err + } + return ensurePgDeltaSSL(ref, containerCertPath), []string{sslRootCertEnv + "=" + ca}, nil +} + +func writePgDeltaCABundleFile(ca string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + abs := filepath.Join(cwd, pgDeltaCABundleRelPath) + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return "", err + } + if err := os.WriteFile(abs, []byte(ca), 0o644); err != nil { + return "", err + } + return "/workspace/" + filepath.ToSlash(pgDeltaCABundleRelPath), nil +} + +func ensurePgDeltaSSL(dbURL, sslrootcertPath string) string { + parsed, err := url.Parse(dbURL) + if err != nil { + return dbURL + } + query := parsed.Query() + switch query.Get("sslmode") { + case "verify-ca", "verify-full": + default: + query.Set("sslmode", "verify-ca") + } + if len(sslrootcertPath) > 0 { + query.Set("sslrootcert", sslrootcertPath) + } + parsed.RawQuery = query.Encode() + return parsed.String() +} + +// EnsurePgDeltaVerifyCA is kept for tests that assert URL sslmode behaviour. +func EnsurePgDeltaVerifyCA(dbURL string) string { + return ensurePgDeltaSSL(dbURL, "") +} diff --git a/apps/cli-go/internal/gen/types/pgdelta_conn_test.go b/apps/cli-go/internal/gen/types/pgdelta_conn_test.go new file mode 100644 index 0000000000..eb29a279ef --- /dev/null +++ b/apps/cli-go/internal/gen/types/pgdelta_conn_test.go @@ -0,0 +1,53 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEnsurePgDeltaVerifyCA(t *testing.T) { + t.Run("adds verify-ca when sslmode is absent", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?connect_timeout=10" + got := EnsurePgDeltaVerifyCA(input) + assert.Contains(t, got, "sslmode=verify-ca") + assert.Contains(t, got, "connect_timeout=10") + }) + + t.Run("preserves existing verify-ca", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=verify-ca" + assert.Equal(t, input, EnsurePgDeltaVerifyCA(input)) + }) + + t.Run("preserves existing verify-full", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=verify-full" + assert.Equal(t, input, EnsurePgDeltaVerifyCA(input)) + }) + + t.Run("replaces require with verify-ca", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=require" + got := EnsurePgDeltaVerifyCA(input) + assert.Contains(t, got, "sslmode=verify-ca") + assert.NotContains(t, got, "sslmode=require") + }) +} + +func TestEnsurePgDeltaSSLAddsRootCertPath(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?connect_timeout=10" + got := ensurePgDeltaSSL(input, "/workspace/supabase/.temp/pgdelta/supabase-ca-bundle.crt") + assert.Contains(t, got, "sslmode=verify-ca") + assert.Contains(t, got, "sslrootcert=%2Fworkspace%2Fsupabase%2F.temp%2Fpgdelta%2Fsupabase-ca-bundle.crt") +} + +func TestIsSupabaseHostedPostgresURL(t *testing.T) { + assert.True(t, isSupabaseHostedPostgresURL("postgresql://postgres@db.ref.supabase.co:5432/postgres")) + assert.True(t, isSupabaseHostedPostgresURL("postgresql://supabase_admin@aws-0-us-east-2.pooler.supabase.com:5432/postgres")) + assert.False(t, isSupabaseHostedPostgresURL("postgresql://postgres@localhost:5432/postgres")) +} + +func TestPreparePgDeltaPostgresRefNonPostgres(t *testing.T) { + ref, env, err := PreparePgDeltaPostgresRef(t.Context(), "supabase/.temp/catalog.json", PgDeltaTargetSSLRootCert) + assert.NoError(t, err) + assert.Equal(t, "supabase/.temp/catalog.json", ref) + assert.Empty(t, env) +} diff --git a/apps/cli/package.json b/apps/cli/package.json index 0f5041fb34..99877f1af5 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -24,7 +24,8 @@ "access": "public" }, "scripts": { - "build": "pnpm build:next && pnpm build:legacy && pnpm build:shim", + "build": "pnpm build:go-sidecar && pnpm build:next && pnpm build:legacy && pnpm build:shim", + "build:go-sidecar": "cp ../cli-go/supabase-go dist/supabase-go", "build:next": "bun build src/next/main.ts --compile --outfile dist/supabase-next", "build:legacy": "bun build src/legacy/main.ts --compile --outfile dist/supabase-legacy", "build:shim": "bun build src/shared/cli/bin.ts --outfile dist/supabase.js --target node", From b8635d417933b5f98a46cbcceb712cd763e5bdc2 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 21 May 2026 21:22:30 +0200 Subject: [PATCH 03/10] feat(cli): enhance local pg-delta testing support Added documentation for testing local pg-delta builds in CONTRIBUTING.md, detailing the steps to publish local changes and configure the CLI to use a local npm registry. Updated the RunEdgeRuntimeScript function to accept additional options for handling npm registry configurations, allowing for scoped .npmrc files and environment variable forwarding. Introduced PgDeltaNpmRegistryOption to manage npm registry settings and added tests to ensure correct behavior. Refactored related functions to integrate these enhancements, improving the overall local development experience for pg-delta. --- apps/cli-go/CONTRIBUTING.md | 49 ++++++++++++ apps/cli-go/internal/db/diff/diff_test.go | 8 +- apps/cli-go/internal/db/diff/pgdelta.go | 6 +- apps/cli-go/internal/db/pgcache/cache.go | 2 +- apps/cli-go/internal/pgdelta/apply.go | 2 +- apps/cli-go/internal/utils/edgeruntime.go | 78 +++++++++++++++++-- .../cli-go/internal/utils/edgeruntime_test.go | 47 +++++++++++ apps/cli-go/internal/utils/pgdelta_local.go | 44 +++++++++++ .../internal/utils/pgdelta_local_test.go | 63 +++++++++++++++ apps/cli-go/pkg/config/config_test.go | 2 +- apps/cli-go/pkg/config/pgdelta_local.go | 14 ++++ 11 files changed, 299 insertions(+), 16 deletions(-) create mode 100644 apps/cli-go/internal/utils/edgeruntime_test.go create mode 100644 apps/cli-go/internal/utils/pgdelta_local.go create mode 100644 apps/cli-go/internal/utils/pgdelta_local_test.go create mode 100644 apps/cli-go/pkg/config/pgdelta_local.go diff --git a/apps/cli-go/CONTRIBUTING.md b/apps/cli-go/CONTRIBUTING.md index 00473f3139..39d0d33d85 100644 --- a/apps/cli-go/CONTRIBUTING.md +++ b/apps/cli-go/CONTRIBUTING.md @@ -41,3 +41,52 @@ go test ./... -race -v -count=1 -failfast ## API client The Supabase API client is generated from OpenAPI spec. See [our guide](api/README.md) for updating the client and types. + +## Testing local pg-delta builds + +To exercise unpublished `@supabase/pg-delta` changes inside CLI edge-runtime scripts (`db pull`, `db diff`, `db push`, etc.), publish a local build via Verdaccio in [pg-toolbelt](https://github.com/supabase/pg-toolbelt) and point the CLI at that registry. + +### 1. Start Verdaccio (pg-toolbelt) + +```sh +cd pg-toolbelt +bun run verdaccio:start +``` + +Verdaccio listens on `http://localhost:4873`. `@supabase/*` packages you publish locally are served from local storage; other `@supabase/*` dependencies (for example `@supabase/pg-topo`) are proxied to npmjs. + +### 2. Publish a local pg-delta build + +After changing `packages/pg-delta`: + +```sh +bun run pg-delta:publish-local \ + --write-version-to=/path/to/test-project/supabase/.temp/pgdelta-version +``` + +This publishes a fresh `0.0.0-local.` version and restores `package.json` afterward. The version file tells the CLI which npm version to request (`EffectivePgDeltaNpmVersion`). + +Re-run whenever you change pg-delta source. + +### 3. Run the CLI against the local registry + +Set `PGDELTA_NPM_REGISTRY` to a URL reachable **from inside the edge-runtime Docker container**: + +```sh +# Docker Desktop (macOS / Windows) +export PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 + +# Linux (Docker 20.10+) +export PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 +# or: export PGDELTA_NPM_REGISTRY=http://172.17.0.1:4873 +``` + +Then run any pg-delta-backed command, for example: + +```sh +supabase db pull --db-url "$DATABASE_URL" --diff-engine pg-delta +``` + +When set, the CLI injects a scoped `.npmrc` and forwards `NPM_CONFIG_REGISTRY` into the edge-runtime container (`PgDeltaNpmRegistryOption` in `internal/utils/pgdelta_local.go`). + +Unset `PGDELTA_NPM_REGISTRY` to return to the npmjs version pinned in config / `supabase/.temp/pgdelta-version`. diff --git a/apps/cli-go/internal/db/diff/diff_test.go b/apps/cli-go/internal/db/diff/diff_test.go index f14c6a9f73..fb49df6ff8 100644 --- a/apps/cli-go/internal/db/diff/diff_test.go +++ b/apps/cli-go/internal/db/diff/diff_test.go @@ -205,7 +205,7 @@ func TestDiffDatabase(t *testing.T) { // Run test result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorIs(t, err, errNetwork) assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -236,7 +236,7 @@ func TestDiffDatabase(t *testing.T) { // Run test result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorContains(t, err, "test-shadow-db container is not running: exited") assert.Empty(t, apitest.ListUnmatchedRequests()) }) @@ -268,7 +268,7 @@ func TestDiffDatabase(t *testing.T) { // Run test result, err := DiffDatabase(context.Background(), []string{"public"}, dbConfig, io.Discard, fsys, DiffSchemaMigra, false, conn.Intercept) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorContains(t, err, `ERROR: schema "public" already exists (SQLSTATE 42P06) At statement: 0 create schema public`) @@ -333,7 +333,7 @@ create schema public`) } }) // Check error - assert.Empty(t, result.SQL) + assert.Empty(t, result) assert.ErrorContains(t, err, "error diffing schema") assert.Empty(t, apitest.ListUnmatchedRequests()) }) diff --git a/apps/cli-go/internal/db/diff/pgdelta.go b/apps/cli-go/internal/db/diff/pgdelta.go index 787e56cb33..d4a63e1634 100644 --- a/apps/cli-go/internal/db/diff/pgdelta.go +++ b/apps/cli-go/internal/db/diff/pgdelta.go @@ -133,7 +133,7 @@ func DiffPgDeltaRefDetailed(ctx context.Context, sourceRef, targetRef string, sc } var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error diffing schema", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error diffing schema", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return PgDeltaDiffResult{}, err } return PgDeltaDiffResult{ @@ -181,7 +181,7 @@ func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef strin } var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaDeclarativeExportScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting declarative schema", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting declarative schema", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return DeclarativeOutput{}, err } if stdout.Len() == 0 { @@ -214,7 +214,7 @@ func ExportCatalogPgDelta(ctx context.Context, targetRef, role string, options . } var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaCatalogExportScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return "", err } snapshot := strings.TrimSpace(stdout.String()) diff --git a/apps/cli-go/internal/db/pgcache/cache.go b/apps/cli-go/internal/db/pgcache/cache.go index ea9bb202aa..5b518464ba 100644 --- a/apps/cli-go/internal/db/pgcache/cache.go +++ b/apps/cli-go/internal/db/pgcache/cache.go @@ -254,7 +254,7 @@ func exportCatalog(ctx context.Context, targetRef string, options ...func(*pgx.C binds := []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"} var stdout, stderr bytes.Buffer script := config.InterpolatePgDeltaScript(config.Config(&utils.Config), pgDeltaCatalogExportTS) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error exporting pg-delta catalog", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return "", err } snapshot := strings.TrimSpace(stdout.String()) diff --git a/apps/cli-go/internal/pgdelta/apply.go b/apps/cli-go/internal/pgdelta/apply.go index f9009a3202..22fca756a6 100644 --- a/apps/cli-go/internal/pgdelta/apply.go +++ b/apps/cli-go/internal/pgdelta/apply.go @@ -323,7 +323,7 @@ func ApplyDeclarative(ctx context.Context, config pgconn.Config, fsys afero.Fs) fmt.Fprintln(os.Stderr, "Applying declarative schemas via pg-delta...") var stdout, stderr bytes.Buffer script := pkgconfig.InterpolatePgDeltaScript(pkgconfig.Config(&utils.Config), pgDeltaDeclarativeApplyScript) - if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error running pg-delta script", &stdout, &stderr); err != nil { + if err := utils.RunEdgeRuntimeScript(ctx, env, script, binds, "error running pg-delta script", &stdout, &stderr, utils.PgDeltaNpmRegistryOption()); err != nil { return err } diff --git a/apps/cli-go/internal/utils/edgeruntime.go b/apps/cli-go/internal/utils/edgeruntime.go index 06a42b464c..7708a15511 100644 --- a/apps/cli-go/internal/utils/edgeruntime.go +++ b/apps/cli-go/internal/utils/edgeruntime.go @@ -3,6 +3,7 @@ package utils import ( "bytes" "context" + "fmt" "strings" "github.com/docker/docker/api/types/container" @@ -11,23 +12,67 @@ import ( "github.com/spf13/viper" ) +// edgeRuntimeFile is a single file dropped into the edge-runtime container's +// working directory before the configured command is run. +type edgeRuntimeFile struct { + name string + content string +} + +// edgeRuntimeOptions accumulates the optional inputs assembled by +// EdgeRuntimeOption functions and consumed by RunEdgeRuntimeScript. +type edgeRuntimeOptions struct { + extraFiles []edgeRuntimeFile + extraEnv []string +} + +// EdgeRuntimeOption customizes a RunEdgeRuntimeScript invocation. The current +// shape (extra files dropped alongside index.ts, extra container env vars) +// covers the local-pg-delta use case; extend the option struct as new needs +// arrive instead of adding more positional arguments. +type EdgeRuntimeOption func(*edgeRuntimeOptions) + +// WithExtraFile schedules an extra file alongside `index.ts` in the container. +// Useful for project-local config files (e.g. `.npmrc`, `deno.json`) that need +// to live next to the script Deno is asked to run. +func WithExtraFile(name, content string) EdgeRuntimeOption { + return func(o *edgeRuntimeOptions) { + o.extraFiles = append(o.extraFiles, edgeRuntimeFile{name: name, content: content}) + } +} + +// WithExtraEnv appends container env entries in `KEY=value` form. +func WithExtraEnv(entries ...string) EdgeRuntimeOption { + return func(o *edgeRuntimeOptions) { + o.extraEnv = append(o.extraEnv, entries...) + } +} + // RunEdgeRuntimeScript executes a TypeScript program inside the configured Edge // Runtime container and streams stdout/stderr back to the caller. -func RunEdgeRuntimeScript(ctx context.Context, env []string, script string, binds []string, errPrefix string, stdout, stderr *bytes.Buffer) error { +func RunEdgeRuntimeScript(ctx context.Context, env []string, script string, binds []string, errPrefix string, stdout, stderr *bytes.Buffer, opts ...EdgeRuntimeOption) error { + state := &edgeRuntimeOptions{} + for _, opt := range opts { + if opt != nil { + opt(state) + } + } cmd := []string{"edge-runtime", "start", "--main-service=."} if viper.GetBool("DEBUG") { cmd = append(cmd, "--verbose") } cmdString := strings.Join(cmd, " ") - entrypoint := []string{"sh", "-c", `cat <<'EOF' > index.ts && ` + cmdString + ` -` + script + ` -EOF -`} + files := append([]edgeRuntimeFile{{name: "index.ts", content: script}}, state.extraFiles...) + entrypoint := []string{"sh", "-c", buildEdgeRuntimeEntrypoint(files, cmdString)} + combinedEnv := env + if len(state.extraEnv) > 0 { + combinedEnv = append(append([]string{}, env...), state.extraEnv...) + } if err := DockerRunOnceWithConfig( ctx, container.Config{ Image: Config.EdgeRuntime.Image, - Env: env, + Env: combinedEnv, Entrypoint: entrypoint, }, container.HostConfig{ @@ -43,3 +88,24 @@ EOF } return nil } + +// buildEdgeRuntimeEntrypoint emits a `sh -c` body that writes each file via a +// here-document and then runs cmd. All heredoc openers are joined with `&&` +// before the bodies so bash stacks them in declaration order; each body is +// terminated with a unique sentinel so file contents can contain `EOF` safely. +func buildEdgeRuntimeEntrypoint(files []edgeRuntimeFile, cmd string) string { + if len(files) == 0 { + return cmd + "\n" + } + var head strings.Builder + var bodies strings.Builder + for i, f := range files { + sentinel := fmt.Sprintf("__EDGE_RT_FILE_%d__", i) + fmt.Fprintf(&head, "cat <<'%s' > %s && ", sentinel, f.name) + fmt.Fprintf(&bodies, "%s\n%s\n", f.content, sentinel) + } + head.WriteString(cmd) + head.WriteString("\n") + head.WriteString(bodies.String()) + return head.String() +} diff --git a/apps/cli-go/internal/utils/edgeruntime_test.go b/apps/cli-go/internal/utils/edgeruntime_test.go new file mode 100644 index 0000000000..3f2a74ada4 --- /dev/null +++ b/apps/cli-go/internal/utils/edgeruntime_test.go @@ -0,0 +1,47 @@ +package utils + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildEdgeRuntimeEntrypoint(t *testing.T) { + t.Run("emits a single heredoc when only the script is provided", func(t *testing.T) { + got := buildEdgeRuntimeEntrypoint( + []edgeRuntimeFile{{name: "index.ts", content: "console.log('hi')"}}, + "edge-runtime start --main-service=.", + ) + assert.True(t, strings.HasPrefix(got, "cat <<'__EDGE_RT_FILE_0__' > index.ts && edge-runtime start --main-service=.\n")) + assert.Contains(t, got, "console.log('hi')\n__EDGE_RT_FILE_0__\n") + }) + + t.Run("chains heredocs in declaration order so each cat reads the matching body", func(t *testing.T) { + got := buildEdgeRuntimeEntrypoint( + []edgeRuntimeFile{ + {name: "index.ts", content: "TS_CONTENT"}, + {name: ".npmrc", content: "NPMRC_CONTENT"}, + }, + "edge-runtime start --main-service=.", + ) + // Both cat declarations must come before any body, separated by &&. + assert.Contains(t, got, "cat <<'__EDGE_RT_FILE_0__' > index.ts && cat <<'__EDGE_RT_FILE_1__' > .npmrc && edge-runtime start --main-service=.") + // Bodies must follow in the same order as the declarations. + idxScript := strings.Index(got, "TS_CONTENT") + idxNpmrc := strings.Index(got, "NPMRC_CONTENT") + require.Greater(t, idxScript, 0) + require.Greater(t, idxNpmrc, idxScript, ".npmrc body must come after index.ts body") + // Sentinels close each body so user content containing `EOF` cannot + // terminate the heredoc early. + assert.Contains(t, got, "TS_CONTENT\n__EDGE_RT_FILE_0__") + assert.Contains(t, got, "NPMRC_CONTENT\n__EDGE_RT_FILE_1__") + assert.True(t, strings.HasSuffix(got, "\n")) + }) + + t.Run("returns just the command when no files are provided", func(t *testing.T) { + got := buildEdgeRuntimeEntrypoint(nil, "edge-runtime start --main-service=.") + assert.Equal(t, "edge-runtime start --main-service=.\n", got) + }) +} diff --git a/apps/cli-go/internal/utils/pgdelta_local.go b/apps/cli-go/internal/utils/pgdelta_local.go new file mode 100644 index 0000000000..50232b238e --- /dev/null +++ b/apps/cli-go/internal/utils/pgdelta_local.go @@ -0,0 +1,44 @@ +package utils + +import ( + "os" + "strings" + + "github.com/supabase/cli/pkg/config" +) + +// PgDeltaNpmRegistryOption returns an EdgeRuntimeOption that points the +// edge-runtime container at a user-controlled npm registry when +// PGDELTA_NPM_REGISTRY is set. It applies three coordinated overrides: +// +// 1. Writes a project-local `.npmrc` with a `@supabase`-scoped registry +// line. Deno honors `.npmrc` for scoped registries when discovered in +// the cwd or parents (Deno >= 1.39), so this keeps every non-`@supabase` +// npm specifier on npmjs. +// 2. Forwards the canonical `NPM_CONFIG_REGISTRY` env var into the +// container. This is the universal npm/Deno escape hatch — it routes +// every `npm:` specifier through the chosen registry regardless of +// whether the host runtime reads `.npmrc`. Verdaccio's `npmjs` uplink +// proxies any non-`@supabase` packages back to npmjs, so widening the +// scope is safe and protects us against edge-runtime image variants +// that ignore `.npmrc`. +// 3. Forwards `PGDELTA_NPM_REGISTRY` itself into the container. +// +// Returns nil when the env var is unset or whitespace-only, which makes it +// safe to pass unconditionally to RunEdgeRuntimeScript (nil options are +// ignored). +func PgDeltaNpmRegistryOption() EdgeRuntimeOption { + registry := strings.TrimSpace(os.Getenv(config.PgDeltaNpmRegistryEnv)) + if registry == "" { + return nil + } + npmrc := WithExtraFile(".npmrc", "@supabase:registry="+registry+"\n") + envFwd := WithExtraEnv( + config.PgDeltaNpmRegistryEnv+"="+registry, + "NPM_CONFIG_REGISTRY="+registry, + ) + return func(o *edgeRuntimeOptions) { + npmrc(o) + envFwd(o) + } +} diff --git a/apps/cli-go/internal/utils/pgdelta_local_test.go b/apps/cli-go/internal/utils/pgdelta_local_test.go new file mode 100644 index 0000000000..656e17c4b2 --- /dev/null +++ b/apps/cli-go/internal/utils/pgdelta_local_test.go @@ -0,0 +1,63 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/supabase/cli/pkg/config" +) + +func TestPgDeltaNpmRegistryOption(t *testing.T) { + t.Run("returns nil when PGDELTA_NPM_REGISTRY is unset", func(t *testing.T) { + t.Setenv(config.PgDeltaNpmRegistryEnv, "") + assert.Nil(t, PgDeltaNpmRegistryOption()) + }) + + t.Run("writes a scoped .npmrc and forwards both PGDELTA_NPM_REGISTRY and NPM_CONFIG_REGISTRY when set", func(t *testing.T) { + t.Setenv(config.PgDeltaNpmRegistryEnv, "http://host.docker.internal:4873") + opt := PgDeltaNpmRegistryOption() + require.NotNil(t, opt) + + state := &edgeRuntimeOptions{} + opt(state) + require.Len(t, state.extraFiles, 1) + assert.Equal(t, ".npmrc", state.extraFiles[0].name) + assert.Equal(t, + "@supabase:registry=http://host.docker.internal:4873\n", + state.extraFiles[0].content, + ) + // NPM_CONFIG_REGISTRY is the universal escape hatch for runtimes + // that ignore .npmrc (e.g. some supabase/edge-runtime variants); + // PGDELTA_NPM_REGISTRY is forwarded so scripts can read the configured + // registry URL when needed. + assert.Equal(t, + []string{ + "PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873", + "NPM_CONFIG_REGISTRY=http://host.docker.internal:4873", + }, + state.extraEnv, + ) + }) + + t.Run("trims surrounding whitespace from the registry URL", func(t *testing.T) { + t.Setenv(config.PgDeltaNpmRegistryEnv, " http://localhost:4873 ") + opt := PgDeltaNpmRegistryOption() + require.NotNil(t, opt) + + state := &edgeRuntimeOptions{} + opt(state) + require.Len(t, state.extraFiles, 1) + assert.Equal(t, + "@supabase:registry=http://localhost:4873\n", + state.extraFiles[0].content, + ) + assert.Equal(t, + []string{ + "PGDELTA_NPM_REGISTRY=http://localhost:4873", + "NPM_CONFIG_REGISTRY=http://localhost:4873", + }, + state.extraEnv, + ) + }) +} diff --git a/apps/cli-go/pkg/config/config_test.go b/apps/cli-go/pkg/config/config_test.go index d7bca3948d..06dd1e59a7 100644 --- a/apps/cli-go/pkg/config/config_test.go +++ b/apps/cli-go/pkg/config/config_test.go @@ -288,7 +288,7 @@ enabled = true t.Run("InterpolatePgDeltaScript substitutes placeholder", func(t *testing.T) { c := NewConfig() require.NoError(t, c.Load("", fs.MapFS{})) - // Embedded TS pins use this semver literal before InterpolatePgDeltaScript runs. + // Embedded TS pins use this semver literal before InterpolatePgDeltaScript runs. got := InterpolatePgDeltaScript(Config(&c), `from "npm:@supabase/pg-delta@1.0.0-alpha.20";`) assert.Equal(t, `from "npm:@supabase/pg-delta@`+DefaultPgDeltaNpmVersion+`";`, got) }) diff --git a/apps/cli-go/pkg/config/pgdelta_local.go b/apps/cli-go/pkg/config/pgdelta_local.go new file mode 100644 index 0000000000..683137f1b0 --- /dev/null +++ b/apps/cli-go/pkg/config/pgdelta_local.go @@ -0,0 +1,14 @@ +package config + +// PgDeltaNpmRegistryEnv is the env var that, when set to an npm registry URL +// reachable from the edge-runtime container, routes Deno's `npm:` resolution +// for `@supabase/pg-delta` through that registry instead of the public +// npmjs.org. Pair with the pg-toolbelt `bun run pg-delta:publish-local` script +// to iterate on local pg-delta changes without republishing to npmjs. +// +// See apps/cli-go/CONTRIBUTING.md#testing-local-pg-delta-builds for the +// Verdaccio workflow (CLI maintainers only). +// +// Typical value when running pg-toolbelt's Verdaccio on Docker Desktop: +// PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 +const PgDeltaNpmRegistryEnv = "PGDELTA_NPM_REGISTRY" From 2e05996e0f0335e0abbc56d95a3320c5483d1300 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 4 Jun 2026 10:55:28 +0200 Subject: [PATCH 04/10] chore: revert submodules repo --- .repos/effect | 2 +- .repos/effect-v3 | 2 +- .repos/lalph | 2 +- .repos/process-compose | 2 +- .repos/t3code | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.repos/effect b/.repos/effect index aae8797b9c..49b5a569ea 160000 --- a/.repos/effect +++ b/.repos/effect @@ -1 +1 @@ -Subproject commit aae8797b9cb383be0c182dd58d03d787c354238b +Subproject commit 49b5a569ea6c9d459f3db9cb2f150ca9d04b3cd0 diff --git a/.repos/effect-v3 b/.repos/effect-v3 index 70ce155cd7..e71ba68273 160000 --- a/.repos/effect-v3 +++ b/.repos/effect-v3 @@ -1 +1 @@ -Subproject commit 70ce155cd73a3b4cd723fe955454b5837b428f76 +Subproject commit e71ba68273026a1a2c1ace7218bdb206b0d3386d diff --git a/.repos/lalph b/.repos/lalph index 203f1ec28f..0fddf20a7d 160000 --- a/.repos/lalph +++ b/.repos/lalph @@ -1 +1 @@ -Subproject commit 203f1ec28f26d3a4f18c0f3e092eae3695de1842 +Subproject commit 0fddf20a7d70391bb583fb0abadc4223b618ec4d diff --git a/.repos/process-compose b/.repos/process-compose index cd7f6af235..a4038d6698 160000 --- a/.repos/process-compose +++ b/.repos/process-compose @@ -1 +1 @@ -Subproject commit cd7f6af235149a075385f3b8b54d635e83dc0f52 +Subproject commit a4038d669818c35fc68fc7fc240b39e371ce0e7a diff --git a/.repos/t3code b/.repos/t3code index 91a03e0747..d1e85c4e8f 160000 --- a/.repos/t3code +++ b/.repos/t3code @@ -1 +1 @@ -Subproject commit 91a03e074751e9dc732d0dddcd7b3a291caba34f +Subproject commit d1e85c4e8fdef82fbaded9539532b754080419e0 From 8968db7de88280f6a5304d580d11b1066db900a0 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 4 Jun 2026 16:51:43 +0200 Subject: [PATCH 05/10] refactor(cli): update pg-delta pull logic and tests - Renamed test cases for clarity and updated assertions to reflect new behavior regarding the use of the pg-delta flag and experimental config. - Modified `shouldUseDeclarativePgDeltaPull` function to remove reliance on the `usePgDelta` variable. - Enhanced debug capture logic in the diffing process to ensure proper handling of empty diffs. - Updated TypeScript template to use the latest pg-delta version (1.0.0-alpha.25). - Adjusted comments and formatting in various files for consistency and clarity. --- apps/cli-go/cmd/db.go | 2 +- apps/cli-go/cmd/db_pull_routing_test.go | 17 +++++++++++++---- apps/cli-go/internal/db/diff/diff.go | 14 ++++++-------- .../internal/db/diff/templates/pgdelta.ts | 4 ++-- apps/cli-go/pkg/config/config_test.go | 2 +- apps/cli-go/pkg/config/pgdelta_local.go | 3 ++- apps/cli-go/pkg/config/pgdelta_version.go | 2 +- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/apps/cli-go/cmd/db.go b/apps/cli-go/cmd/db.go index e4487deb3e..223f4ec63b 100644 --- a/apps/cli-go/cmd/db.go +++ b/apps/cli-go/cmd/db.go @@ -362,7 +362,7 @@ func shouldUseDeclarativePgDeltaPull(usePgDeltaDiff bool) bool { if usePgDeltaDiff { return false } - return shouldUsePgDelta() || usePgDelta + return shouldUsePgDelta() } func init() { diff --git a/apps/cli-go/cmd/db_pull_routing_test.go b/apps/cli-go/cmd/db_pull_routing_test.go index 8a01e782f8..c85c92200f 100644 --- a/apps/cli-go/cmd/db_pull_routing_test.go +++ b/apps/cli-go/cmd/db_pull_routing_test.go @@ -3,24 +3,33 @@ package cmd import ( "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) func TestShouldUseDeclarativePgDeltaPull(t *testing.T) { - t.Run("migration pg-delta wins over experimental config", func(t *testing.T) { + t.Run("diff-engine pg-delta keeps the migration-file workflow", func(t *testing.T) { usePgDelta = false t.Cleanup(func() { usePgDelta = false }) assert.False(t, shouldUseDeclarativePgDeltaPull(true)) }) - t.Run("experimental config without diff-engine uses declarative", func(t *testing.T) { + t.Run("no flag and no config means not declarative", func(t *testing.T) { usePgDelta = false t.Cleanup(func() { usePgDelta = false }) - // Simulate config enabled via shouldUsePgDelta's IsPgDeltaEnabled path indirectly: - // when neither flag nor config is set, declarative is off. assert.False(t, shouldUseDeclarativePgDeltaPull(false)) }) + t.Run("experimental config enables declarative", func(t *testing.T) { + usePgDelta = false + viper.Set("EXPERIMENTAL_PG_DELTA", true) + t.Cleanup(func() { + usePgDelta = false + viper.Set("EXPERIMENTAL_PG_DELTA", false) + }) + assert.True(t, shouldUseDeclarativePgDeltaPull(false)) + }) + t.Run("use-pg-delta flag forces declarative", func(t *testing.T) { usePgDelta = true t.Cleanup(func() { usePgDelta = false }) diff --git a/apps/cli-go/internal/db/diff/diff.go b/apps/cli-go/internal/db/diff/diff.go index be4cc002d5..429a75594e 100644 --- a/apps/cli-go/internal/db/diff/diff.go +++ b/apps/cli-go/internal/db/diff/diff.go @@ -212,22 +212,20 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w } else { fmt.Fprintln(w, "Diffing schemas...") } - var debugCapture *PgDeltaDebugCapture if IsPgDeltaDebugEnabled() && usePgDelta { + // Capture the shadow baseline catalog and edge-runtime stderr so an + // empty diff can be inspected later. DiffPgDeltaRefDetailed mirrors the + // pg-delta differ but additionally surfaces stderr, which differ() drops. + debugCapture := &PgDeltaDebugCapture{} if snapshot, exportErr := exportCatalogPgDelta(ctx, utils.ToPostgresURL(shadowConfig), "postgres", options...); exportErr == nil { - debugCapture = &PgDeltaDebugCapture{SourceCatalog: snapshot} + debugCapture.SourceCatalog = snapshot } else { fmt.Fprintf(w, "Warning: failed to export shadow pg-delta catalog: %v\n", exportErr) } - } - if IsPgDeltaDebugEnabled() && usePgDelta { result, err := DiffPgDeltaRefDetailed(ctx, utils.ToPostgresURL(shadowConfig), utils.ToPostgresURL(config), schema, pgDeltaFormatOptions(), options...) if err != nil { return DatabaseDiff{}, err } - if debugCapture == nil { - debugCapture = &PgDeltaDebugCapture{} - } debugCapture.Stderr = result.Stderr return DatabaseDiff{SQL: result.SQL, Debug: debugCapture}, nil } @@ -235,7 +233,7 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w if err != nil { return DatabaseDiff{}, err } - return DatabaseDiff{SQL: output, Debug: debugCapture}, nil + return DatabaseDiff{SQL: output}, nil } func migrateBaseDatabase(ctx context.Context, config pgconn.Config, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { diff --git a/apps/cli-go/internal/db/diff/templates/pgdelta.ts b/apps/cli-go/internal/db/diff/templates/pgdelta.ts index cccd1bc95b..306fed6a73 100644 --- a/apps/cli-go/internal/db/diff/templates/pgdelta.ts +++ b/apps/cli-go/internal/db/diff/templates/pgdelta.ts @@ -2,8 +2,8 @@ import { createPlan, deserializeCatalog, formatSqlStatements, -} from "npm:@supabase/pg-delta@1.0.0-alpha.25"; -import { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.25/integrations/supabase"; +} from "npm:@supabase/pg-delta@1.0.0-alpha.20"; +import { supabase } from "npm:@supabase/pg-delta@1.0.0-alpha.20/integrations/supabase"; async function resolveInput(ref: string | undefined) { if (!ref) { diff --git a/apps/cli-go/pkg/config/config_test.go b/apps/cli-go/pkg/config/config_test.go index 92d43b3de0..2a1a189076 100644 --- a/apps/cli-go/pkg/config/config_test.go +++ b/apps/cli-go/pkg/config/config_test.go @@ -289,7 +289,7 @@ enabled = true t.Run("InterpolatePgDeltaScript substitutes placeholder", func(t *testing.T) { c := NewConfig() require.NoError(t, c.Load("", fs.MapFS{})) - // Embedded TS pins use this semver literal before InterpolatePgDeltaScript runs. + // Embedded TS pins use this semver literal before InterpolatePgDeltaScript runs. got := InterpolatePgDeltaScript(Config(&c), `from "npm:@supabase/pg-delta@1.0.0-alpha.20";`) assert.Equal(t, `from "npm:@supabase/pg-delta@`+DefaultPgDeltaNpmVersion+`";`, got) }) diff --git a/apps/cli-go/pkg/config/pgdelta_local.go b/apps/cli-go/pkg/config/pgdelta_local.go index 683137f1b0..bbdd443d13 100644 --- a/apps/cli-go/pkg/config/pgdelta_local.go +++ b/apps/cli-go/pkg/config/pgdelta_local.go @@ -10,5 +10,6 @@ package config // Verdaccio workflow (CLI maintainers only). // // Typical value when running pg-toolbelt's Verdaccio on Docker Desktop: -// PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 +// +// PGDELTA_NPM_REGISTRY=http://host.docker.internal:4873 const PgDeltaNpmRegistryEnv = "PGDELTA_NPM_REGISTRY" diff --git a/apps/cli-go/pkg/config/pgdelta_version.go b/apps/cli-go/pkg/config/pgdelta_version.go index 297a4c34b1..2e4053767f 100644 --- a/apps/cli-go/pkg/config/pgdelta_version.go +++ b/apps/cli-go/pkg/config/pgdelta_version.go @@ -4,7 +4,7 @@ import "strings" // DefaultPgDeltaNpmVersion is the npm dist-tag/version used for @supabase/pg-delta // when supabase/.temp/pgdelta-version is absent or empty. -const DefaultPgDeltaNpmVersion = "1.0.0-alpha.22" +const DefaultPgDeltaNpmVersion = "1.0.0-alpha.25" const pgDeltaNpmVersionPlaceholder = "1.0.0-alpha.20" From 78d5c66875e24fb3735089e384db6fac4443b5a6 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 5 Jun 2026 12:52:29 +0200 Subject: [PATCH 06/10] refactor(cli): fix go-lints --- apps/cli-go/internal/db/diff/pgdelta.go | 16 ++++++++-------- apps/cli-go/internal/gen/types/pgdelta_conn.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/cli-go/internal/db/diff/pgdelta.go b/apps/cli-go/internal/db/diff/pgdelta.go index d4a63e1634..d9d5edfc46 100644 --- a/apps/cli-go/internal/db/diff/pgdelta.go +++ b/apps/cli-go/internal/db/diff/pgdelta.go @@ -75,13 +75,13 @@ func appendPgDeltaPostgresEnv( ref string, sslRootCertEnv string, options ...func(*pgx.ConnConfig), -) (string, []string, error) { +) ([]string, error) { preparedRef, sslEnv, err := types.PreparePgDeltaPostgresRef(ctx, ref, sslRootCertEnv, options...) if err != nil { - return "", nil, err + return nil, err } env = append(env, name+"="+containerRef(preparedRef)) - return preparedRef, append(env, sslEnv...), nil + return append(env, sslEnv...), nil } // DiffPgDelta diffs source and target Postgres configs via pg-delta. @@ -108,12 +108,12 @@ func DiffPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []s func DiffPgDeltaRefDetailed(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (PgDeltaDiffResult, error) { var env []string var err error - targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) if err != nil { return PgDeltaDiffResult{}, err } if len(sourceRef) > 0 { - sourceRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) + env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) if err != nil { return PgDeltaDiffResult{}, err } @@ -156,12 +156,12 @@ func DeclarativeExportPgDelta(ctx context.Context, source, target pgconn.Config, func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef string, schema []string, formatOptions string, options ...func(*pgx.ConnConfig)) (DeclarativeOutput, error) { var env []string var err error - targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) if err != nil { return DeclarativeOutput{}, err } if len(sourceRef) > 0 { - sourceRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) + env, err = appendPgDeltaPostgresEnv(ctx, env, "SOURCE", sourceRef, types.PgDeltaSourceSSLRootCert, options...) if err != nil { return DeclarativeOutput{}, err } @@ -199,7 +199,7 @@ func DeclarativeExportPgDeltaRef(ctx context.Context, sourceRef, targetRef strin func ExportCatalogPgDelta(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { var env []string var err error - targetRef, env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) + env, err = appendPgDeltaPostgresEnv(ctx, env, "TARGET", targetRef, types.PgDeltaTargetSSLRootCert, options...) if err != nil { return "", err } diff --git a/apps/cli-go/internal/gen/types/pgdelta_conn.go b/apps/cli-go/internal/gen/types/pgdelta_conn.go index 994eecc3b9..c4b9eaea0f 100644 --- a/apps/cli-go/internal/gen/types/pgdelta_conn.go +++ b/apps/cli-go/internal/gen/types/pgdelta_conn.go @@ -84,7 +84,7 @@ func writePgDeltaCABundleFile(ca string) (string, error) { if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { return "", err } - if err := os.WriteFile(abs, []byte(ca), 0o644); err != nil { + if err := os.WriteFile(abs, []byte(ca), 0o600); err != nil { return "", err } return "/workspace/" + filepath.ToSlash(pgDeltaCABundleRelPath), nil From aee93d19ee865927c28875d06d23a76f3282460a Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 5 Jun 2026 13:26:56 +0200 Subject: [PATCH 07/10] refactor(cli): update golangci configuration and remove unused function - Updated the golangci-lint configuration to correct the exclusion format for the ST1003 rule. - Removed the unused `diffWithStream` function from `internal/db/diff/diff.go` to clean up the codebase. --- apps/cli-go/.golangci.yml | 7 ++---- apps/cli-go/internal/db/diff/diff.go | 36 ---------------------------- 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/apps/cli-go/.golangci.yml b/apps/cli-go/.golangci.yml index 4063a03995..711143ee31 100644 --- a/apps/cli-go/.golangci.yml +++ b/apps/cli-go/.golangci.yml @@ -16,7 +16,7 @@ linters: exclusions: generated: lax rules: - - text: 'ST1003:' + - text: "ST1003:" linters: - staticcheck - path: _test\.go @@ -27,10 +27,7 @@ linters: linters: - gosec text: G117 - - path: internal/db/diff/diff\.go - linters: - - unused - text: diffWithStream is unused + presets: - comments - common-false-positives diff --git a/apps/cli-go/internal/db/diff/diff.go b/apps/cli-go/internal/db/diff/diff.go index 429a75594e..0b1cb7ca5c 100644 --- a/apps/cli-go/internal/db/diff/diff.go +++ b/apps/cli-go/internal/db/diff/diff.go @@ -1,7 +1,6 @@ package diff import ( - "bytes" "context" "fmt" "io" @@ -22,7 +21,6 @@ import ( "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" "github.com/spf13/afero" - "github.com/spf13/viper" "github.com/supabase/cli/internal/db/start" "github.com/supabase/cli/internal/pgdelta" "github.com/supabase/cli/internal/utils" @@ -250,37 +248,3 @@ func migrateBaseDatabase(ctx context.Context, config pgconn.Config, migrations [ defer conn.Close(context.Background()) return migration.SeedGlobals(ctx, migrations, conn, afero.NewIOFS(fsys)) } - -func diffWithStream(ctx context.Context, env []string, script string, stdout io.Writer) error { - cmd := utils.EdgeRuntimeStartCmd() - if viper.GetBool("DEBUG") { - cmd = append(cmd, "--verbose") - } - cmdString := strings.Join(cmd, " ") - entrypoint := []string{"sh", "-c", `cat <<'EOF' > index.ts && ` + cmdString + ` -` + script + ` -EOF -`} - var stderr bytes.Buffer - if err := utils.DockerRunOnceWithConfig( - ctx, - container.Config{ - Image: utils.Config.EdgeRuntime.Image, - Env: env, - Entrypoint: entrypoint, - }, - container.HostConfig{ - Binds: []string{utils.EdgeRuntimeId + ":/root/.cache/deno:rw"}, - NetworkMode: network.NetworkHost, - }, - network.NetworkingConfig{}, - "", - stdout, - &stderr, - // The "main worker has been destroyed" message may not appear at the start of stderr - // (e.g. preceded by other Deno runtime output), so use Contains instead of HasPrefix. - ); err != nil && !strings.Contains(stderr.String(), "main worker has been destroyed") { - return errors.Errorf("error diffing schema: %w:\n%s", err, stderr.String()) - } - return nil -} From 591ad1a8156942292dcb31793e2fbabd1f98ea93 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 5 Jun 2026 16:04:54 +0200 Subject: [PATCH 08/10] chore: update DefaultPgDeltaNpmVersion to 1.0.0-alpha.27 --- apps/cli-go/pkg/config/pgdelta_version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli-go/pkg/config/pgdelta_version.go b/apps/cli-go/pkg/config/pgdelta_version.go index 2e4053767f..26be07f8a0 100644 --- a/apps/cli-go/pkg/config/pgdelta_version.go +++ b/apps/cli-go/pkg/config/pgdelta_version.go @@ -4,7 +4,7 @@ import "strings" // DefaultPgDeltaNpmVersion is the npm dist-tag/version used for @supabase/pg-delta // when supabase/.temp/pgdelta-version is absent or empty. -const DefaultPgDeltaNpmVersion = "1.0.0-alpha.25" +const DefaultPgDeltaNpmVersion = "1.0.0-alpha.27" const pgDeltaNpmVersionPlaceholder = "1.0.0-alpha.20" From a7dc0f1faaee82fb646d41fc0f2b94326b7bf342 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 5 Jun 2026 14:39:41 +0000 Subject: [PATCH 09/10] refactor(cli): tighten pg-delta TLS plumbing and name disambiguation Review follow-ups for PR #5255: - Give SOURCE and TARGET distinct on-disk CA bundle filenames (pgdelta-source-ca.crt vs pgdelta-target-ca.crt) so a diff between two remote databases with different CAs cannot share one file. caBundleFilename derives the name from the env var. - Replace strings.Contains(host, "pooler.supabase.com") with an exact match plus a ".pooler.supabase.com" suffix check so look-alike hostnames (e.g. pooler.supabase.com.example.org) do not match. - Unexport the test-only EnsurePgDeltaVerifyCA helper; tests now exercise ensurePgDeltaSSL directly with an empty sslrootcert. - Rename pull.exportCatalogPgDelta to pull.exportTargetCatalog to distinguish it from the diff package's identically named test seam and to reflect that it captures the remote (target) catalog for the empty-pull debug bundle. --- .../internal/db/pull/pgdelta_pull_debug.go | 4 +-- .../db/pull/pgdelta_pull_debug_test.go | 12 +++---- .../cli-go/internal/gen/types/pgdelta_conn.go | 33 ++++++++++++------- .../internal/gen/types/pgdelta_conn_test.go | 33 ++++++++++++------- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go b/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go index 10881cc939..0cb35072a4 100644 --- a/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go +++ b/apps/cli-go/internal/db/pull/pgdelta_pull_debug.go @@ -16,7 +16,7 @@ import ( "github.com/supabase/cli/internal/utils" ) -var exportCatalogPgDelta = diff.ExportCatalogPgDelta +var exportTargetCatalog = diff.ExportCatalogPgDelta func saveEmptyPgDeltaPullDebug( ctx context.Context, @@ -28,7 +28,7 @@ func saveEmptyPgDeltaPullDebug( if capture == nil { capture = &diff.PgDeltaDebugCapture{} } - targetCatalog, err := exportCatalogPgDelta(ctx, utils.ToPostgresURL(config), "postgres", options...) + targetCatalog, err := exportTargetCatalog(ctx, utils.ToPostgresURL(config), "postgres", options...) if err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to export remote pg-delta catalog: %v\n", err) } diff --git a/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go b/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go index 6a25edb1a4..1eef2cb655 100644 --- a/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go +++ b/apps/cli-go/internal/db/pull/pgdelta_pull_debug_test.go @@ -18,11 +18,11 @@ import ( func TestSaveEmptyPgDeltaPullDebug(t *testing.T) { t.Setenv("PGDELTA_DEBUG", "1") fsys := afero.NewMemMapFs() - original := exportCatalogPgDelta + original := exportTargetCatalog t.Cleanup(func() { - exportCatalogPgDelta = original + exportTargetCatalog = original }) - exportCatalogPgDelta = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { + exportTargetCatalog = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { return `{"schema":"public","name":"airports"}`, nil } config := pgconn.Config{ @@ -70,11 +70,11 @@ func TestSaveEmptyPgDeltaPullDebug(t *testing.T) { func TestSaveEmptyPgDeltaPullDebugUsesTempDir(t *testing.T) { fsys := afero.NewMemMapFs() - original := exportCatalogPgDelta + original := exportTargetCatalog t.Cleanup(func() { - exportCatalogPgDelta = original + exportTargetCatalog = original }) - exportCatalogPgDelta = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { + exportTargetCatalog = func(ctx context.Context, targetRef, role string, options ...func(*pgx.ConnConfig)) (string, error) { return `{}`, nil } debugDir, err := saveEmptyPgDeltaPullDebug(context.Background(), pgconn.Config{}, &diff.PgDeltaDebugCapture{}, fsys) diff --git a/apps/cli-go/internal/gen/types/pgdelta_conn.go b/apps/cli-go/internal/gen/types/pgdelta_conn.go index c4b9eaea0f..4c3136c84e 100644 --- a/apps/cli-go/internal/gen/types/pgdelta_conn.go +++ b/apps/cli-go/internal/gen/types/pgdelta_conn.go @@ -13,7 +13,7 @@ import ( const ( PgDeltaSourceSSLRootCert = "PGDELTA_SOURCE_SSLROOTCERT" PgDeltaTargetSSLRootCert = "PGDELTA_TARGET_SSLROOTCERT" - pgDeltaCABundleRelPath = "supabase/.temp/pgdelta/supabase-ca-bundle.crt" + pgDeltaCABundleDir = "supabase/.temp/pgdelta" ) func isPostgresURL(ref string) bool { @@ -27,7 +27,8 @@ func isSupabaseHostedPostgresURL(dbURL string) bool { } host := strings.ToLower(parsed.Hostname()) return strings.HasSuffix(host, ".supabase.co") || - strings.Contains(host, "pooler.supabase.com") + host == "pooler.supabase.com" || + strings.HasSuffix(host, ".pooler.supabase.com") } // pgDeltaRootCA returns the CA bundle pg-delta should use for a Postgres URL. @@ -47,6 +48,20 @@ func pgDeltaRootCA(ctx context.Context, dbURL string, options ...func(*pgx.ConnC return "", nil } +// caBundleFilename returns the per-ref filename for the in-container CA +// bundle. SOURCE and TARGET use distinct files so a diff between two +// remotes with different CAs cannot accidentally share a single bundle. +func caBundleFilename(sslRootCertEnv string) string { + switch sslRootCertEnv { + case PgDeltaSourceSSLRootCert: + return "pgdelta-source-ca.crt" + case PgDeltaTargetSSLRootCert: + return "pgdelta-target-ca.crt" + default: + return "pgdelta-ca.crt" + } +} + // PreparePgDeltaPostgresRef configures a Postgres URL and env vars for pg-delta. // // pg-delta disables TLS when sslmode is absent and only reads PGDELTA_*_SSLROOTCERT @@ -68,26 +83,27 @@ func PreparePgDeltaPostgresRef( if len(ca) == 0 { return ref, nil, nil } - containerCertPath, err := writePgDeltaCABundleFile(ca) + containerCertPath, err := writePgDeltaCABundleFile(ca, caBundleFilename(sslRootCertEnv)) if err != nil { return "", nil, err } return ensurePgDeltaSSL(ref, containerCertPath), []string{sslRootCertEnv + "=" + ca}, nil } -func writePgDeltaCABundleFile(ca string) (string, error) { +func writePgDeltaCABundleFile(ca, filename string) (string, error) { cwd, err := os.Getwd() if err != nil { return "", err } - abs := filepath.Join(cwd, pgDeltaCABundleRelPath) + relPath := filepath.Join(pgDeltaCABundleDir, filename) + abs := filepath.Join(cwd, relPath) if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { return "", err } if err := os.WriteFile(abs, []byte(ca), 0o600); err != nil { return "", err } - return "/workspace/" + filepath.ToSlash(pgDeltaCABundleRelPath), nil + return "/workspace/" + filepath.ToSlash(relPath), nil } func ensurePgDeltaSSL(dbURL, sslrootcertPath string) string { @@ -107,8 +123,3 @@ func ensurePgDeltaSSL(dbURL, sslrootcertPath string) string { parsed.RawQuery = query.Encode() return parsed.String() } - -// EnsurePgDeltaVerifyCA is kept for tests that assert URL sslmode behaviour. -func EnsurePgDeltaVerifyCA(dbURL string) string { - return ensurePgDeltaSSL(dbURL, "") -} diff --git a/apps/cli-go/internal/gen/types/pgdelta_conn_test.go b/apps/cli-go/internal/gen/types/pgdelta_conn_test.go index eb29a279ef..60a472279a 100644 --- a/apps/cli-go/internal/gen/types/pgdelta_conn_test.go +++ b/apps/cli-go/internal/gen/types/pgdelta_conn_test.go @@ -6,43 +6,54 @@ import ( "github.com/stretchr/testify/assert" ) -func TestEnsurePgDeltaVerifyCA(t *testing.T) { +func TestEnsurePgDeltaSSL(t *testing.T) { t.Run("adds verify-ca when sslmode is absent", func(t *testing.T) { input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?connect_timeout=10" - got := EnsurePgDeltaVerifyCA(input) + got := ensurePgDeltaSSL(input, "") assert.Contains(t, got, "sslmode=verify-ca") assert.Contains(t, got, "connect_timeout=10") }) t.Run("preserves existing verify-ca", func(t *testing.T) { input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=verify-ca" - assert.Equal(t, input, EnsurePgDeltaVerifyCA(input)) + assert.Equal(t, input, ensurePgDeltaSSL(input, "")) }) t.Run("preserves existing verify-full", func(t *testing.T) { input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=verify-full" - assert.Equal(t, input, EnsurePgDeltaVerifyCA(input)) + assert.Equal(t, input, ensurePgDeltaSSL(input, "")) }) t.Run("replaces require with verify-ca", func(t *testing.T) { input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?sslmode=require" - got := EnsurePgDeltaVerifyCA(input) + got := ensurePgDeltaSSL(input, "") assert.Contains(t, got, "sslmode=verify-ca") assert.NotContains(t, got, "sslmode=require") }) -} -func TestEnsurePgDeltaSSLAddsRootCertPath(t *testing.T) { - input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?connect_timeout=10" - got := ensurePgDeltaSSL(input, "/workspace/supabase/.temp/pgdelta/supabase-ca-bundle.crt") - assert.Contains(t, got, "sslmode=verify-ca") - assert.Contains(t, got, "sslrootcert=%2Fworkspace%2Fsupabase%2F.temp%2Fpgdelta%2Fsupabase-ca-bundle.crt") + t.Run("adds the sslrootcert path when provided", func(t *testing.T) { + input := "postgresql://postgres:secret@db.example.supabase.co:5432/postgres?connect_timeout=10" + got := ensurePgDeltaSSL(input, "/workspace/supabase/.temp/pgdelta/pgdelta-target-ca.crt") + assert.Contains(t, got, "sslmode=verify-ca") + assert.Contains(t, got, "sslrootcert=%2Fworkspace%2Fsupabase%2F.temp%2Fpgdelta%2Fpgdelta-target-ca.crt") + }) } func TestIsSupabaseHostedPostgresURL(t *testing.T) { assert.True(t, isSupabaseHostedPostgresURL("postgresql://postgres@db.ref.supabase.co:5432/postgres")) assert.True(t, isSupabaseHostedPostgresURL("postgresql://supabase_admin@aws-0-us-east-2.pooler.supabase.com:5432/postgres")) + assert.True(t, isSupabaseHostedPostgresURL("postgresql://supabase_admin@pooler.supabase.com:5432/postgres")) assert.False(t, isSupabaseHostedPostgresURL("postgresql://postgres@localhost:5432/postgres")) + // Suffix match rejects look-alike hostnames that merely contain the + // pooler domain as a substring (e.g. an attacker-controlled host like + // pooler.supabase.com.example.org). + assert.False(t, isSupabaseHostedPostgresURL("postgresql://postgres@pooler.supabase.com.example.org:5432/postgres")) +} + +func TestCABundleFilename(t *testing.T) { + assert.Equal(t, "pgdelta-source-ca.crt", caBundleFilename(PgDeltaSourceSSLRootCert)) + assert.Equal(t, "pgdelta-target-ca.crt", caBundleFilename(PgDeltaTargetSSLRootCert)) + assert.Equal(t, "pgdelta-ca.crt", caBundleFilename("")) } func TestPreparePgDeltaPostgresRefNonPostgres(t *testing.T) { From 2c92ca397550724c544532f66ad51a86b6c1b64e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 5 Jun 2026 15:31:44 +0000 Subject: [PATCH 10/10] fix(cli): create dist dir before copying go sidecar binary The build:go-sidecar step runs first in the build chain, before any bun --compile step has created dist/, so `cp ../cli-go/supabase-go dist/supabase-go` failed with "No such file or directory" and broke nx run supabase:build (and every e2e shard that depends on it). Prefix the copy with `mkdir -p dist` so the destination exists regardless of step order; it is idempotent when dist/ already exists. --- apps/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli/package.json b/apps/cli/package.json index 034d1142a2..ae788b916a 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -25,7 +25,7 @@ }, "scripts": { "build": "pnpm build:go-sidecar && pnpm build:next && pnpm build:legacy && pnpm build:shim", - "build:go-sidecar": "cp ../cli-go/supabase-go dist/supabase-go", + "build:go-sidecar": "mkdir -p dist && cp ../cli-go/supabase-go dist/supabase-go", "build:next": "bun build src/next/main.ts --compile --outfile dist/supabase-next", "build:legacy": "bun build src/legacy/main.ts --compile --outfile dist/supabase-legacy", "build:shim": "bun build src/shared/cli/bin.ts --outfile dist/supabase.js --target node",