From daa1fd3e44fdd7870d7d6c8b6110a0851511eb34 Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Tue, 16 Jun 2026 15:32:26 -0300 Subject: [PATCH 01/47] feat(cdc): native SQLite CDC source (db.cdc.sqlite) Why: The runtime could stream row changes from Postgres (db.cdc.postgres) but had no equivalent for SQLite. SQLite has no logical-replication slot, so the native mechanism is the preupdate hook, which observes row changes on the connection the runtime writes through. What: Adds a db.cdc.sqlite registry kind backed by a supervised Source that installs SQLite preupdate + commit/rollback hooks on the target db.sql.sqlite pool's writer connection and emits insert/update/delete (plus a gap-free snapshot bootstrap) through the existing, engine-agnostic cdc Lua module (cdc.stream/list_sources/source). How: - service/sql: a build-tagged hook seam (sqlite_preupdate_hook) registers a ConnectHook-enabled driver and installs/clears hooks on a raw *sqlite3.SQLiteConn; the factory selects the driver transparently. - service/cdc/sqlite: the Source buffers preupdate rows per transaction and flushes atomically on commit via a bounded handoff to a drain goroutine. Column names/affinity resolve over a dedicated read-only connection, and checkpoints write through a separate plain-driver connection, so the writer-blocking commit hook can never deadlock against schema resolution or checkpoint writes. A laggard subscriber is closed rather than allowed to stall the writer. Durable snapshot/offset state lives in wippy_cdc_offsets in the source DB. - api/service/cdc: db.cdc.sqlite kind, SQLiteConfig, and a composite inspector/streamer so the Lua module observes both engines. - boot: kind-specific listeners (db.cdc.postgres + db.cdc.sqlite) feed the composite. Capture is in-process and live-only: changes made while the runtime is down, or by an external writer, are not captured. The checkpoint exists for snapshot-gating and idempotent dedupe, not replay. Building requires the sqlite_preupdate_hook tag (added to the Makefile); without it the source fails loudly instead of silently capturing nothing. --- Makefile | 22 +- api/service/cdc/composite.go | 44 +++ api/service/cdc/composite_test.go | 69 ++++ api/service/cdc/config.go | 1 + api/service/cdc/config_sqlite.go | 36 ++ api/service/cdc/config_sqlite_test.go | 35 ++ api/service/cdc/context.go | 3 + api/service/cdc/errors.go | 2 + boot/components/service/storage/cdc.go | 23 +- service/cdc/sqlite/bench_test.go | 20 + service/cdc/sqlite/checkpoint.go | 60 +++ service/cdc/sqlite/decode.go | 52 +++ service/cdc/sqlite/decode_test.go | 64 ++++ service/cdc/sqlite/errors.go | 55 +++ service/cdc/sqlite/integration_test.go | 286 ++++++++++++++ service/cdc/sqlite/manager.go | 248 ++++++++++++ service/cdc/sqlite/manager_test.go | 74 ++++ service/cdc/sqlite/snapshot.go | 118 ++++++ service/cdc/sqlite/source.go | 466 +++++++++++++++++++++++ service/cdc/sqlite/source_stub.go | 9 + service/cdc/sqlite/source_stub_test.go | 17 + service/cdc/sqlite/source_tagged_test.go | 27 ++ service/cdc/sqlite/subscribers.go | 182 +++++++++ service/cdc/sqlite/subscribers_test.go | 131 +++++++ service/sql/errors.go | 10 +- service/sql/factory.go | 4 +- service/sql/sqlite_cdc.go | 139 +++++++ test.sh | 5 +- 28 files changed, 2179 insertions(+), 23 deletions(-) create mode 100644 api/service/cdc/composite.go create mode 100644 api/service/cdc/composite_test.go create mode 100644 api/service/cdc/config_sqlite.go create mode 100644 api/service/cdc/config_sqlite_test.go create mode 100644 service/cdc/sqlite/bench_test.go create mode 100644 service/cdc/sqlite/checkpoint.go create mode 100644 service/cdc/sqlite/decode.go create mode 100644 service/cdc/sqlite/decode_test.go create mode 100644 service/cdc/sqlite/errors.go create mode 100644 service/cdc/sqlite/integration_test.go create mode 100644 service/cdc/sqlite/manager.go create mode 100644 service/cdc/sqlite/manager_test.go create mode 100644 service/cdc/sqlite/snapshot.go create mode 100644 service/cdc/sqlite/source.go create mode 100644 service/cdc/sqlite/source_stub.go create mode 100644 service/cdc/sqlite/source_stub_test.go create mode 100644 service/cdc/sqlite/source_tagged_test.go create mode 100644 service/cdc/sqlite/subscribers.go create mode 100644 service/cdc/sqlite/subscribers_test.go create mode 100644 service/sql/sqlite_cdc.go diff --git a/Makefile b/Makefile index ad3b72592..89b926828 100644 --- a/Makefile +++ b/Makefile @@ -13,9 +13,9 @@ test: go test ./system/... -v -race -short go test ./service/... -v -race -short go test ./cluster/... -v -race -short - go test --tags "fts5 sqlite_vec treesitter" ./runtime/... -v -race -short + go test --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" ./runtime/... -v -race -short go test ./boot/... -v -race -short - go test --tags "fts5 sqlite_vec treesitter" ./cmd/... -v -race -short + go test --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" ./cmd/... -v -race -short test-system: go test ./internal/... -v -race @@ -25,7 +25,7 @@ test-system: test-runtime: go test ./internal/... -v -race go test ./api/... -v -race - go test --tags "fts5 sqlite_vec treesitter" ./runtime/... -v -race + go test --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" ./runtime/... -v -race test-service: go test ./internal/... -v -race @@ -45,7 +45,7 @@ test-network: .PHONY: lint lint: - go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.8.0 run --timeout=10m --build-tags=race ./... + go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.8.0 run --timeout=10m --build-tags=race,sqlite_preupdate_hook ./... # Mutation testing with gremlins. Coverage is scoped to the directory gremlins # runs from, so target a package subtree via MUTATE_DIR. workers=1 keeps per- @@ -87,7 +87,7 @@ build-wippy: build-wippy-local .PHONY: build-wippy-local build-wippy-local: mkdir -p ./dist - CGO_ENABLED=1 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-$(shell go env GOOS)-$(shell go env GOARCH) \ @@ -99,7 +99,7 @@ build-wippy-all: build-wippy-linux-amd64 build-wippy-linux-arm64 build-wippy-dar .PHONY: build-wippy-linux-amd64 build-wippy-linux-amd64: mkdir -p ./dist - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-linux-amd64 \ @@ -109,7 +109,7 @@ build-wippy-linux-amd64: build-wippy-linux-arm64: mkdir -p ./dist CGO_LDFLAGS="" CGO_CFLAGS="" CC=aarch64-linux-gnu-gcc \ - CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-linux-arm64 \ @@ -118,7 +118,7 @@ build-wippy-linux-arm64: .PHONY: build-wippy-darwin-amd64 build-wippy-darwin-amd64: mkdir -p ./dist - CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-darwin-amd64 \ @@ -127,7 +127,7 @@ build-wippy-darwin-amd64: .PHONY: build-wippy-darwin-arm64 build-wippy-darwin-arm64: mkdir -p ./dist - CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-darwin-arm64 \ @@ -137,7 +137,7 @@ build-wippy-darwin-arm64: build-wippy-windows-amd64: mkdir -p ./dist CGO_LDFLAGS="" CGO_CFLAGS="" CC=x86_64-w64-mingw32-gcc \ - CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter" \ + CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" \ -ldflags="$(WIPPY_LDFLAGS)" \ -trimpath \ -o ./dist/wippy-windows-amd64.exe \ @@ -170,4 +170,4 @@ build-sign-wippy-windows: build-wippy-windows-amd64 sign-wippy-windows .PHONY: run-wippy run-wippy: - go run --tags "fts5 sqlite_vec treesitter" -ldflags="$(WIPPY_LDFLAGS)" ./cmd/wippy/ $(ARGS) + go run --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" -ldflags="$(WIPPY_LDFLAGS)" ./cmd/wippy/ $(ARGS) diff --git a/api/service/cdc/composite.go b/api/service/cdc/composite.go new file mode 100644 index 000000000..a5f6adb6c --- /dev/null +++ b/api/service/cdc/composite.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import "context" + +type Engine interface { + SourceInspector + SourceStreamer +} + +type composite struct { + engines []Engine +} + +func NewComposite(engines ...Engine) *composite { + return &composite{engines: engines} +} + +func (c *composite) List() []SourceInfo { + out := make([]SourceInfo, 0) + for _, e := range c.engines { + out = append(out, e.List()...) + } + return out +} + +func (c *composite) Get(name string) (SourceInfo, bool) { + for _, e := range c.engines { + if info, ok := e.Get(name); ok { + return info, true + } + } + return SourceInfo{}, false +} + +func (c *composite) Stream(ctx context.Context, name string, opts StreamOptions) (ChangeStream, SourceInfo, error) { + for _, e := range c.engines { + if _, ok := e.Get(name); ok { + return e.Stream(ctx, name, opts) + } + } + return nil, SourceInfo{}, ErrSourceNotFound +} diff --git a/api/service/cdc/composite_test.go b/api/service/cdc/composite_test.go new file mode 100644 index 000000000..b06a4997e --- /dev/null +++ b/api/service/cdc/composite_test.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeStream struct{ ch chan Change } + +func (f *fakeStream) Changes() <-chan Change { return f.ch } +func (f *fakeStream) Close() {} + +type fakeEngine struct { + infos map[string]SourceInfo + opened []string +} + +func (e *fakeEngine) List() []SourceInfo { + out := make([]SourceInfo, 0, len(e.infos)) + for _, i := range e.infos { + out = append(out, i) + } + return out +} + +func (e *fakeEngine) Get(name string) (SourceInfo, bool) { + i, ok := e.infos[name] + return i, ok +} + +func (e *fakeEngine) Stream(_ context.Context, name string, _ StreamOptions) (ChangeStream, SourceInfo, error) { + e.opened = append(e.opened, name) + return &fakeStream{ch: make(chan Change)}, e.infos[name], nil +} + +func TestCompositeListAggregates(t *testing.T) { + a := &fakeEngine{infos: map[string]SourceInfo{"pg": {Name: "pg", Engine: "postgres"}}} + b := &fakeEngine{infos: map[string]SourceInfo{"lite": {Name: "lite", Engine: "sqlite"}}} + c := NewComposite(a, b) + + infos := c.List() + assert.Len(t, infos, 2) +} + +func TestCompositeGetAndStreamRouting(t *testing.T) { + a := &fakeEngine{infos: map[string]SourceInfo{"pg": {Name: "pg", Engine: "postgres"}}} + b := &fakeEngine{infos: map[string]SourceInfo{"lite": {Name: "lite", Engine: "sqlite"}}} + c := NewComposite(a, b) + + info, ok := c.Get("lite") + require.True(t, ok) + assert.Equal(t, "sqlite", info.Engine) + + _, _, err := c.Stream(context.Background(), "lite", StreamOptions{}) + require.NoError(t, err) + assert.Equal(t, []string{"lite"}, b.opened) + assert.Empty(t, a.opened) +} + +func TestCompositeStreamNotFound(t *testing.T) { + c := NewComposite(&fakeEngine{infos: map[string]SourceInfo{}}) + _, _, err := c.Stream(context.Background(), "missing", StreamOptions{}) + assert.ErrorIs(t, err, ErrSourceNotFound) +} diff --git a/api/service/cdc/config.go b/api/service/cdc/config.go index 953f283b0..c2b7064d1 100644 --- a/api/service/cdc/config.go +++ b/api/service/cdc/config.go @@ -11,6 +11,7 @@ import ( const ( Postgres registry.Kind = "db.cdc.postgres" + SQLite registry.Kind = "db.cdc.sqlite" ) const ( diff --git a/api/service/cdc/config_sqlite.go b/api/service/cdc/config_sqlite.go new file mode 100644 index 000000000..928192aab --- /dev/null +++ b/api/service/cdc/config_sqlite.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "time" + + "github.com/wippyai/runtime/api/supervisor" +) + +type SQLiteConfig struct { + DBResource string `json:"db_resource"` + Name string `json:"name,omitempty"` + StatusInterval string `json:"status_interval,omitempty"` + Tables []string `json:"tables,omitempty"` + Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` + Snapshot bool `json:"snapshot,omitempty"` +} + +func (c *SQLiteConfig) InitDefaults() { + c.Lifecycle.InitDefaults() +} + +func (c *SQLiteConfig) Validate() error { + if c.DBResource == "" { + return ErrDBResourceRequired + } + if _, err := c.StatusDuration(); err != nil { + return err + } + return nil +} + +func (c *SQLiteConfig) StatusDuration() (time.Duration, error) { + return parseInterval(c.StatusInterval) +} diff --git a/api/service/cdc/config_sqlite_test.go b/api/service/cdc/config_sqlite_test.go new file mode 100644 index 000000000..b49f6a292 --- /dev/null +++ b/api/service/cdc/config_sqlite_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSQLiteConfigValidate(t *testing.T) { + missing := &SQLiteConfig{} + assert.ErrorIs(t, missing.Validate(), ErrDBResourceRequired) + + badInterval := &SQLiteConfig{DBResource: "app:db", StatusInterval: "nope"} + assert.ErrorIs(t, badInterval.Validate(), ErrInvalidInterval) + + negative := &SQLiteConfig{DBResource: "app:db", StatusInterval: "-5s"} + assert.ErrorIs(t, negative.Validate(), ErrInvalidInterval) + + ok := &SQLiteConfig{DBResource: "app:db", StatusInterval: "5s", Tables: []string{"users"}, Snapshot: true} + require.NoError(t, ok.Validate()) + + d, err := ok.StatusDuration() + require.NoError(t, err) + assert.Equal(t, "5s", d.String()) +} + +func TestSQLiteConfigZeroInterval(t *testing.T) { + cfg := &SQLiteConfig{DBResource: "app:db"} + d, err := cfg.StatusDuration() + require.NoError(t, err) + assert.Equal(t, int64(0), int64(d)) +} diff --git a/api/service/cdc/context.go b/api/service/cdc/context.go index cc0f98df0..2ea298b19 100644 --- a/api/service/cdc/context.go +++ b/api/service/cdc/context.go @@ -8,6 +8,9 @@ type SourceInfo struct { Name string `json:"name"` Slot string `json:"slot"` Publication string `json:"publication,omitempty"` + Engine string `json:"engine,omitempty"` + File string `json:"file,omitempty"` + DBResource string `json:"db_resource,omitempty"` Tables []string `json:"tables,omitempty"` Streaming bool `json:"streaming,omitempty"` Failover bool `json:"failover,omitempty"` diff --git a/api/service/cdc/errors.go b/api/service/cdc/errors.go index ffdeae95b..f26752037 100644 --- a/api/service/cdc/errors.go +++ b/api/service/cdc/errors.go @@ -15,4 +15,6 @@ var ( ErrInvalidInterval = apierror.New(apierror.Invalid, "interval must be a non-negative duration (e.g. 10s)").WithRetryable(apierror.False) ErrFailoverTemporary = apierror.New(apierror.Invalid, "failover cannot be set on a temporary slot").WithRetryable(apierror.False) ErrInvalidSnapshotFetchSize = apierror.New(apierror.Invalid, "snapshot_fetch_size must be non-negative").WithRetryable(apierror.False) + ErrDBResourceRequired = apierror.New(apierror.Invalid, "db_resource is required").WithRetryable(apierror.False) + ErrSourceNotFound = apierror.New(apierror.NotFound, "cdc source not found").WithRetryable(apierror.False) ) diff --git a/boot/components/service/storage/cdc.go b/boot/components/service/storage/cdc.go index b6ce5bd83..8ae6b65a5 100644 --- a/boot/components/service/storage/cdc.go +++ b/boot/components/service/storage/cdc.go @@ -10,31 +10,42 @@ import ( "github.com/wippyai/runtime/api/event" logapi "github.com/wippyai/runtime/api/logs" "github.com/wippyai/runtime/api/payload" + resourceapi "github.com/wippyai/runtime/api/resource" cdcapi "github.com/wippyai/runtime/api/service/cdc" bootpkg "github.com/wippyai/runtime/boot" bootsystem "github.com/wippyai/runtime/boot/components/system" - cdc "github.com/wippyai/runtime/service/cdc/postgres" + pgcdc "github.com/wippyai/runtime/service/cdc/postgres" + sqlitecdc "github.com/wippyai/runtime/service/cdc/sqlite" ) func CDC() boot.Component { return boot.New(boot.P{ Name: CDCName, - DependsOn: []boot.Name{bootsystem.EnvironmentName}, + DependsOn: []boot.Name{bootsystem.EnvironmentName, bootsystem.ResourcesName}, Load: func(ctx context.Context) (context.Context, error) { logger := logapi.GetLogger(ctx) dtt := payload.GetTranscoder(ctx) bus := event.GetBus(ctx) envRegistry := envapi.GetRegistry(ctx) + resReg := resourceapi.GetRegistry(ctx) handlers := bootpkg.GetHandlerRegistry(ctx) - manager, err := cdc.NewManager(dtt, bus, logger.Named("cdc"), envRegistry) + pgManager, err := pgcdc.NewManager(dtt, bus, logger.Named("cdc.postgres"), envRegistry) if err != nil { return ctx, NewCDCManagerError(err) } - handlers.RegisterListener("db.cdc.*", manager) - ctx = cdcapi.WithSourceInspector(ctx, manager) - ctx = cdcapi.WithSourceStreamer(ctx, manager) + sqliteManager, err := sqlitecdc.NewManager(dtt, bus, logger.Named("cdc.sqlite"), resReg) + if err != nil { + return ctx, NewCDCManagerError(err) + } + + handlers.RegisterListener("db.cdc.postgres", pgManager) + handlers.RegisterListener("db.cdc.sqlite", sqliteManager) + + composite := cdcapi.NewComposite(pgManager, sqliteManager) + ctx = cdcapi.WithSourceInspector(ctx, composite) + ctx = cdcapi.WithSourceStreamer(ctx, composite) return ctx, nil }, }) diff --git a/service/cdc/sqlite/bench_test.go b/service/cdc/sqlite/bench_test.go new file mode 100644 index 000000000..3de73c668 --- /dev/null +++ b/service/cdc/sqlite/bench_test.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import "testing" + +func BenchmarkMapRow(b *testing.B) { + cols := []columnInfo{ + {name: "id"}, + {name: "email", text: true}, + {name: "balance"}, + {name: "blob"}, + } + vals := []any{int64(1), []byte("user@example.com"), 42.5, []byte{0x00, 0x01, 0x02}} + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = mapRow(cols, vals) + } +} diff --git a/service/cdc/sqlite/checkpoint.go b/service/cdc/sqlite/checkpoint.go new file mode 100644 index 000000000..bc1a7c3aa --- /dev/null +++ b/service/cdc/sqlite/checkpoint.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" +) + +const createOffsetsSQL = `CREATE TABLE IF NOT EXISTS ` + offsetsTable + ` ( + source TEXT PRIMARY KEY, + last_seq INTEGER NOT NULL DEFAULT 0, + snapshot_done INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +)` + +func ensureOffsets(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, createOffsetsSQL) + return err +} + +func loadOffset(ctx context.Context, db *sql.DB, source string) (snapshotDone bool, lastSeq uint64, err error) { + row := db.QueryRowContext(ctx, "SELECT snapshot_done, last_seq FROM "+offsetsTable+" WHERE source = ?", source) + var done int + var seq int64 + switch scanErr := row.Scan(&done, &seq); scanErr { + case nil: + return done != 0, uint64(seq), nil + case sql.ErrNoRows: + return false, 0, nil + default: + return false, 0, scanErr + } +} + +func saveSnapshotDone(ctx context.Context, db *sql.DB, source string) error { + _, err := db.ExecContext(ctx, + "INSERT INTO "+offsetsTable+" (source, snapshot_done, updated_at) VALUES (?, 1, datetime('now')) "+ + "ON CONFLICT(source) DO UPDATE SET snapshot_done = 1, updated_at = datetime('now')", + source) + return err +} + +func saveOffset(ctx context.Context, db *sql.DB, source string, seq uint64) error { + if db == nil { + return nil + } + _, err := db.ExecContext(ctx, + "INSERT INTO "+offsetsTable+" (source, last_seq, updated_at) VALUES (?, ?, datetime('now')) "+ + "ON CONFLICT(source) DO UPDATE SET last_seq = MAX(last_seq, excluded.last_seq), updated_at = datetime('now')", + source, int64(seq)) + return err +} + +func deleteOffset(ctx context.Context, db *sql.DB, source string) error { + _, err := db.ExecContext(ctx, "DELETE FROM "+offsetsTable+" WHERE source = ?", source) + return err +} diff --git a/service/cdc/sqlite/decode.go b/service/cdc/sqlite/decode.go new file mode 100644 index 000000000..9decc12d0 --- /dev/null +++ b/service/cdc/sqlite/decode.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "strconv" + "strings" + "unicode/utf8" +) + +type columnInfo struct { + name string + text bool +} + +func textAffinity(declType string) bool { + t := strings.ToUpper(declType) + if strings.Contains(t, "BLOB") || t == "" { + return false + } + return strings.Contains(t, "CHAR") || strings.Contains(t, "CLOB") || strings.Contains(t, "TEXT") +} + +func mapRow(cols []columnInfo, vals []any) map[string]any { + if vals == nil { + return nil + } + out := make(map[string]any, len(vals)) + for i, v := range vals { + name, text := columnAt(cols, i) + out[name] = normalizeValue(v, text) + } + return out +} + +func columnAt(cols []columnInfo, i int) (string, bool) { + if i < len(cols) { + return cols[i].name, cols[i].text + } + return "column" + strconv.Itoa(i), false +} + +func normalizeValue(v any, text bool) any { + b, ok := v.([]byte) + if !ok { + return v + } + if text && utf8.Valid(b) { + return string(b) + } + return b +} diff --git a/service/cdc/sqlite/decode_test.go b/service/cdc/sqlite/decode_test.go new file mode 100644 index 000000000..f1307111f --- /dev/null +++ b/service/cdc/sqlite/decode_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTextAffinity(t *testing.T) { + cases := map[string]bool{ + "TEXT": true, + "VARCHAR(255)": true, + "CLOB": true, + "nchar": true, + "INTEGER": false, + "REAL": false, + "BLOB": false, + "": false, + "NUMERIC": false, + } + for decl, want := range cases { + assert.Equalf(t, want, textAffinity(decl), "decl=%q", decl) + } +} + +func TestMapRowNilForMissingSide(t *testing.T) { + assert.Nil(t, mapRow([]columnInfo{{name: "id"}}, nil)) +} + +func TestMapRowDecodesTextBytesByAffinity(t *testing.T) { + cols := []columnInfo{ + {name: "id", text: false}, + {name: "email", text: true}, + {name: "payload", text: false}, + } + vals := []any{int64(7), []byte("a@b.com"), []byte{0x00, 0x01, 0x02}} + + out := mapRow(cols, vals) + + assert.Equal(t, int64(7), out["id"]) + assert.Equal(t, "a@b.com", out["email"]) + assert.Equal(t, []byte{0x00, 0x01, 0x02}, out["payload"]) +} + +func TestMapRowKeepsInvalidUTF8AsBytes(t *testing.T) { + cols := []columnInfo{{name: "data", text: true}} + invalid := []byte{0xff, 0xfe, 0xfd} + out := mapRow(cols, []any{invalid}) + assert.Equal(t, invalid, out["data"]) +} + +func TestMapRowFallbackColumnNames(t *testing.T) { + out := mapRow(nil, []any{int64(1), "x"}) + assert.Equal(t, int64(1), out["column0"]) + assert.Equal(t, "x", out["column1"]) +} + +func TestNormalizeValuePassThrough(t *testing.T) { + assert.Equal(t, int64(3), normalizeValue(int64(3), true)) + assert.Equal(t, 1.5, normalizeValue(1.5, true)) + assert.Nil(t, normalizeValue(nil, true)) +} diff --git a/service/cdc/sqlite/errors.go b/service/cdc/sqlite/errors.go new file mode 100644 index 000000000..714a00c1f --- /dev/null +++ b/service/cdc/sqlite/errors.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "errors" + + "github.com/wippyai/runtime/api/attrs" + apierror "github.com/wippyai/runtime/api/error" + "github.com/wippyai/runtime/api/registry" +) + +var ( + ErrSourceClosed = errors.New("cdc: source is closed") + ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) + ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) + ErrResourceRegRequired = apierror.New(apierror.Invalid, "resource registry is required").WithRetryable(apierror.False) + ErrPreupdateTagRequired = apierror.New(apierror.Internal, "sqlite cdc requires the sqlite_preupdate_hook build tag").WithRetryable(apierror.False) + ErrNoSourceStreamer = apierror.New(apierror.Internal, "cdc source streamer not available").WithRetryable(apierror.False) + ErrChangeBacklogOverflow = apierror.New(apierror.Unavailable, "sqlite cdc change backlog overflow").WithRetryable(apierror.True) +) + +func NewUnsupportedEntryKindError(kind registry.Kind) apierror.Error { + return apierror.New(apierror.Invalid, "unsupported entry kind"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"kind": kind})) +} + +func NewServiceExistsError(id registry.ID) apierror.Error { + return apierror.New(apierror.Conflict, "cdc service already exists"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"id": id.String()})) +} + +func NewServiceNotFoundError(id registry.ID) apierror.Error { + return apierror.New(apierror.NotFound, "cdc service not found"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"id": id.String()})) +} + +func NewInvalidConfigError(err error) apierror.Error { + apiErr := apierror.New(apierror.Invalid, "invalid cdc configuration").WithRetryable(apierror.False) + if err != nil { + apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) + } + return apiErr +} + +func NewSourceCreationError(err error) apierror.Error { + apiErr := apierror.New(apierror.Internal, "failed to create cdc source").WithRetryable(apierror.False) + if err != nil { + apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) + } + return apiErr +} diff --git a/service/cdc/sqlite/integration_test.go b/service/cdc/sqlite/integration_test.go new file mode 100644 index 000000000..0c1a7718a --- /dev/null +++ b/service/cdc/sqlite/integration_test.go @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build integration && sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + config "github.com/wippyai/runtime/api/service/cdc" + sqlconfig "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +type fakeResource struct{ res sqlservice.DBResource } + +func (f *fakeResource) Get() (any, error) { return f.res, nil } +func (f *fakeResource) Release() {} + +type fakeRegistry struct{ db *sql.DB } + +func (r *fakeRegistry) Acquire(context.Context, registry.ID, resource.AccessMode) (resource.Resource[any], error) { + return &fakeResource{res: sqlservice.DBResource{DB: r.db, Type: sqlconfig.SQLite}}, nil +} +func (r *fakeRegistry) List() ([]registry.ID, error) { return nil, nil } +func (r *fakeRegistry) Exists(registry.ID) bool { return true } + +func openPool(t *testing.T) (*sql.DB, string) { + t.Helper() + file := filepath.Join(t.TempDir(), "app.db") + db, err := sql.Open("sqlite3_wippy", "file:"+file+"?mode=rwc") + require.NoError(t, err) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + _, err = db.Exec("PRAGMA journal_mode=WAL") + require.NoError(t, err) + return db, file +} + +func newSource(t *testing.T, db *sql.DB, opts sourceOptions) *Source { + t.Helper() + opts.res = &fakeRegistry{db: db} + opts.dbResource = registry.NewID("app", "db") + if opts.name == "" { + opts.name = "test-src" + } + if opts.statusInterval == "" { + opts.statusInterval = "1s" + } + h, err := buildSource(opts) + require.NoError(t, err) + return h.(*Source) +} + +func waitChange(t *testing.T, ch <-chan config.Change) config.Change { + t.Helper() + select { + case c := <-ch: + return c + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for change") + return config.Change{} + } +} + +func TestIntegrationInsertUpdateDelete(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT, balance REAL)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + _, err = db.Exec(`INSERT INTO users (id, email, balance) VALUES (1, 'a@b.com', 42.5)`) + require.NoError(t, err) + ins := waitChange(t, stream.Changes()) + assert.Equal(t, "insert", ins.Op) + assert.Equal(t, "users", ins.Table) + assert.Equal(t, "a@b.com", ins.After["email"]) + assert.Equal(t, 42.5, ins.After["balance"]) + assert.Equal(t, int64(1), ins.After["id"]) + assert.Nil(t, ins.Before) + + _, err = db.Exec(`UPDATE users SET balance = 99.0 WHERE id = 1`) + require.NoError(t, err) + upd := waitChange(t, stream.Changes()) + assert.Equal(t, "update", upd.Op) + assert.Equal(t, 42.5, upd.Before["balance"]) + assert.Equal(t, 99.0, upd.After["balance"]) + + _, err = db.Exec(`DELETE FROM users WHERE id = 1`) + require.NoError(t, err) + del := waitChange(t, stream.Changes()) + assert.Equal(t, "delete", del.Op) + assert.Equal(t, "a@b.com", del.Before["email"]) + assert.Nil(t, del.After) +} + +func TestIntegrationValueFidelity(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT, qty INTEGER, price REAL, blob BLOB, note TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + _, err = db.Exec(`INSERT INTO items (id, name, qty, price, blob, note) VALUES (1, 'widget', 7, 1.25, X'00ff10', NULL)`) + require.NoError(t, err) + + c := waitChange(t, stream.Changes()) + assert.Equal(t, "widget", c.After["name"]) + assert.Equal(t, int64(7), c.After["qty"]) + assert.Equal(t, 1.25, c.After["price"]) + assert.Equal(t, []byte{0x00, 0xff, 0x10}, c.After["blob"]) + assert.Nil(t, c.After["note"]) +} + +func TestIntegrationRollbackDiscarded(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + tx, err := db.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO t (id, v) VALUES (1, 'rolled-back')`) + require.NoError(t, err) + require.NoError(t, tx.Rollback()) + + _, err = db.Exec(`INSERT INTO t (id, v) VALUES (2, 'committed')`) + require.NoError(t, err) + + c := waitChange(t, stream.Changes()) + assert.Equal(t, "insert", c.Op) + assert.Equal(t, "committed", c.After["v"]) + assert.Equal(t, int64(2), c.After["id"]) +} + +func TestIntegrationSnapshotBootstrap(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{snapshot: true}) + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + snap := waitChange(t, stream.Changes()) + assert.Equal(t, "snapshot", snap.Op) + assert.Equal(t, "existing@b.com", snap.After["email"]) + + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (2, 'new@b.com')`) + require.NoError(t, err) + live := waitChange(t, stream.Changes()) + assert.Equal(t, "insert", live.Op) + assert.Equal(t, "new@b.com", live.After["email"]) +} + +func TestIntegrationRestartKeepsCheckpoint(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) + require.NoError(t, err) + + first := newSource(t, db, sourceOptions{snapshot: true, name: "src"}) + s1 := first.Subscribe(config.StreamOptions{}) + _, err = first.Start(context.Background()) + require.NoError(t, err) + snap := waitChange(t, s1.Changes()) + require.Equal(t, "snapshot", snap.Op) + s1.Close() + require.NoError(t, first.Stop(context.Background())) + + second := newSource(t, db, sourceOptions{snapshot: true, name: "src"}) + s2 := second.Subscribe(config.StreamOptions{}) + defer s2.Close() + _, err = second.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = second.Stop(context.Background()) }() + + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (2, 'new@b.com')`) + require.NoError(t, err) + + got := waitChange(t, s2.Changes()) + assert.Equal(t, "insert", got.Op, "restart must not re-snapshot; first event should be the live insert") + assert.Equal(t, "new@b.com", got.After["email"]) +} + +func TestIntegrationLaggardDoesNotStallWrites(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + laggard := src.Subscribe(config.StreamOptions{Buffer: 1}) + defer laggard.Close() + + done := make(chan error, 1) + go func() { + for i := 0; i < 500; i++ { + if _, e := db.Exec(`INSERT INTO t (v) VALUES ('x')`); e != nil { + done <- e + return + } + } + done <- nil + }() + + select { + case e := <-done: + require.NoError(t, e) + case <-time.After(10 * time.Second): + t.Fatal("writes stalled: a non-reading subscriber blocked the writer") + } + + reader := src.Subscribe(config.StreamOptions{}) + defer reader.Close() + _, err = db.Exec(`INSERT INTO t (v) VALUES ('final')`) + require.NoError(t, err) + + got := waitChange(t, reader.Changes()) + assert.Equal(t, "insert", got.Op) + assert.Equal(t, "final", got.After["v"]) +} + +func TestIntegrationTableAllowlist(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{tables: []string{"users"}}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + _, err = db.Exec(`INSERT INTO orders (id, v) VALUES (1, 'ignored')`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO users (id, v) VALUES (1, 'captured')`) + require.NoError(t, err) + + c := waitChange(t, stream.Changes()) + assert.Equal(t, "users", c.Table) + assert.Equal(t, "captured", c.After["v"]) +} diff --git a/service/cdc/sqlite/manager.go b/service/cdc/sqlite/manager.go new file mode 100644 index 000000000..f8f7b19d2 --- /dev/null +++ b/service/cdc/sqlite/manager.go @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "context" + "sync" + + "github.com/wippyai/runtime/api/event" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + config "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + entryutil "github.com/wippyai/runtime/internal/entry" + "go.uber.org/zap" +) + +type sourceHandle interface { + supervisor.Service + Subscribe(opts config.StreamOptions) config.ChangeStream + closeSubscriptions() + markDrop() +} + +type sourceOptions struct { + res resource.Registry + log *zap.Logger + dbResource registry.ID + name string + statusInterval string + tables []string + snapshot bool +} + +type Manager struct { + dtt payload.Transcoder + bus event.Bus + res resource.Registry + log *zap.Logger + sources map[registry.ID]sourceHandle + infos map[registry.ID]config.SourceInfo + infosByName map[string]registry.ID + mu sync.Mutex +} + +func NewManager(dtt payload.Transcoder, bus event.Bus, log *zap.Logger, res resource.Registry) (*Manager, error) { + if dtt == nil { + return nil, ErrTranscoderRequired + } + if bus == nil { + return nil, ErrEventBusRequired + } + if res == nil { + return nil, ErrResourceRegRequired + } + if log == nil { + log = zap.NewNop() + } + return &Manager{ + dtt: dtt, + bus: bus, + res: res, + log: log, + sources: make(map[registry.ID]sourceHandle), + infos: make(map[registry.ID]config.SourceInfo), + infosByName: make(map[string]registry.ID), + }, nil +} + +func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + + if entry.Kind != config.SQLite { + return NewUnsupportedEntryKindError(entry.Kind) + } + if _, exists := m.sources[entry.ID]; exists { + return NewServiceExistsError(entry.ID) + } + + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) + if err != nil { + return NewInvalidConfigError(err) + } + if err := cfg.Validate(); err != nil { + return NewInvalidConfigError(err) + } + + src, err := buildSource(m.sourceOptions(entry, cfg)) + if err != nil { + return NewSourceCreationError(err) + } + + m.sources[entry.ID] = src + m.storeInfo(entry, cfg) + m.register(ctx, entry, src, cfg.Lifecycle) + return nil +} + +func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + + if entry.Kind != config.SQLite { + return NewUnsupportedEntryKindError(entry.Kind) + } + if _, exists := m.sources[entry.ID]; !exists { + return NewServiceNotFoundError(entry.ID) + } + + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) + if err != nil { + return NewInvalidConfigError(err) + } + if err := cfg.Validate(); err != nil { + return NewInvalidConfigError(err) + } + + src, err := buildSource(m.sourceOptions(entry, cfg)) + if err != nil { + return NewSourceCreationError(err) + } + + if old := m.sources[entry.ID]; old != nil { + old.closeSubscriptions() + } + m.removeInfo(entry.ID) + m.unregister(ctx, entry) + + m.sources[entry.ID] = src + m.storeInfo(entry, cfg) + m.register(ctx, entry, src, cfg.Lifecycle) + return nil +} + +func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + + src, exists := m.sources[entry.ID] + if !exists { + return NewServiceNotFoundError(entry.ID) + } + src.markDrop() + src.closeSubscriptions() + m.removeInfo(entry.ID) + m.unregister(ctx, entry) + delete(m.sources, entry.ID) + return nil +} + +func (m *Manager) sourceOptions(entry registry.Entry, cfg *config.SQLiteConfig) sourceOptions { + return sourceOptions{ + res: m.res, + log: m.log.With(zap.String("id", entry.ID.String())), + name: entry.ID.String(), + dbResource: registry.ParseID(cfg.DBResource), + tables: cfg.Tables, + statusInterval: cfg.StatusInterval, + snapshot: cfg.Snapshot, + } +} + +func (m *Manager) storeInfo(entry registry.Entry, cfg *config.SQLiteConfig) { + info := config.SourceInfo{ + Name: entry.ID.String(), + Engine: "sqlite", + DBResource: cfg.DBResource, + Tables: append([]string(nil), cfg.Tables...), + Snapshot: cfg.Snapshot, + } + m.infos[entry.ID] = info + m.infosByName[info.Name] = entry.ID +} + +func (m *Manager) removeInfo(id registry.ID) { + if info, ok := m.infos[id]; ok { + if current, present := m.infosByName[info.Name]; present && current == id { + delete(m.infosByName, info.Name) + } + delete(m.infos, id) + } +} + +func (m *Manager) List() []config.SourceInfo { + m.mu.Lock() + defer m.mu.Unlock() + + out := make([]config.SourceInfo, 0, len(m.infos)) + for _, info := range m.infos { + out = append(out, info) + } + return out +} + +func (m *Manager) Get(name string) (config.SourceInfo, bool) { + m.mu.Lock() + defer m.mu.Unlock() + + if id, ok := m.infosByName[name]; ok { + if info, present := m.infos[id]; present { + return info, true + } + } + return config.SourceInfo{}, false +} + +func (m *Manager) Stream(_ context.Context, name string, opts config.StreamOptions) (config.ChangeStream, config.SourceInfo, error) { + m.mu.Lock() + src, info, ok := m.lookupSourceLocked(name) + m.mu.Unlock() + if !ok { + return nil, config.SourceInfo{}, NewServiceNotFoundError(registry.ParseID(name)) + } + return src.Subscribe(opts), info, nil +} + +func (m *Manager) lookupSourceLocked(name string) (sourceHandle, config.SourceInfo, bool) { + if id, ok := m.infosByName[name]; ok { + if src := m.sources[id]; src != nil { + return src, m.infos[id], true + } + } + return nil, config.SourceInfo{}, false +} + +func (m *Manager) register(ctx context.Context, entry registry.Entry, src sourceHandle, lifecycle supervisor.LifecycleConfig) { + m.bus.Send(ctx, event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRegister, + Path: entry.ID.String(), + Data: &supervisor.Entry{ + Service: src, + Config: lifecycle, + }, + }) + m.log.Info("added sqlite cdc source", zap.String("id", entry.ID.String()), zap.String("kind", entry.Kind)) +} + +func (m *Manager) unregister(ctx context.Context, entry registry.Entry) { + m.bus.Send(ctx, event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRemove, + Path: entry.ID.String(), + }) + m.log.Info("removed sqlite cdc source", zap.String("id", entry.ID.String())) +} diff --git a/service/cdc/sqlite/manager_test.go b/service/cdc/sqlite/manager_test.go new file mode 100644 index 000000000..c4dca30dc --- /dev/null +++ b/service/cdc/sqlite/manager_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/cdc" +) + +func newInspectorManager() *Manager { + return &Manager{ + sources: map[registry.ID]sourceHandle{}, + infos: map[registry.ID]config.SourceInfo{}, + infosByName: map[string]registry.ID{}, + } +} + +func TestNewManagerValidation(t *testing.T) { + _, err := NewManager(nil, nil, nil, nil) + assert.ErrorIs(t, err, ErrTranscoderRequired) +} + +func TestManagerStoreAndList(t *testing.T) { + m := newInspectorManager() + idA := registry.NewID("test", "id-a") + idB := registry.NewID("test", "id-b") + + m.storeInfo(registry.Entry{ID: idA, Kind: config.SQLite}, &config.SQLiteConfig{ + DBResource: "app:db", + Tables: []string{"users"}, + Snapshot: true, + }) + m.storeInfo(registry.Entry{ID: idB, Kind: config.SQLite}, &config.SQLiteConfig{ + DBResource: "app:db2", + }) + + infos := m.List() + require.Len(t, infos, 2) + names := []string{infos[0].Name, infos[1].Name} + sort.Strings(names) + assert.Equal(t, []string{idA.String(), idB.String()}, names) + + got, ok := m.Get(idA.String()) + require.True(t, ok) + assert.Equal(t, "sqlite", got.Engine) + assert.Equal(t, "app:db", got.DBResource) + assert.Equal(t, []string{"users"}, got.Tables) + assert.True(t, got.Snapshot) +} + +func TestManagerGetMiss(t *testing.T) { + m := newInspectorManager() + _, ok := m.Get("missing") + assert.False(t, ok) +} + +func TestManagerRemoveInfo(t *testing.T) { + m := newInspectorManager() + id := registry.NewID("test", "id-a") + m.storeInfo(registry.Entry{ID: id, Kind: config.SQLite}, &config.SQLiteConfig{DBResource: "app:db"}) + + m.removeInfo(id) + + _, ok := m.Get(id.String()) + assert.False(t, ok) + assert.Empty(t, m.List()) + assert.NotContains(t, m.infosByName, id.String()) +} diff --git a/service/cdc/sqlite/snapshot.go b/service/cdc/sqlite/snapshot.go new file mode 100644 index 000000000..132b49705 --- /dev/null +++ b/service/cdc/sqlite/snapshot.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "strconv" + "strings" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +func (s *Source) runSnapshot(ctx context.Context, conn *sql.Conn) error { + tables, err := s.snapshotTables(ctx, conn) + if err != nil { + return err + } + for _, table := range tables { + if err := s.snapshotTable(ctx, conn, table); err != nil { + return err + } + } + return nil +} + +func (s *Source) snapshotTables(ctx context.Context, conn *sql.Conn) ([]string, error) { + rows, err := conn.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var tables []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + if s.tableAllowed(name) { + tables = append(tables, name) + } + } + return tables, rows.Err() +} + +func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, table string) error { + cols := s.columnsFor(ctx, table) + + rows, err := conn.QueryContext(ctx, "SELECT * FROM "+quoteIdent(table)) //nolint:gosec // quoted identifier from sqlite_master; SQLite cannot bind table names as parameters + if err != nil { + return err + } + defer func() { _ = rows.Close() }() + + names, err := rows.Columns() + if err != nil { + return err + } + if len(cols) == 0 { + cols = columnsFromNames(names) + } + + for rows.Next() { + vals := make([]any, len(names)) + ptrs := make([]any, len(names)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + return err + } + seq := s.seq.Add(1) + s.subs.publish(ctx, config.Change{ + Source: s.name, + Op: "snapshot", + Table: table, + Relation: table, + After: mapRow(cols, vals), + LSN: strconv.FormatUint(seq, 10), + }) + } + return rows.Err() +} + +func resolveColumns(ctx context.Context, db *sql.DB, table string) ([]columnInfo, error) { + rows, err := db.QueryContext(ctx, "PRAGMA table_info("+quoteIdent(table)+")") + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var cols []columnInfo + for rows.Next() { + var cid, notnull, pk int + var name, declType string + var dflt any + if err := rows.Scan(&cid, &name, &declType, ¬null, &dflt, &pk); err != nil { + return nil, err + } + cols = append(cols, columnInfo{name: name, text: textAffinity(declType)}) + } + return cols, rows.Err() +} + +func columnsFromNames(names []string) []columnInfo { + cols := make([]columnInfo, len(names)) + for i, n := range names { + cols[i] = columnInfo{name: n} + } + return cols +} + +func quoteIdent(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go new file mode 100644 index 000000000..9469bdeb0 --- /dev/null +++ b/service/cdc/sqlite/source.go @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "fmt" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "go.uber.org/zap" + + "github.com/wippyai/runtime/api/metrics" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + config "github.com/wippyai/runtime/api/service/cdc" + sqlconfig "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +const ( + offsetsTable = "wippy_cdc_offsets" + changesCounter = "wippy_cdc_changes_total" + walGauge = "wippy_cdc_wal_size_bytes" + defaultStatusInterval = 30 * time.Second + commitQueueSize = 256 + auxBusyTimeoutMillisec = 5000 +) + +type capturedChange struct { + table string + old []any + new []any + op int + rowid int64 +} + +type Source struct { + poolRes resource.Resource[any] + res resource.Registry + readDB *sql.DB + checkpointDB *sql.DB + runDone chan struct{} + commits chan []capturedChange + subs *subscribers + writerDB *sql.DB + cols map[string][]columnInfo + cancel context.CancelFunc + tables map[string]struct{} + log *zap.Logger + dbResID registry.ID + file string + name string + pending []capturedChange + statusInterval time.Duration + seq atomic.Uint64 + colMu sync.RWMutex + mu sync.Mutex + pendMu sync.Mutex + stopped atomic.Bool + dropCP atomic.Bool + snap bool +} + +func buildSource(opts sourceOptions) (sourceHandle, error) { + log := opts.log + if log == nil { + log = zap.NewNop() + } + interval := defaultStatusInterval + if opts.statusInterval != "" { + d, err := time.ParseDuration(opts.statusInterval) + if err != nil || d < 0 { + return nil, fmt.Errorf("invalid status_interval %q", opts.statusInterval) + } + if d > 0 { + interval = d + } + } + return &Source{ + log: log, + res: opts.res, + subs: newSubscribers(), + name: opts.name, + statusInterval: interval, + dbResID: opts.dbResource, + tables: filterSet(opts.tables), + cols: make(map[string][]columnInfo), + snap: opts.snapshot, + }, nil +} + +func (s *Source) Subscribe(opts config.StreamOptions) config.ChangeStream { + return s.subs.subscribe(opts) +} + +func (s *Source) closeSubscriptions() { + s.subs.closeAll() +} + +func (s *Source) markDrop() { + s.dropCP.Store(true) +} + +func (s *Source) PreUpdate(op int, table string, rowid int64, old, new []any) { + if !s.tableAllowed(table) { + return + } + s.pendMu.Lock() + s.pending = append(s.pending, capturedChange{op: op, table: table, rowid: rowid, old: old, new: new}) + s.pendMu.Unlock() +} + +func (s *Source) Commit() { + s.pendMu.Lock() + batch := s.pending + s.pending = nil + s.pendMu.Unlock() + if len(batch) == 0 { + return + } + select { + case s.commits <- batch: + case <-s.runDone: + } +} + +func (s *Source) Rollback() { + s.pendMu.Lock() + s.pending = nil + s.pendMu.Unlock() +} + +func (s *Source) tableAllowed(table string) bool { + lower := strings.ToLower(table) + if lower == offsetsTable { + return false + } + if len(s.tables) == 0 { + return true + } + _, ok := s.tables[lower] + return ok +} + +func (s *Source) Start(ctx context.Context) (<-chan any, error) { + if s.stopped.Load() { + return nil, ErrSourceClosed + } + + dbRes, res, err := s.acquirePool(ctx) + if err != nil { + return nil, err + } + writerDB := dbRes.DB + + conn, err := writerDB.Conn(ctx) + if err != nil { + res.Release() + return nil, fmt.Errorf("acquire writer conn: %w", err) + } + + var file string + if rawErr := conn.Raw(func(dc any) error { + f, e := sqlservice.InstallCDCHooksOnRaw(dc, s) + file = f + return e + }); rawErr != nil { + _ = conn.Close() + res.Release() + return nil, rawErr + } + sqlservice.RegisterCDCSink(file, s) + + s.mu.Lock() + s.poolRes = res + s.writerDB = writerDB + s.file = file + s.mu.Unlock() + + readDB, checkpointDB, err := openAuxConns(file) + if err != nil { + s.abortStart(ctx, conn, writerDB, file) + return nil, err + } + + s.mu.Lock() + s.readDB = readDB + s.checkpointDB = checkpointDB + s.mu.Unlock() + + if err := ensureOffsets(ctx, checkpointDB); err != nil { + s.abortStart(ctx, conn, writerDB, file) + return nil, err + } + snapDone, lastSeq, loadErr := loadOffset(ctx, checkpointDB, s.name) + if loadErr != nil { + s.log.Warn("load cdc offset failed; treating as fresh", zap.Error(loadErr)) + } + if lastSeq > s.seq.Load() { + s.seq.Store(lastSeq) + } + + runCtx, cancel := context.WithCancel(ctx) + status := make(chan any, 8) + runDone := make(chan struct{}) + commits := make(chan []capturedChange, commitQueueSize) + + s.mu.Lock() + if s.stopped.Load() { + s.mu.Unlock() + cancel() + s.abortStart(ctx, conn, writerDB, file) + return nil, ErrSourceClosed + } + s.cancel = cancel + s.runDone = runDone + s.commits = commits + s.mu.Unlock() + + doSnapshot := s.snap && !snapDone + if doSnapshot { + if err := s.runSnapshot(runCtx, conn); err != nil { + cancel() + s.abortStart(ctx, conn, writerDB, file) + return nil, fmt.Errorf("snapshot: %w", err) + } + if serr := saveSnapshotDone(runCtx, checkpointDB, s.name); serr != nil { + s.log.Warn("persist snapshot completion failed; restart may re-snapshot", zap.Error(serr)) + } + } + + go s.run(runCtx, status, runDone, metrics.GetCollector(ctx)) + + _ = conn.Close() + + select { + case status <- "sqlite cdc started": + default: + } + s.log.Info("sqlite cdc source started", + zap.String("file", s.file), + zap.Bool("snapshot", doSnapshot)) + return status, nil +} + +func (s *Source) abortStart(ctx context.Context, conn *sql.Conn, writerDB *sql.DB, file string) { + _ = conn.Close() + s.detachHooks(ctx, writerDB, file) + s.releaseResources(ctx) +} + +func (s *Source) acquirePool(ctx context.Context) (sqlservice.DBResource, resource.Resource[any], error) { + res, err := s.res.Acquire(ctx, s.dbResID, resource.ModeNormal) + if err != nil { + return sqlservice.DBResource{}, nil, fmt.Errorf("acquire db resource: %w", err) + } + dbAny, err := res.Get() + if err != nil { + res.Release() + return sqlservice.DBResource{}, nil, fmt.Errorf("get db resource: %w", err) + } + dbRes, ok := dbAny.(sqlservice.DBResource) + if !ok { + res.Release() + return sqlservice.DBResource{}, nil, fmt.Errorf("resource %s is not a database", s.name) + } + if dbRes.Type != sqlconfig.SQLite { + res.Release() + return sqlservice.DBResource{}, nil, fmt.Errorf("resource %s is not a sqlite database (kind %s)", s.name, dbRes.Type) + } + return dbRes, res, nil +} + +func (s *Source) Stop(ctx context.Context) error { + if !s.stopped.CompareAndSwap(false, true) { + return nil + } + defer s.closeSubscriptions() + + s.mu.Lock() + cancel := s.cancel + runDone := s.runDone + writerDB := s.writerDB + file := s.file + s.mu.Unlock() + + if cancel != nil { + cancel() + } + if runDone != nil { + select { + case <-runDone: + case <-ctx.Done(): + return ctx.Err() + } + } + + if writerDB != nil { + s.detachHooks(ctx, writerDB, file) + } + if s.dropCP.Load() { + s.mu.Lock() + cpDB := s.checkpointDB + s.mu.Unlock() + if cpDB != nil { + _ = deleteOffset(ctx, cpDB, s.name) + } + } + s.releaseResources(ctx) + return nil +} + +func (s *Source) detachHooks(ctx context.Context, writerDB *sql.DB, file string) { + sqlservice.UnregisterCDCSink(file) + conn, err := writerDB.Conn(ctx) + if err != nil { + return + } + _ = conn.Raw(sqlservice.ClearCDCHooksOnRaw) + _ = conn.Close() +} + +func (s *Source) releaseResources(_ context.Context) { + s.mu.Lock() + readDB := s.readDB + cpDB := s.checkpointDB + res := s.poolRes + s.readDB = nil + s.checkpointDB = nil + s.poolRes = nil + s.mu.Unlock() + + if readDB != nil { + _ = readDB.Close() + } + if cpDB != nil { + _ = cpDB.Close() + } + if res != nil { + res.Release() + } +} + +func (s *Source) run(ctx context.Context, status chan any, runDone chan struct{}, mc metrics.Collector) { + defer close(runDone) + defer close(status) + + ticker := time.NewTicker(s.statusInterval) + defer ticker.Stop() + + for { + select { + case batch := <-s.commits: + s.process(ctx, batch, mc) + case <-ticker.C: + s.onTick(ctx, mc) + case <-ctx.Done(): + s.drainRemaining(ctx, mc) + return + } + } +} + +func (s *Source) drainRemaining(ctx context.Context, mc metrics.Collector) { + for { + select { + case batch := <-s.commits: + s.process(ctx, batch, mc) + default: + return + } + } +} + +func (s *Source) process(ctx context.Context, batch []capturedChange, mc metrics.Collector) { + for _, ch := range batch { + cols := s.columnsFor(ctx, ch.table) + op := opString(ch.op) + seq := s.seq.Add(1) + change := config.Change{ + Source: s.name, + Op: op, + Table: ch.table, + Relation: ch.table, + Before: mapRow(cols, ch.old), + After: mapRow(cols, ch.new), + LSN: strconv.FormatUint(seq, 10), + } + s.subs.publish(ctx, change) + if mc != nil { + mc.CounterInc(changesCounter, metrics.Labels{"source": s.name, "op": op}) + } + } +} + +func (s *Source) onTick(ctx context.Context, mc metrics.Collector) { + if mc != nil { + if info, err := os.Stat(s.file + "-wal"); err == nil { + mc.GaugeSet(walGauge, float64(info.Size()), metrics.Labels{"source": s.name}) + } + } + if seq := s.seq.Load(); seq > 0 { + if err := saveOffset(ctx, s.checkpointDB, s.name, seq); err != nil { + s.log.Warn("persist cdc offset failed", zap.Error(err)) + } + } +} + +func (s *Source) columnsFor(ctx context.Context, table string) []columnInfo { + s.colMu.RLock() + cols, ok := s.cols[table] + s.colMu.RUnlock() + if ok { + return cols + } + + cols, err := resolveColumns(ctx, s.readDB, table) + if err != nil { + s.log.Warn("resolve columns failed; emitting positional column names", + zap.String("table", table), zap.Error(err)) + return nil + } + s.colMu.Lock() + s.cols[table] = cols + s.colMu.Unlock() + return cols +} + +func opString(op int) string { + switch op { + case sqlservice.CDCInsert: + return "insert" + case sqlservice.CDCUpdate: + return "update" + case sqlservice.CDCDelete: + return "delete" + default: + return "unknown" + } +} + +func openAuxConns(file string) (read, checkpoint *sql.DB, err error) { + read, err = sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)+"&_query_only=ON") + if err != nil { + return nil, nil, fmt.Errorf("open read connection: %w", err) + } + read.SetMaxOpenConns(1) + read.SetMaxIdleConns(1) + + checkpoint, err = sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)) + if err != nil { + _ = read.Close() + return nil, nil, fmt.Errorf("open checkpoint connection: %w", err) + } + checkpoint.SetMaxOpenConns(1) + checkpoint.SetMaxIdleConns(1) + return read, checkpoint, nil +} diff --git a/service/cdc/sqlite/source_stub.go b/service/cdc/sqlite/source_stub.go new file mode 100644 index 000000000..8feb9cd2a --- /dev/null +++ b/service/cdc/sqlite/source_stub.go @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build !sqlite_preupdate_hook + +package sqlite + +func buildSource(_ sourceOptions) (sourceHandle, error) { + return nil, ErrPreupdateTagRequired +} diff --git a/service/cdc/sqlite/source_stub_test.go b/service/cdc/sqlite/source_stub_test.go new file mode 100644 index 000000000..086d7859c --- /dev/null +++ b/service/cdc/sqlite/source_stub_test.go @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build !sqlite_preupdate_hook + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildSourceRequiresTag(t *testing.T) { + src, err := buildSource(sourceOptions{name: "x"}) + assert.Nil(t, src) + assert.ErrorIs(t, err, ErrPreupdateTagRequired) +} diff --git a/service/cdc/sqlite/source_tagged_test.go b/service/cdc/sqlite/source_tagged_test.go new file mode 100644 index 000000000..c98672d36 --- /dev/null +++ b/service/cdc/sqlite/source_tagged_test.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" +) + +func TestBuildSourceWithTag(t *testing.T) { + h, err := buildSource(sourceOptions{name: "x", dbResource: registry.NewID("app", "db")}) + require.NoError(t, err) + require.NotNil(t, h) + _, ok := h.(*Source) + assert.True(t, ok) +} + +func TestBuildSourceRejectsBadInterval(t *testing.T) { + _, err := buildSource(sourceOptions{name: "x", statusInterval: "nope"}) + assert.Error(t, err) +} diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go new file mode 100644 index 000000000..05c451505 --- /dev/null +++ b/service/cdc/sqlite/subscribers.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "context" + "strings" + "sync" + "sync/atomic" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +const ( + defaultStreamBuffer = 128 + maxStreamBuffer = 65536 +) + +type subscribers struct { + m map[uint64]*subscription + mu sync.RWMutex + next uint64 +} + +func newSubscribers() *subscribers { + return &subscribers{m: make(map[uint64]*subscription)} +} + +func (s *subscribers) subscribe(opts config.StreamOptions) config.ChangeStream { + buffer := opts.Buffer + if buffer <= 0 { + buffer = defaultStreamBuffer + } + if buffer > maxStreamBuffer { + buffer = maxStreamBuffer + } + + s.mu.Lock() + s.next++ + sub := &subscription{ + parent: s, + id: s.next, + in: make(chan config.Change, buffer), + out: make(chan config.Change, buffer), + done: make(chan struct{}), + tables: filterSet(opts.Tables), + ops: filterSet(opts.Ops), + } + s.m[sub.id] = sub + s.mu.Unlock() + + go sub.run() + return sub +} + +func (s *subscribers) publish(ctx context.Context, change config.Change) { + s.mu.RLock() + matched := make([]*subscription, 0, len(s.m)) + for _, sub := range s.m { + if sub.matches(change) { + matched = append(matched, sub) + } + } + s.mu.RUnlock() + + for _, sub := range matched { + sub.send(ctx, change) + } +} + +func (s *subscribers) remove(id uint64) { + s.mu.Lock() + delete(s.m, id) + s.mu.Unlock() +} + +func (s *subscribers) closeAll() { + s.mu.Lock() + subs := make([]*subscription, 0, len(s.m)) + for id, sub := range s.m { + subs = append(subs, sub) + delete(s.m, id) + } + s.mu.Unlock() + + for _, sub := range subs { + sub.Close() + } +} + +type subscription struct { + parent *subscribers + in chan config.Change + out chan config.Change + done chan struct{} + tables map[string]struct{} + ops map[string]struct{} + id uint64 + once sync.Once + closed atomic.Bool +} + +func (s *subscription) Changes() <-chan config.Change { + return s.out +} + +func (s *subscription) Close() { + s.once.Do(func() { + s.closed.Store(true) + if s.parent != nil { + s.parent.remove(s.id) + } + close(s.done) + }) +} + +func (s *subscription) run() { + defer close(s.out) + for { + select { + case <-s.done: + return + default: + } + select { + case change := <-s.in: + select { + case <-s.done: + return + case s.out <- change: + } + case <-s.done: + return + } + } +} + +func (s *subscription) send(_ context.Context, change config.Change) { + if s.closed.Load() { + return + } + select { + case s.in <- change: + default: + s.Close() + } +} + +func (s *subscription) matches(change config.Change) bool { + if len(s.ops) > 0 { + if _, ok := s.ops[strings.ToLower(change.Op)]; !ok { + return false + } + } + if len(s.tables) > 0 { + if _, ok := s.tables[strings.ToLower(change.Relation)]; ok { + return true + } + if _, ok := s.tables[strings.ToLower(change.Table)]; ok { + return true + } + return false + } + return true +} + +func filterSet(values []string) map[string]struct{} { + if len(values) == 0 { + return nil + } + out := make(map[string]struct{}, len(values)) + for _, v := range values { + v = strings.ToLower(strings.TrimSpace(v)) + if v != "" { + out[v] = struct{}{} + } + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go new file mode 100644 index 000000000..20e192a91 --- /dev/null +++ b/service/cdc/sqlite/subscribers_test.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +func TestFilterSet(t *testing.T) { + assert.Nil(t, filterSet(nil)) + assert.Nil(t, filterSet([]string{" ", ""})) + + got := filterSet([]string{"Users", " orders ", "users"}) + assert.Contains(t, got, "users") + assert.Contains(t, got, "orders") + assert.Len(t, got, 2) +} + +func TestSubscriptionMatches(t *testing.T) { + all := &subscription{} + assert.True(t, all.matches(config.Change{Op: "insert", Table: "users"})) + + byOp := &subscription{ops: map[string]struct{}{"insert": {}}} + assert.True(t, byOp.matches(config.Change{Op: "insert"})) + assert.False(t, byOp.matches(config.Change{Op: "delete"})) + + byTable := &subscription{tables: map[string]struct{}{"users": {}}} + assert.True(t, byTable.matches(config.Change{Op: "insert", Table: "users"})) + assert.True(t, byTable.matches(config.Change{Op: "insert", Relation: "users"})) + assert.False(t, byTable.matches(config.Change{Op: "insert", Table: "orders"})) +} + +func TestSubscribersPublishAndClose(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe(config.StreamOptions{}) + + subs.publish(context.Background(), config.Change{Op: "insert", Table: "users", Source: "s"}) + + select { + case change := <-stream.Changes(): + assert.Equal(t, "insert", change.Op) + assert.Equal(t, "users", change.Table) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for change") + } + + subs.closeAll() + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok, "channel should be closed after closeAll") + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for close") + } +} + +func TestSubscribeBufferClamp(t *testing.T) { + subs := newSubscribers() + + def := subs.subscribe(config.StreamOptions{Buffer: 0}).(*subscription) + assert.Equal(t, defaultStreamBuffer, cap(def.in)) + + neg := subs.subscribe(config.StreamOptions{Buffer: -5}).(*subscription) + assert.Equal(t, defaultStreamBuffer, cap(neg.in)) + + exact := subs.subscribe(config.StreamOptions{Buffer: 7}).(*subscription) + assert.Equal(t, 7, cap(exact.in)) + + huge := subs.subscribe(config.StreamOptions{Buffer: maxStreamBuffer + 100}).(*subscription) + assert.Equal(t, maxStreamBuffer, cap(huge.in)) +} + +func TestSubscribeAssignsUniqueIncreasingIDs(t *testing.T) { + subs := newSubscribers() + a := subs.subscribe(config.StreamOptions{}).(*subscription) + b := subs.subscribe(config.StreamOptions{}).(*subscription) + assert.Equal(t, uint64(1), a.id) + assert.Equal(t, uint64(2), b.id) +} + +func TestPublishNeverBlocksAndClosesLaggard(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe(config.StreamOptions{Buffer: 1}) + + done := make(chan struct{}) + go func() { + for i := 0; i < 1000; i++ { + subs.publish(context.Background(), config.Change{Op: "insert", Table: "t"}) + } + close(done) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("publish blocked on a non-reading subscriber") + } + + for { + select { + case _, ok := <-stream.Changes(): + if !ok { + return + } + case <-time.After(2 * time.Second): + t.Fatal("laggard subscription was not closed on overflow") + } + } +} + +func TestSubscribersFilterByOp(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe(config.StreamOptions{Ops: []string{"delete"}}) + defer stream.Close() + + subs.publish(context.Background(), config.Change{Op: "insert", Table: "users"}) + subs.publish(context.Background(), config.Change{Op: "delete", Table: "users"}) + + select { + case change := <-stream.Changes(): + require.Equal(t, "delete", change.Op) + case <-time.After(2 * time.Second): + t.Fatal("timed out") + } +} diff --git a/service/sql/errors.go b/service/sql/errors.go index c82e58bac..c02e71933 100644 --- a/service/sql/errors.go +++ b/service/sql/errors.go @@ -9,10 +9,12 @@ import ( ) var ( - ErrPoolClosed = apierror.New(apierror.Unavailable, "connection pool is closed").WithRetryable(apierror.False) - ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) - ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) - ErrPoolFactoryRequired = apierror.New(apierror.Invalid, "pool factory is required").WithRetryable(apierror.False) + ErrPoolClosed = apierror.New(apierror.Unavailable, "connection pool is closed").WithRetryable(apierror.False) + ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) + ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) + ErrPoolFactoryRequired = apierror.New(apierror.Invalid, "pool factory is required").WithRetryable(apierror.False) + ErrNotSQLiteConn = apierror.New(apierror.Invalid, "underlying connection is not a SQLite connection").WithRetryable(apierror.False) + ErrCDCMemoryUnsupported = apierror.New(apierror.Invalid, "sqlite cdc requires a file-backed database").WithRetryable(apierror.False) ) func NewPingError(err error) apierror.Error { diff --git a/service/sql/factory.go b/service/sql/factory.go index 5810709d6..46d69a563 100644 --- a/service/sql/factory.go +++ b/service/sql/factory.go @@ -19,6 +19,8 @@ type PoolFactoryAPI interface { CreateSQLitePool(ctx context.Context, cfg *config.SQLiteConfig) (*ConnPool, error) } +var sqliteDriverName = "sqlite3" + // DefaultPoolFactory is the default implementation of PoolFactoryAPI type DefaultPoolFactory struct{} @@ -76,7 +78,7 @@ func (f *DefaultPoolFactory) CreateSQLitePool(ctx context.Context, cfg *config.S dsn = "file:" + cfg.File + "?mode=rwc" } - db, err := sql.Open("sqlite3", dsn) + db, err := sql.Open(sqliteDriverName, dsn) if err != nil { return nil, NewSQLiteConnectionCreationError(err) } diff --git a/service/sql/sqlite_cdc.go b/service/sql/sqlite_cdc.go new file mode 100644 index 000000000..7e51fba33 --- /dev/null +++ b/service/sql/sqlite_cdc.go @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sql + +import ( + "database/sql" + "path/filepath" + "sync" + + "github.com/mattn/go-sqlite3" +) + +const sqliteCDCDriver = "sqlite3_wippy" + +const ( + CDCInsert = sqlite3.SQLITE_INSERT + CDCUpdate = sqlite3.SQLITE_UPDATE + CDCDelete = sqlite3.SQLITE_DELETE +) + +type CDCSink interface { + PreUpdate(op int, table string, rowid int64, old, new []any) + Commit() + Rollback() +} + +var ( + cdcMu sync.RWMutex + cdcSinks = make(map[string]CDCSink) +) + +func init() { + sql.Register(sqliteCDCDriver, &sqlite3.SQLiteDriver{ConnectHook: cdcConnectHook}) + sqliteDriverName = sqliteCDCDriver +} + +func cdcConnectHook(conn *sqlite3.SQLiteConn) error { + file := normalizeCDCPath(conn.GetFilename("main")) + if file == "" { + return nil + } + cdcMu.RLock() + sink, ok := cdcSinks[file] + cdcMu.RUnlock() + if ok { + bindCDCHooks(conn, sink) + } + return nil +} + +func RegisterCDCSink(file string, sink CDCSink) { + cdcMu.Lock() + cdcSinks[file] = sink + cdcMu.Unlock() +} + +func UnregisterCDCSink(file string) { + cdcMu.Lock() + delete(cdcSinks, file) + cdcMu.Unlock() +} + +func InstallCDCHooksOnRaw(raw any, sink CDCSink) (string, error) { + conn, ok := raw.(*sqlite3.SQLiteConn) + if !ok { + return "", ErrNotSQLiteConn + } + file := normalizeCDCPath(conn.GetFilename("main")) + if file == "" { + return "", ErrCDCMemoryUnsupported + } + bindCDCHooks(conn, sink) + return file, nil +} + +func ClearCDCHooksOnRaw(raw any) error { + conn, ok := raw.(*sqlite3.SQLiteConn) + if !ok { + return ErrNotSQLiteConn + } + conn.RegisterPreUpdateHook(nil) + conn.RegisterCommitHook(nil) + conn.RegisterRollbackHook(nil) + return nil +} + +func normalizeCDCPath(path string) string { + if path == "" { + return "" + } + abs, err := filepath.Abs(path) + if err != nil { + return filepath.Clean(path) + } + return abs +} + +func bindCDCHooks(conn *sqlite3.SQLiteConn, sink CDCSink) { + conn.RegisterPreUpdateHook(func(d sqlite3.SQLitePreUpdateData) { + count := d.Count() + var oldRow, newRow []any + var rowid int64 + switch d.Op { + case sqlite3.SQLITE_INSERT: + newRow = scanPreUpdateRow(&d, count, true) + rowid = d.NewRowID + case sqlite3.SQLITE_DELETE: + oldRow = scanPreUpdateRow(&d, count, false) + rowid = d.OldRowID + case sqlite3.SQLITE_UPDATE: + oldRow = scanPreUpdateRow(&d, count, false) + newRow = scanPreUpdateRow(&d, count, true) + rowid = d.NewRowID + } + sink.PreUpdate(d.Op, d.TableName, rowid, oldRow, newRow) + }) + conn.RegisterCommitHook(func() int { + sink.Commit() + return 0 + }) + conn.RegisterRollbackHook(func() { + sink.Rollback() + }) +} + +func scanPreUpdateRow(d *sqlite3.SQLitePreUpdateData, count int, isNew bool) []any { + if count <= 0 { + return nil + } + vals := make([]any, count) + if isNew { + _ = d.New(vals...) + } else { + _ = d.Old(vals...) + } + return vals +} diff --git a/test.sh b/test.sh index 75e4cae47..9027de57a 100755 --- a/test.sh +++ b/test.sh @@ -26,7 +26,10 @@ for arg in "$@"; do esac done -go test ./api/service/cdc ./service/cdc/postgres ./runtime/lua/modules/cdc ./boot/components/dispatchers +go test ./api/service/cdc ./service/cdc/postgres ./service/cdc/sqlite ./runtime/lua/modules/cdc ./boot/components/dispatchers + +echo "running sqlite cdc integration tests (local temp file, no docker)" +CGO_ENABLED=1 go test -tags "integration sqlite_preupdate_hook" ./service/cdc/sqlite if [[ -n "${WIPPY_CDC_IT_REPL_DSN:-}" && -n "${WIPPY_CDC_IT_ADMIN_DSN:-}" ]]; then go test -tags integration ./service/cdc/postgres From 29ee80a4134f5d00ea4279e3fa087bff5f1fb6a3 Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Thu, 18 Jun 2026 11:58:12 -0300 Subject: [PATCH 02/47] refactor(sql): decompose engines into a self-registering registry Why: PR #351 bolted SQLite CDC specifics onto the generalized service/sql driver (a build-tagged preupdate-hook file, a mutable driver-name global, and CDC-only errors), and the package dispatched engines through kind switches. Adding a database therefore meant editing core dispatch code, and the generalized driver carried engine- and CDC-specific knowledge it should not have. What: service/sql core now exposes only two public seams, RegisterEngine and RegisterDriver, and dispatches purely via engineFor(kind); the manager, factory, and ConnPool.UpdateConfig no longer switch on engine kind. An EngineConfig contract lets the generic create/update lifecycle validate and read lifecycle settings without knowing the concrete type. Built-in engines move into self-registering sub-packages that use only the public API: engine/standard (Postgres and MySQL, each with its own DSN builder, removing the buildDSN/getDriver kind switch), engine/sqlite (DSN, WAL, single-writer tuning), and engine/all (blank-imports the built-ins). Boot blank-imports engine/all as the single wiring point. The database/sql driver override is applied centrally in createPool, so engines stay override-agnostic. All SQLite CDC hook code (custom sqlite3_wippy driver, sink registry, preupdate scan, install/clear-on-raw, CDC errors) moves to service/cdc/sqlite/hook.go and registers its driver through RegisterDriver, leaving service/sql with zero CDC knowledge. Adding a database is now a new self-registering package that touches no existing core file. --- api/service/sql/config.go | 18 ++ boot/components/service/storage/sql.go | 3 + .../{sql/sqlite_cdc.go => cdc/sqlite/hook.go} | 45 ++-- service/cdc/sqlite/source.go | 14 +- service/sql/conn.go | 155 ++---------- service/sql/conn_test.go | 109 --------- service/sql/driver.go | 38 +++ service/sql/driver_test.go | 95 ++++++++ service/sql/engine.go | 113 +++++++++ service/sql/engine/all/all.go | 12 + service/sql/engine/sqlite/sqlite.go | 89 +++++++ service/sql/engine/sqlite/sqlite_test.go | 78 ++++++ service/sql/engine/standard/standard.go | 222 ++++++++++++++++++ service/sql/engine/standard/standard_test.go | 192 +++++++++++++++ service/sql/engines_stub_test.go | 101 ++++++++ service/sql/errors.go | 32 +-- service/sql/factory.go | 102 ++------ service/sql/factory_test.go | 204 ++++++---------- service/sql/manager.go | 147 ++---------- service/sql/manager_test.go | 165 +++---------- 20 files changed, 1165 insertions(+), 769 deletions(-) rename service/{sql/sqlite_cdc.go => cdc/sqlite/hook.go} (63%) create mode 100644 service/sql/driver.go create mode 100644 service/sql/driver_test.go create mode 100644 service/sql/engine.go create mode 100644 service/sql/engine/all/all.go create mode 100644 service/sql/engine/sqlite/sqlite.go create mode 100644 service/sql/engine/sqlite/sqlite_test.go create mode 100644 service/sql/engine/standard/standard.go create mode 100644 service/sql/engine/standard/standard_test.go create mode 100644 service/sql/engines_stub_test.go diff --git a/api/service/sql/config.go b/api/service/sql/config.go index f1cd6d443..a7d75ce91 100644 --- a/api/service/sql/config.go +++ b/api/service/sql/config.go @@ -36,6 +36,14 @@ const ( DefaultMaxLifetime = 1 * time.Hour ) +// EngineConfig is the contract every engine configuration satisfies, letting the +// generic pool lifecycle validate and read lifecycle settings without knowing the +// concrete engine type. +type EngineConfig interface { + Validate() error + LifecycleConfig() supervisor.LifecycleConfig +} + type ( // PoolConfig defines settings for a database connection pool PoolConfig struct { @@ -111,6 +119,11 @@ func (c *SQLiteConfig) InitDefaults() { c.Lifecycle.InitDefaults() } +// LifecycleConfig returns the supervisor lifecycle settings for the database. +func (c *DBConfig) LifecycleConfig() supervisor.LifecycleConfig { + return c.Lifecycle +} + // Validate checks if the DBConfig has all required fields set to valid values func (c *DBConfig) Validate() error { if c.Host == "" && c.HostEnv == "" { @@ -148,6 +161,11 @@ func (c *DBConfig) Validate() error { return nil } +// LifecycleConfig returns the supervisor lifecycle settings for the database. +func (c *SQLiteConfig) LifecycleConfig() supervisor.LifecycleConfig { + return c.Lifecycle +} + // Validate checks if the SQLiteConfig has all required fields set to valid values func (c *SQLiteConfig) Validate() error { if c.File == "" { diff --git a/boot/components/service/storage/sql.go b/boot/components/service/storage/sql.go index a4ad16e9c..b59786e91 100644 --- a/boot/components/service/storage/sql.go +++ b/boot/components/service/storage/sql.go @@ -13,6 +13,9 @@ import ( bootpkg "github.com/wippyai/runtime/boot" bootsystem "github.com/wippyai/runtime/boot/components/system" "github.com/wippyai/runtime/service/sql" + + // Register the built-in SQL engines (postgres, mysql, sqlite) with the manager. + _ "github.com/wippyai/runtime/service/sql/engine/all" ) func SQL() boot.Component { diff --git a/service/sql/sqlite_cdc.go b/service/cdc/sqlite/hook.go similarity index 63% rename from service/sql/sqlite_cdc.go rename to service/cdc/sqlite/hook.go index 7e51fba33..a8a2a5f27 100644 --- a/service/sql/sqlite_cdc.go +++ b/service/cdc/sqlite/hook.go @@ -2,25 +2,38 @@ //go:build sqlite_preupdate_hook -package sql +package sqlite import ( - "database/sql" "path/filepath" "sync" + "database/sql" + "github.com/mattn/go-sqlite3" + + apierror "github.com/wippyai/runtime/api/error" + sqlconfig "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" ) +// sqliteCDCDriver is a SQLite driver variant whose ConnectHook rebinds preupdate +// hooks on every connection the pool opens to a file with a registered sink. const sqliteCDCDriver = "sqlite3_wippy" const ( - CDCInsert = sqlite3.SQLITE_INSERT - CDCUpdate = sqlite3.SQLITE_UPDATE - CDCDelete = sqlite3.SQLITE_DELETE + cdcInsert = sqlite3.SQLITE_INSERT + cdcUpdate = sqlite3.SQLITE_UPDATE + cdcDelete = sqlite3.SQLITE_DELETE +) + +var ( + errNotSQLiteConn = apierror.New(apierror.Invalid, "underlying connection is not a SQLite connection").WithRetryable(apierror.False) + errCDCMemoryUnsupported = apierror.New(apierror.Invalid, "sqlite cdc requires a file-backed database").WithRetryable(apierror.False) ) -type CDCSink interface { +// cdcSink receives row-level changes observed on the writer connection. +type cdcSink interface { PreUpdate(op int, table string, rowid int64, old, new []any) Commit() Rollback() @@ -28,12 +41,12 @@ type CDCSink interface { var ( cdcMu sync.RWMutex - cdcSinks = make(map[string]CDCSink) + cdcSinks = make(map[string]cdcSink) ) func init() { sql.Register(sqliteCDCDriver, &sqlite3.SQLiteDriver{ConnectHook: cdcConnectHook}) - sqliteDriverName = sqliteCDCDriver + sqlservice.RegisterDriver(sqlconfig.SQLite, sqliteCDCDriver) } func cdcConnectHook(conn *sqlite3.SQLiteConn) error { @@ -50,35 +63,35 @@ func cdcConnectHook(conn *sqlite3.SQLiteConn) error { return nil } -func RegisterCDCSink(file string, sink CDCSink) { +func registerSink(file string, sink cdcSink) { cdcMu.Lock() cdcSinks[file] = sink cdcMu.Unlock() } -func UnregisterCDCSink(file string) { +func unregisterSink(file string) { cdcMu.Lock() delete(cdcSinks, file) cdcMu.Unlock() } -func InstallCDCHooksOnRaw(raw any, sink CDCSink) (string, error) { +func installHooksOnRaw(raw any, sink cdcSink) (string, error) { conn, ok := raw.(*sqlite3.SQLiteConn) if !ok { - return "", ErrNotSQLiteConn + return "", errNotSQLiteConn } file := normalizeCDCPath(conn.GetFilename("main")) if file == "" { - return "", ErrCDCMemoryUnsupported + return "", errCDCMemoryUnsupported } bindCDCHooks(conn, sink) return file, nil } -func ClearCDCHooksOnRaw(raw any) error { +func clearHooksOnRaw(raw any) error { conn, ok := raw.(*sqlite3.SQLiteConn) if !ok { - return ErrNotSQLiteConn + return errNotSQLiteConn } conn.RegisterPreUpdateHook(nil) conn.RegisterCommitHook(nil) @@ -97,7 +110,7 @@ func normalizeCDCPath(path string) string { return abs } -func bindCDCHooks(conn *sqlite3.SQLiteConn, sink CDCSink) { +func bindCDCHooks(conn *sqlite3.SQLiteConn, sink cdcSink) { conn.RegisterPreUpdateHook(func(d sqlite3.SQLitePreUpdateData) { count := d.Count() var oldRow, newRow []any diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index 9469bdeb0..aaa1aa27e 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -169,7 +169,7 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { var file string if rawErr := conn.Raw(func(dc any) error { - f, e := sqlservice.InstallCDCHooksOnRaw(dc, s) + f, e := installHooksOnRaw(dc, s) file = f return e }); rawErr != nil { @@ -177,7 +177,7 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { res.Release() return nil, rawErr } - sqlservice.RegisterCDCSink(file, s) + registerSink(file, s) s.mu.Lock() s.poolRes = res @@ -319,12 +319,12 @@ func (s *Source) Stop(ctx context.Context) error { } func (s *Source) detachHooks(ctx context.Context, writerDB *sql.DB, file string) { - sqlservice.UnregisterCDCSink(file) + unregisterSink(file) conn, err := writerDB.Conn(ctx) if err != nil { return } - _ = conn.Raw(sqlservice.ClearCDCHooksOnRaw) + _ = conn.Raw(clearHooksOnRaw) _ = conn.Close() } @@ -436,11 +436,11 @@ func (s *Source) columnsFor(ctx context.Context, table string) []columnInfo { func opString(op int) string { switch op { - case sqlservice.CDCInsert: + case cdcInsert: return "insert" - case sqlservice.CDCUpdate: + case cdcUpdate: return "update" - case sqlservice.CDCDelete: + case cdcDelete: return "delete" default: return "unknown" diff --git a/service/sql/conn.go b/service/sql/conn.go index 57bd07c7e..b8f81a6ea 100644 --- a/service/sql/conn.go +++ b/service/sql/conn.go @@ -5,10 +5,6 @@ package sql import ( "context" "database/sql" - "net/url" - "sort" - "strconv" - "strings" "sync" "sync/atomic" @@ -71,46 +67,35 @@ func (p *ConnPool) Stop(ctx context.Context) error { } } -// UpdateConfig updates the pool configuration +// UpdateConfig updates the pool configuration. It delegates engine-specific +// validation and tuning to the engine registered for the pool's kind. func (p *ConnPool) UpdateConfig(cfg any) error { if p.closed.Load() { return ErrPoolClosed } - switch c := cfg.(type) { - case *config.DBConfig: - if p.kind == config.SQLite { - return NewInvalidConfigTypeError("DBConfig", config.SQLite) - } - - if err := c.Validate(); err != nil { - return NewInvalidConfigError(err) - } - - p.db.SetMaxOpenConns(c.Pool.MaxOpen) - p.db.SetMaxIdleConns(c.Pool.MaxIdle) - p.db.SetConnMaxLifetime(c.Pool.MaxLifetime) + ec, ok := cfg.(config.EngineConfig) + if !ok { + return NewUnsupportedConfigTypeError(p.kind) + } - var cfg any = c - p.config.Store(&cfg) + eng, ok := engineFor(p.kind) + if !ok { + return NewUnsupportedConfigTypeError(p.kind) + } - case *config.SQLiteConfig: - if p.kind != config.SQLite { - return NewInvalidConfigTypeError("SQLiteConfig", p.kind) - } + if err := eng.ValidateConfigType(ec); err != nil { + return err + } - if err := c.Validate(); err != nil { - return NewInvalidConfigError(err) - } + if err := ec.Validate(); err != nil { + return NewInvalidConfigError(err) + } - p.db.SetConnMaxLifetime(c.Pool.MaxLifetime) + eng.Tune(p.db, ec) - var cfg any = c - p.config.Store(&cfg) - - default: - return NewUnsupportedConfigTypeError(p.kind) - } + var stored any = ec + p.config.Store(&stored) return nil } @@ -137,108 +122,6 @@ func (p *ConnPool) Acquire( return newDBConn(p, p.db, p.kind), nil } -// Helper to build DSN string for different database types -func buildDSN(kind registry.Kind, cfg *config.DBConfig) (string, error) { - switch kind { - case config.Postgres: - opts := buildPostgresOptionsString(cfg.Options) - var b strings.Builder - b.Grow(128) - b.WriteString("host=") - b.WriteString(cfg.Host) - b.WriteString(" port=") - b.WriteString(strconv.Itoa(cfg.Port)) - b.WriteString(" user=") - b.WriteString(cfg.Username) - b.WriteString(" password=") - b.WriteString(cfg.Password) - b.WriteString(" dbname=") - b.WriteString(cfg.Database) - if opts != "" { - b.WriteString(" ") - b.WriteString(opts) - } - return b.String(), nil - - case config.MySQL: - opts := buildMySQLOptionsString(cfg.Options) - var b strings.Builder - b.Grow(128) - b.WriteString(cfg.Username) - b.WriteString(":") - b.WriteString(cfg.Password) - b.WriteString("@tcp(") - b.WriteString(cfg.Host) - b.WriteString(":") - b.WriteString(strconv.Itoa(cfg.Port)) - b.WriteString(")/") - b.WriteString(cfg.Database) - if opts != "" { - b.WriteString("?") - b.WriteString(opts) - } - return b.String(), nil - - default: - return "", NewUnsupportedDatabaseTypeError(kind) - } -} - -func getDriver(kind registry.Kind) string { - switch kind { - case config.Postgres: - return "postgres" - case config.MySQL: - return "mysql" - default: - return kind - } -} - -// buildPostgresOptionsString renders lib/pq keyword/value options. -func buildPostgresOptionsString(options map[string]string) string { - if len(options) == 0 { - return "" - } - - keys := make([]string, 0, len(options)) - for k := range options { - keys = append(keys, k) - } - sort.Strings(keys) - - var b strings.Builder - b.Grow(len(options) * 20) - for i, k := range keys { - if i > 0 { - b.WriteString(" ") - } - b.WriteString(k) - b.WriteString("=") - b.WriteString(options[k]) - } - - return b.String() -} - -func buildMySQLOptionsString(options map[string]string) string { - if len(options) == 0 { - return "" - } - - values := url.Values{} - for k, v := range options { - values.Set(k, v) - } - return values.Encode() -} - -// Helper kept for older internal tests and benchmarks. PostgreSQL was the only -// historical caller that used this space-separated keyword/value form. -func buildOptionsString(options map[string]string) string { - return buildPostgresOptionsString(options) -} - // DBConn represents a database connection resource type DBConn struct { pool *ConnPool diff --git a/service/sql/conn_test.go b/service/sql/conn_test.go index d1997a4a9..d104f7d7f 100644 --- a/service/sql/conn_test.go +++ b/service/sql/conn_test.go @@ -305,87 +305,6 @@ func TestConnPool_UpdateConfigAfterClose(t *testing.T) { assert.Contains(t, err.Error(), "closed") } -func TestBuildDSN(t *testing.T) { - tests := []struct { - cfg *apiconfig.DBConfig - name string - kind string - wantErr bool - }{ - { - name: "postgres", - kind: apiconfig.Postgres, - cfg: &apiconfig.DBConfig{ - Host: "localhost", Port: 5432, Database: "db", - Username: "user", Password: "pass", - }, - wantErr: false, - }, - { - name: "mysql", - kind: apiconfig.MySQL, - cfg: &apiconfig.DBConfig{ - Host: "localhost", Port: 3306, Database: "db", - Username: "user", Password: "pass", - }, - wantErr: false, - }, - { - name: "unsupported", - kind: "db.unknown", - cfg: &apiconfig.DBConfig{}, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := buildDSN(tt.kind, tt.cfg) - if tt.wantErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestBuildOptionsString(t *testing.T) { - t.Run("empty options", func(t *testing.T) { - result := buildOptionsString(nil) - assert.Empty(t, result) - }) - - t.Run("single option", func(t *testing.T) { - result := buildOptionsString(map[string]string{"sslmode": "disable"}) - assert.Equal(t, "sslmode=disable", result) - }) - - t.Run("postgres options are stable and space separated", func(t *testing.T) { - result := buildPostgresOptionsString(map[string]string{ - "sslmode": "disable", - "connect_timeout": "10", - "application_name": "test", - }) - assert.Equal(t, "application_name=test connect_timeout=10 sslmode=disable", result) - }) - - t.Run("mysql options are stable query parameters", func(t *testing.T) { - result := buildMySQLOptionsString(map[string]string{ - "charset": "utf8mb4", - "parseTime": "true", - "timeout": "2s", - }) - assert.Equal(t, "charset=utf8mb4&parseTime=true&timeout=2s", result) - }) -} - -func TestGetDriver(t *testing.T) { - assert.Equal(t, "postgres", getDriver(apiconfig.Postgres)) - assert.Equal(t, "mysql", getDriver(apiconfig.MySQL)) - assert.Equal(t, "unknown", getDriver("unknown")) -} - // Benchmarks func newBenchPool(b *testing.B) *ConnPool { @@ -455,31 +374,3 @@ func BenchmarkConnPool_ConcurrentAcquire(b *testing.B) { } }) } - -func BenchmarkBuildDSN_Postgres(b *testing.B) { - cfg := &apiconfig.DBConfig{ - Host: "localhost", Port: 5432, Database: "db", - Username: "user", Password: "pass", - Options: map[string]string{"sslmode": "disable"}, - } - - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - _, _ = buildDSN(apiconfig.Postgres, cfg) - } -} - -func BenchmarkBuildOptionsString(b *testing.B) { - opts := map[string]string{ - "sslmode": "disable", - "connect_timeout": "10", - "application_name": "test", - } - - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - _ = buildOptionsString(opts) - } -} diff --git a/service/sql/driver.go b/service/sql/driver.go new file mode 100644 index 000000000..7cafa44f9 --- /dev/null +++ b/service/sql/driver.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import ( + "sync" + + "github.com/wippyai/runtime/api/registry" +) + +// driverOverrides lets an out-of-tree extension swap the database/sql driver an +// engine opens, keyed by registry kind. It is the only seam through which engine +// connection behavior (for example, a SQLite build that installs preupdate hooks) +// is altered without modifying the core package. +var ( + driverMu sync.RWMutex + driverOverrides = make(map[registry.Kind]string) +) + +// RegisterDriver overrides the database/sql driver name used for the given kind. +// Extensions call this from an init function, before any pool is created. +func RegisterDriver(kind registry.Kind, name string) { + driverMu.Lock() + driverOverrides[kind] = name + driverMu.Unlock() +} + +// driverName returns the registered override for kind, falling back to def. +func driverName(kind registry.Kind, def string) string { + driverMu.RLock() + name, ok := driverOverrides[kind] + driverMu.RUnlock() + if ok { + return name + } + + return def +} diff --git a/service/sql/driver_test.go b/service/sql/driver_test.go new file mode 100644 index 000000000..9ae3cab82 --- /dev/null +++ b/service/sql/driver_test.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + "go.uber.org/zap" +) + +// fixedConfigEngine returns a preset SQLite config regardless of the entry, so driver +// tests can drive createPool to sql.Open without a transcoder. Prepare can be forced +// to fail to exercise the create lifecycle's close-on-error branch. +type fixedConfigEngine struct { + prepareErr error + kind registry.Kind + driver string +} + +func (e fixedConfigEngine) Kind() registry.Kind { return e.kind } + +func (e fixedConfigEngine) DriverName() string { return e.driver } + +func (fixedConfigEngine) DecodeConfig(context.Context, payload.Transcoder, registry.Entry) (config.EngineConfig, error) { + return &config.SQLiteConfig{File: ":memory:", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, nil +} + +func (fixedConfigEngine) ResolveEnv(context.Context, EngineDeps, config.EngineConfig) error { + return nil +} + +func (fixedConfigEngine) BuildDSN(config.EngineConfig) (string, error) { + return ":memory:", nil +} + +func (e fixedConfigEngine) Prepare(context.Context, *sql.DB, config.EngineConfig) error { + return e.prepareErr +} + +func (fixedConfigEngine) Tune(*sql.DB, config.EngineConfig) {} + +func (fixedConfigEngine) ValidateConfigType(config.EngineConfig) error { + return nil +} + +func TestDriverNameFallbackAndOverride(t *testing.T) { + const kind = registry.Kind("db.sql.drivernametest") + + assert.Equal(t, "default-drv", driverName(kind, "default-drv")) + + RegisterDriver(kind, "override-drv") + assert.Equal(t, "override-drv", driverName(kind, "default-drv")) +} + +func TestCreatePoolAppliesDriverOverride(t *testing.T) { + const kind = registry.Kind("db.sql.overridetest") + RegisterEngine(fixedConfigEngine{kind: kind, driver: "sqlite3"}) + + factory := &DefaultPoolFactory{} + deps := EngineDeps{Log: zap.NewNop()} + entry := registry.Entry{ID: registry.NewID("test", "ov"), Kind: kind, Data: payload.New("x")} + + pool, _, err := factory.CreatePool(context.Background(), deps, entry) + require.NoError(t, err) + require.NotNil(t, pool) + require.NoError(t, pool.Stop(context.Background())) + + RegisterDriver(kind, "sentinel-missing-driver") + overridden, _, err := factory.CreatePool(context.Background(), deps, entry) + require.Error(t, err) + assert.Nil(t, overridden) +} + +func TestCreatePoolClosesOnPrepareError(t *testing.T) { + const kind = registry.Kind("db.sql.preparefailtest") + prepErr := errors.New("prepare boom") + RegisterEngine(fixedConfigEngine{kind: kind, driver: "sqlite3", prepareErr: prepErr}) + + entry := registry.Entry{ID: registry.NewID("test", "pf"), Kind: kind, Data: payload.New("x")} + pool, _, err := (&DefaultPoolFactory{}).CreatePool(context.Background(), EngineDeps{Log: zap.NewNop()}, entry) + + require.Error(t, err) + assert.Nil(t, pool) + assert.ErrorIs(t, err, prepErr) +} diff --git a/service/sql/engine.go b/service/sql/engine.go new file mode 100644 index 000000000..be45f1af2 --- /dev/null +++ b/service/sql/engine.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import ( + "context" + "database/sql" + + envapi "github.com/wippyai/runtime/api/env" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + "go.uber.org/zap" +) + +// EngineDeps carries the shared collaborators an engine needs to turn a registry +// entry into a configured pool. +type EngineDeps struct { + Transcoder payload.Transcoder + Env envapi.Registry + Log *zap.Logger +} + +// Engine is a self-contained SQL dialect. Each engine knows how to decode its +// configuration, resolve environment overrides, open and tune a pool, and run any +// post-open preparation. Engines register themselves with RegisterEngine, so adding +// a database never touches the factory or manager dispatch surface. +type Engine interface { + Kind() registry.Kind + DriverName() string + DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) + ResolveEnv(ctx context.Context, deps EngineDeps, cfg config.EngineConfig) error + BuildDSN(cfg config.EngineConfig) (string, error) + Prepare(ctx context.Context, db *sql.DB, cfg config.EngineConfig) error + Tune(db *sql.DB, cfg config.EngineConfig) + ValidateConfigType(cfg config.EngineConfig) error +} + +var engines = make(map[registry.Kind]Engine) + +// RegisterEngine adds an engine to the registry under its kind. Intended to be +// called from engine package init functions. +func RegisterEngine(e Engine) { + engines[e.Kind()] = e +} + +// engineFor looks up the engine registered for a kind. +func engineFor(kind registry.Kind) (Engine, bool) { + e, ok := engines[kind] + return e, ok +} + +// createPool runs the generic create lifecycle for a known engine. +func createPool(ctx context.Context, deps EngineDeps, eng Engine, entry registry.Entry) (*ConnPool, config.EngineConfig, error) { + cfg, err := eng.DecodeConfig(ctx, deps.Transcoder, entry) + if err != nil { + return nil, nil, NewInvalidConfigError(err) + } + + if err := eng.ResolveEnv(ctx, deps, cfg); err != nil { + return nil, nil, err + } + + if err := cfg.Validate(); err != nil { + return nil, nil, NewInvalidConfigError(err) + } + + dsn, err := eng.BuildDSN(cfg) + if err != nil { + return nil, nil, NewInvalidDSNError(err) + } + + db, err := sql.Open(driverName(eng.Kind(), eng.DriverName()), dsn) + if err != nil { + return nil, nil, NewConnectionPoolCreationError(err) + } + + if err := eng.Prepare(ctx, db, cfg); err != nil { + _ = db.Close() + return nil, nil, err + } + + eng.Tune(db, cfg) + + pool := &ConnPool{ + kind: eng.Kind(), + db: db, + status: make(chan any, 1), + } + + var cfgAny any = cfg + pool.config.Store(&cfgAny) + + return pool, cfg, nil +} + +// updatePool runs the generic update lifecycle for a known engine. +func updatePool(ctx context.Context, deps EngineDeps, eng Engine, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) { + cfg, err := eng.DecodeConfig(ctx, deps.Transcoder, entry) + if err != nil { + return nil, NewInvalidConfigError(err) + } + + if err := eng.ResolveEnv(ctx, deps, cfg); err != nil { + return nil, err + } + + if err := pool.UpdateConfig(cfg); err != nil { + return nil, NewPoolUpdateError(err) + } + + return cfg, nil +} diff --git a/service/sql/engine/all/all.go b/service/sql/engine/all/all.go new file mode 100644 index 000000000..d63543976 --- /dev/null +++ b/service/sql/engine/all/all.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package all registers every built-in SQL engine via blank import. A composition +// root (for example the storage boot component) imports this package so the standard +// dialects are available; out-of-tree engines register themselves the same way. +package all + +import ( + // Blank imports register the built-in engines with service/sql via init. + _ "github.com/wippyai/runtime/service/sql/engine/sqlite" + _ "github.com/wippyai/runtime/service/sql/engine/standard" +) diff --git a/service/sql/engine/sqlite/sqlite.go b/service/sql/engine/sqlite/sqlite.go new file mode 100644 index 000000000..172d13a84 --- /dev/null +++ b/service/sql/engine/sqlite/sqlite.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package sqlite implements the file-backed SQLite engine. It registers itself with +// service/sql through the public engine seam; a CDC-enabled build overrides the +// underlying driver via service/sql.RegisterDriver, which core applies transparently. +package sqlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + entryutil "github.com/wippyai/runtime/internal/entry" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +// defaultDriver is the stock SQLite driver. A build with the preupdate hook overrides +// it via service/sql.RegisterDriver; the engine itself stays override-agnostic. +const defaultDriver = "sqlite3" + +type engine struct{} + +func init() { + sqlservice.RegisterEngine(engine{}) +} + +func (engine) Kind() registry.Kind { + return config.SQLite +} + +func (engine) DriverName() string { + return defaultDriver +} + +func (engine) DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) { + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + + return cfg, nil +} + +func (engine) ResolveEnv(context.Context, sqlservice.EngineDeps, config.EngineConfig) error { + return nil +} + +func (engine) BuildDSN(ec config.EngineConfig) (string, error) { + cfg, ok := ec.(*config.SQLiteConfig) + if !ok { + return "", sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), config.SQLite) + } + + if cfg.File == ":memory:" { + return ":memory:", nil + } + + return "file:" + cfg.File + "?mode=rwc", nil +} + +func (engine) Prepare(ctx context.Context, db *sql.DB, _ config.EngineConfig) error { + if _, err := db.ExecContext(ctx, "PRAGMA journal_mode=WAL;"); err != nil { + return sqlservice.NewWALModeError(err) + } + + return nil +} + +func (engine) Tune(db *sql.DB, ec config.EngineConfig) { + cfg, ok := ec.(*config.SQLiteConfig) + if !ok { + return + } + + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) +} + +func (engine) ValidateConfigType(ec config.EngineConfig) error { + if _, ok := ec.(*config.SQLiteConfig); !ok { + return sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), config.SQLite) + } + + return nil +} diff --git a/service/sql/engine/sqlite/sqlite_test.go b/service/sql/engine/sqlite/sqlite_test.go new file mode 100644 index 000000000..a5223cf8d --- /dev/null +++ b/service/sql/engine/sqlite/sqlite_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" + "go.uber.org/zap" +) + +func TestKindDriverRegistered(t *testing.T) { + e := engine{} + assert.Equal(t, config.SQLite, e.Kind()) + assert.Equal(t, "sqlite3", e.DriverName()) + + _, _, err := (&sqlservice.DefaultPoolFactory{}).CreatePool( + context.Background(), + sqlservice.EngineDeps{Log: zap.NewNop()}, + registry.Entry{ID: registry.NewID("t", "x"), Kind: config.SQLite, Data: nil}, + ) + require.Error(t, err) + assert.NotContains(t, err.Error(), "unsupported entry kind") +} + +func TestBuildDSN(t *testing.T) { + e := engine{} + + mem, err := e.BuildDSN(&config.SQLiteConfig{File: ":memory:"}) + require.NoError(t, err) + assert.Equal(t, ":memory:", mem) + + file, err := e.BuildDSN(&config.SQLiteConfig{File: "/tmp/app.db"}) + require.NoError(t, err) + assert.Equal(t, "file:/tmp/app.db?mode=rwc", file) + + _, err = e.BuildDSN(&config.DBConfig{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid config type") +} + +func TestPrepareEnablesWAL(t *testing.T) { + file := filepath.Join(t.TempDir(), "app.db") + db, err := sql.Open("sqlite3", "file:"+file+"?mode=rwc") + require.NoError(t, err) + defer func() { _ = db.Close() }() + + require.NoError(t, engine{}.Prepare(context.Background(), db, &config.SQLiteConfig{File: file})) + + var mode string + require.NoError(t, db.QueryRowContext(context.Background(), "PRAGMA journal_mode;").Scan(&mode)) + assert.Equal(t, "wal", mode) +} + +func TestTuneSingleWriter(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + defer func() { _ = db.Close() }() + + engine{}.Tune(db, &config.SQLiteConfig{Pool: config.PoolConfig{MaxLifetime: time.Hour}}) + assert.Equal(t, 1, db.Stats().MaxOpenConnections) +} + +func TestValidateConfigType(t *testing.T) { + require.NoError(t, engine{}.ValidateConfigType(&config.SQLiteConfig{File: ":memory:"})) + err := engine{}.ValidateConfigType(&config.DBConfig{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid config type") +} diff --git a/service/sql/engine/standard/standard.go b/service/sql/engine/standard/standard.go new file mode 100644 index 000000000..8acbf9ded --- /dev/null +++ b/service/sql/engine/standard/standard.go @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package standard implements the network SQL engines (PostgreSQL and MySQL) that +// share DBConfig. It registers itself with service/sql through the public engine +// seam, so the core package carries no knowledge of these dialects. +package standard + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + + envapi "github.com/wippyai/runtime/api/env" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + entryutil "github.com/wippyai/runtime/internal/entry" + sqlservice "github.com/wippyai/runtime/service/sql" + "go.uber.org/zap" +) + +// engine serves a network SQL dialect. Postgres and MySQL share the same DBConfig, +// env resolution, and pool tuning, differing only in driver name and DSN format, so +// each is a separate instance carrying its own DSN builder. +type engine struct { + dsn func(*config.DBConfig) (string, error) + kind registry.Kind + driver string +} + +func init() { + sqlservice.RegisterEngine(engine{kind: config.Postgres, driver: "postgres", dsn: buildPostgresDSN}) + sqlservice.RegisterEngine(engine{kind: config.MySQL, driver: "mysql", dsn: buildMySQLDSN}) +} + +func (e engine) Kind() registry.Kind { + return e.kind +} + +func (e engine) DriverName() string { + return e.driver +} + +func (engine) DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) { + cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + + return cfg, nil +} + +func (e engine) ResolveEnv(ctx context.Context, deps sqlservice.EngineDeps, ec config.EngineConfig) error { + cfg, ok := ec.(*config.DBConfig) + if !ok { + return sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + + if v := resolveEnvVar(ctx, deps.Env, deps.Log, cfg.HostEnv, "host"); v != "" { + cfg.Host = v + } + if v := resolveEnvVar(ctx, deps.Env, deps.Log, cfg.PortEnv, "port"); v != "" { + port, err := strconv.Atoi(v) + if err != nil { + return sqlservice.NewInvalidPortError(cfg.PortEnv, err) + } + cfg.Port = port + } + if v := resolveEnvVar(ctx, deps.Env, deps.Log, cfg.DatabaseEnv, "database"); v != "" { + cfg.Database = v + } + if v := resolveEnvVar(ctx, deps.Env, deps.Log, cfg.UsernameEnv, "username"); v != "" { + cfg.Username = v + } + if v := resolveEnvVar(ctx, deps.Env, deps.Log, cfg.PasswordEnv, "password"); v != "" { + cfg.Password = v + } + + return nil +} + +func (e engine) BuildDSN(ec config.EngineConfig) (string, error) { + cfg, ok := ec.(*config.DBConfig) + if !ok { + return "", sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + + return e.dsn(cfg) +} + +func (engine) Prepare(context.Context, *sql.DB, config.EngineConfig) error { + return nil +} + +func (engine) Tune(db *sql.DB, ec config.EngineConfig) { + cfg, ok := ec.(*config.DBConfig) + if !ok { + return + } + + db.SetMaxOpenConns(cfg.Pool.MaxOpen) + db.SetMaxIdleConns(cfg.Pool.MaxIdle) + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) +} + +func (e engine) ValidateConfigType(ec config.EngineConfig) error { + if _, ok := ec.(*config.DBConfig); !ok { + return sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + + return nil +} + +func buildPostgresDSN(cfg *config.DBConfig) (string, error) { + opts := buildPostgresOptionsString(cfg.Options) + var b strings.Builder + b.Grow(128) + b.WriteString("host=") + b.WriteString(cfg.Host) + b.WriteString(" port=") + b.WriteString(strconv.Itoa(cfg.Port)) + b.WriteString(" user=") + b.WriteString(cfg.Username) + b.WriteString(" password=") + b.WriteString(cfg.Password) + b.WriteString(" dbname=") + b.WriteString(cfg.Database) + if opts != "" { + b.WriteString(" ") + b.WriteString(opts) + } + + return b.String(), nil +} + +func buildMySQLDSN(cfg *config.DBConfig) (string, error) { + opts := buildMySQLOptionsString(cfg.Options) + var b strings.Builder + b.Grow(128) + b.WriteString(cfg.Username) + b.WriteString(":") + b.WriteString(cfg.Password) + b.WriteString("@tcp(") + b.WriteString(cfg.Host) + b.WriteString(":") + b.WriteString(strconv.Itoa(cfg.Port)) + b.WriteString(")/") + b.WriteString(cfg.Database) + if opts != "" { + b.WriteString("?") + b.WriteString(opts) + } + + return b.String(), nil +} + +// buildPostgresOptionsString renders lib/pq keyword/value options. +func buildPostgresOptionsString(options map[string]string) string { + if len(options) == 0 { + return "" + } + + keys := make([]string, 0, len(options)) + for k := range options { + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + b.Grow(len(options) * 20) + for i, k := range keys { + if i > 0 { + b.WriteString(" ") + } + b.WriteString(k) + b.WriteString("=") + b.WriteString(options[k]) + } + + return b.String() +} + +func buildMySQLOptionsString(options map[string]string) string { + if len(options) == 0 { + return "" + } + + values := url.Values{} + for k, v := range options { + values.Set(k, v) + } + + return values.Encode() +} + +// resolveEnvVar looks up an environment variable and returns its value, logging and +// returning empty on any miss. +func resolveEnvVar(ctx context.Context, env envapi.Registry, log *zap.Logger, envVar, field string) string { + if envVar == "" || env == nil { + return "" + } + + val, found, err := env.Lookup(ctx, envVar) + if err != nil { + if log != nil { + log.Warn("failed to lookup env var", zap.String("field", field), zap.String("var", envVar), zap.Error(err)) + } + return "" + } + if !found { + if log != nil { + log.Warn("env var not found", zap.String("field", field), zap.String("var", envVar)) + } + return "" + } + + return val +} diff --git a/service/sql/engine/standard/standard_test.go b/service/sql/engine/standard/standard_test.go new file mode 100644 index 000000000..8fc9da6f7 --- /dev/null +++ b/service/sql/engine/standard/standard_test.go @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: MPL-2.0 + +package standard + +import ( + "context" + "database/sql" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + envapi "github.com/wippyai/runtime/api/env" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" + "go.uber.org/zap" +) + +func sqlOpenMemory(*testing.T) (*sql.DB, error) { + return sql.Open("sqlite3", ":memory:") +} + +type mockEnv struct{ vars map[string]string } + +func newMockEnv() *mockEnv { return &mockEnv{vars: make(map[string]string)} } + +func (m *mockEnv) Get(_ context.Context, name string) (string, error) { + if v, ok := m.vars[name]; ok { + return v, nil + } + return "", envapi.ErrVariableNotFound +} + +func (m *mockEnv) Lookup(_ context.Context, name string) (string, bool, error) { + v, ok := m.vars[name] + return v, ok, nil +} + +func (m *mockEnv) Set(_ context.Context, name, value string) error { + m.vars[name] = value + return nil +} + +func (m *mockEnv) All(_ context.Context) (map[string]string, error) { return m.vars, nil } + +func (m *mockEnv) GetStorage(_ context.Context, _ registry.ID) (envapi.Storage, error) { + return nil, envapi.ErrVariableNotFound +} + +func (m *mockEnv) RegisterStorage(_ registry.ID, _ envapi.Storage) {} + +func TestRegistered(t *testing.T) { + for _, k := range []registry.Kind{config.Postgres, config.MySQL} { + _, _, err := (&sqlservice.DefaultPoolFactory{}).CreatePool( + context.Background(), + sqlservice.EngineDeps{Log: zap.NewNop()}, + registry.Entry{ID: registry.NewID("t", "x"), Kind: k, Data: nil}, + ) + require.Error(t, err) + assert.NotContains(t, err.Error(), "unsupported entry kind", "engine %s must be registered", k) + } +} + +func TestBuildDSN(t *testing.T) { + tests := []struct { + name string + kind registry.Kind + cfg *config.DBConfig + expected string + }{ + { + name: "postgres", + kind: config.Postgres, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"sslmode": "disable"}}, + expected: "host=localhost port=5432 user=user password=pass dbname=testdb sslmode=disable", + }, + { + name: "postgres with connect timeout", + kind: config.Postgres, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"connect_timeout": "2", "sslmode": "disable"}}, + expected: "host=localhost port=5432 user=user password=pass dbname=testdb connect_timeout=2 sslmode=disable", + }, + { + name: "mysql", + kind: config.MySQL, + cfg: &config.DBConfig{Host: "localhost", Port: 3306, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"charset": "utf8mb4"}}, + expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4", + }, + { + name: "mysql with query options", + kind: config.MySQL, + cfg: &config.DBConfig{Host: "localhost", Port: 3306, Database: "testdb", Username: "user", Password: "pass", Options: map[string]string{"charset": "utf8mb4", "parseTime": "true", "timeout": "2s"}}, + expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4&parseTime=true&timeout=2s", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := engine{kind: tt.kind} + if tt.kind == config.Postgres { + e.dsn = buildPostgresDSN + } else { + e.dsn = buildMySQLDSN + } + dsn, err := e.BuildDSN(tt.cfg) + require.NoError(t, err) + assert.Equal(t, tt.expected, dsn) + }) + } +} + +func TestBuildDSN_WrongType(t *testing.T) { + e := engine{kind: config.Postgres, dsn: buildPostgresDSN} + _, err := e.BuildDSN(&config.SQLiteConfig{File: ":memory:"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid config type") +} + +func TestDriverNameAndKind(t *testing.T) { + e := engine{kind: config.Postgres, driver: "postgres"} + assert.Equal(t, config.Postgres, e.Kind()) + assert.Equal(t, "postgres", e.DriverName()) +} + +func TestOptionsStrings(t *testing.T) { + assert.Empty(t, buildPostgresOptionsString(nil)) + assert.Equal(t, "application_name=test connect_timeout=10 sslmode=disable", + buildPostgresOptionsString(map[string]string{"sslmode": "disable", "connect_timeout": "10", "application_name": "test"})) + assert.Empty(t, buildMySQLOptionsString(nil)) + assert.Equal(t, "charset=utf8mb4&parseTime=true&timeout=2s", + buildMySQLOptionsString(map[string]string{"charset": "utf8mb4", "parseTime": "true", "timeout": "2s"})) +} + +func TestResolveEnv(t *testing.T) { + ctx := context.Background() + e := engine{kind: config.Postgres} + + t.Run("resolves all fields", func(t *testing.T) { + env := newMockEnv() + require.NoError(t, env.Set(ctx, "DB_HOST", "env-host")) + require.NoError(t, env.Set(ctx, "DB_PORT", "9999")) + require.NoError(t, env.Set(ctx, "DB_NAME", "env-db")) + require.NoError(t, env.Set(ctx, "DB_USER", "env-user")) + require.NoError(t, env.Set(ctx, "DB_PASS", "env-pass")) + deps := sqlservice.EngineDeps{Env: env, Log: zap.NewNop()} + + cfg := &config.DBConfig{HostEnv: "DB_HOST", PortEnv: "DB_PORT", DatabaseEnv: "DB_NAME", UsernameEnv: "DB_USER", PasswordEnv: "DB_PASS"} + require.NoError(t, e.ResolveEnv(ctx, deps, cfg)) + assert.Equal(t, "env-host", cfg.Host) + assert.Equal(t, 9999, cfg.Port) + assert.Equal(t, "env-db", cfg.Database) + assert.Equal(t, "env-user", cfg.Username) + assert.Equal(t, "env-pass", cfg.Password) + }) + + t.Run("rejects non-numeric port", func(t *testing.T) { + env := newMockEnv() + require.NoError(t, env.Set(ctx, "DB_PORT", "not-a-number")) + deps := sqlservice.EngineDeps{Env: env, Log: zap.NewNop()} + err := e.ResolveEnv(ctx, deps, &config.DBConfig{PortEnv: "DB_PORT"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid port") + }) + + t.Run("no-op when env vars empty", func(t *testing.T) { + deps := sqlservice.EngineDeps{Env: newMockEnv(), Log: zap.NewNop()} + cfg := &config.DBConfig{Host: "static-host"} + require.NoError(t, e.ResolveEnv(ctx, deps, cfg)) + assert.Equal(t, "static-host", cfg.Host) + }) + + t.Run("rejects wrong config type", func(t *testing.T) { + deps := sqlservice.EngineDeps{Env: newMockEnv(), Log: zap.NewNop()} + err := e.ResolveEnv(ctx, deps, &config.SQLiteConfig{File: ":memory:"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid config type") + }) +} + +func TestTuneAndValidateConfigType(t *testing.T) { + e := engine{kind: config.Postgres} + require.NoError(t, e.ValidateConfigType(&config.DBConfig{})) + require.Error(t, e.ValidateConfigType(&config.SQLiteConfig{File: ":memory:"})) + + db, err := sqlOpenMemory(t) + require.NoError(t, err) + defer func() { _ = db.Close() }() + e.Tune(db, &config.DBConfig{Pool: config.PoolConfig{MaxOpen: 7, MaxIdle: 3, MaxLifetime: time.Hour}}) + assert.Equal(t, 7, db.Stats().MaxOpenConnections) +} diff --git a/service/sql/engines_stub_test.go b/service/sql/engines_stub_test.go new file mode 100644 index 000000000..8c631d080 --- /dev/null +++ b/service/sql/engines_stub_test.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import ( + "context" + "database/sql" + "fmt" + + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/sql" + entryutil "github.com/wippyai/runtime/internal/entry" +) + +// stubEngine is a faithful-but-minimal engine registered only for the core dispatch +// tests. The real engines live in service/sql/engine/* and cannot be imported here +// (that would be an import cycle), so these stubs exercise the registry, factory, +// manager, and ConnPool plumbing. Real DSN/env/WAL behavior is tested in the engine +// sub-packages. +type stubEngine struct { + kind registry.Kind + driver string + isSQLite bool +} + +func init() { + RegisterEngine(stubEngine{kind: config.Postgres, driver: "postgres"}) + RegisterEngine(stubEngine{kind: config.MySQL, driver: "mysql"}) + RegisterEngine(stubEngine{kind: config.SQLite, driver: "sqlite3", isSQLite: true}) +} + +func (e stubEngine) Kind() registry.Kind { + return e.kind +} + +func (e stubEngine) DriverName() string { + return e.driver +} + +func (e stubEngine) DecodeConfig(ctx context.Context, dtt payload.Transcoder, entry registry.Entry) (config.EngineConfig, error) { + if e.isSQLite { + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + return cfg, nil + } + + cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, dtt, entry) + if err != nil { + return nil, err + } + return cfg, nil +} + +func (stubEngine) ResolveEnv(context.Context, EngineDeps, config.EngineConfig) error { + return nil +} + +func (e stubEngine) BuildDSN(config.EngineConfig) (string, error) { + if e.isSQLite { + return ":memory:", nil + } + return "host=stub", nil +} + +func (stubEngine) Prepare(context.Context, *sql.DB, config.EngineConfig) error { + return nil +} + +func (e stubEngine) Tune(db *sql.DB, ec config.EngineConfig) { + if e.isSQLite { + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + if cfg, ok := ec.(*config.SQLiteConfig); ok { + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) + } + return + } + + if cfg, ok := ec.(*config.DBConfig); ok { + db.SetMaxOpenConns(cfg.Pool.MaxOpen) + db.SetMaxIdleConns(cfg.Pool.MaxIdle) + db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) + } +} + +func (e stubEngine) ValidateConfigType(ec config.EngineConfig) error { + if e.isSQLite { + if _, ok := ec.(*config.SQLiteConfig); !ok { + return NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + return nil + } + + if _, ok := ec.(*config.DBConfig); !ok { + return NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), e.kind) + } + return nil +} diff --git a/service/sql/errors.go b/service/sql/errors.go index c02e71933..efde5f6b5 100644 --- a/service/sql/errors.go +++ b/service/sql/errors.go @@ -9,12 +9,10 @@ import ( ) var ( - ErrPoolClosed = apierror.New(apierror.Unavailable, "connection pool is closed").WithRetryable(apierror.False) - ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) - ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) - ErrPoolFactoryRequired = apierror.New(apierror.Invalid, "pool factory is required").WithRetryable(apierror.False) - ErrNotSQLiteConn = apierror.New(apierror.Invalid, "underlying connection is not a SQLite connection").WithRetryable(apierror.False) - ErrCDCMemoryUnsupported = apierror.New(apierror.Invalid, "sqlite cdc requires a file-backed database").WithRetryable(apierror.False) + ErrPoolClosed = apierror.New(apierror.Unavailable, "connection pool is closed").WithRetryable(apierror.False) + ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) + ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) + ErrPoolFactoryRequired = apierror.New(apierror.Invalid, "pool factory is required").WithRetryable(apierror.False) ) func NewPingError(err error) apierror.Error { @@ -54,12 +52,6 @@ func NewUnsupportedAccessModeError(mode string) apierror.Error { WithDetails(attrs.NewBagFrom(map[string]any{"mode": mode})) } -func NewUnsupportedDatabaseTypeError(kind registry.Kind) apierror.Error { - return apierror.New(apierror.Invalid, "unsupported database type"). - WithRetryable(apierror.False). - WithDetails(attrs.NewBagFrom(map[string]any{"database_type": kind})) -} - func NewConnectionPoolCreationError(err error) apierror.Error { apiErr := apierror.New(apierror.Internal, "failed to create connection pool").WithRetryable(apierror.False) if err != nil { @@ -68,14 +60,6 @@ func NewConnectionPoolCreationError(err error) apierror.Error { return apiErr } -func NewSQLiteConnectionCreationError(err error) apierror.Error { - apiErr := apierror.New(apierror.Internal, "failed to create SQLite connection").WithRetryable(apierror.False) - if err != nil { - apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) - } - return apiErr -} - func NewWALModeError(err error) apierror.Error { apiErr := apierror.New(apierror.Internal, "failed to enable WAL mode").WithRetryable(apierror.False) if err != nil { @@ -128,11 +112,3 @@ func NewPoolUpdateError(err error) apierror.Error { } return apiErr } - -func NewSQLiteUpdateError(err error) apierror.Error { - apiErr := apierror.New(apierror.Internal, "failed to update SQLite config").WithRetryable(apierror.False) - if err != nil { - apiErr = apiErr.WithDetails(attrs.NewBagFrom(map[string]any{"cause": err.Error()})).WithCause(err) - } - return apiErr -} diff --git a/service/sql/factory.go b/service/sql/factory.go index 46d69a563..3789a26c6 100644 --- a/service/sql/factory.go +++ b/service/sql/factory.go @@ -4,104 +4,42 @@ package sql import ( "context" - "database/sql" "github.com/wippyai/runtime/api/registry" config "github.com/wippyai/runtime/api/service/sql" ) -// PoolFactoryAPI defines the interface for creating database connection pools -type PoolFactoryAPI interface { - // CreateStandardPool creates a connection pool for standard SQL databases (Postgres, MySQL) - CreateStandardPool(ctx context.Context, kind registry.Kind, cfg *config.DBConfig) (*ConnPool, error) - - // CreateSQLitePool creates a connection pool for SQLite databases - CreateSQLitePool(ctx context.Context, cfg *config.SQLiteConfig) (*ConnPool, error) +// Factory creates and updates connection pools. It dispatches to the engine +// registered for an entry's kind, so it never needs per-engine branches. +type Factory interface { + CreatePool(ctx context.Context, deps EngineDeps, entry registry.Entry) (*ConnPool, config.EngineConfig, error) + UpdatePool(ctx context.Context, deps EngineDeps, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) } -var sqliteDriverName = "sqlite3" - -// DefaultPoolFactory is the default implementation of PoolFactoryAPI +// DefaultPoolFactory is the registry-backed Factory used in production. type DefaultPoolFactory struct{} -// NewDefaultPoolFactory creates a new default pool factory -func NewDefaultPoolFactory() PoolFactoryAPI { +// NewDefaultPoolFactory creates a new default pool factory. +func NewDefaultPoolFactory() Factory { return &DefaultPoolFactory{} } -// CreateStandardPool implements PoolFactoryAPI.CreateStandardPool -func (f *DefaultPoolFactory) CreateStandardPool(_ context.Context, kind registry.Kind, cfg *config.DBConfig) (*ConnPool, error) { - if err := cfg.Validate(); err != nil { - return nil, NewInvalidConfigError(err) - } - - dsn, err := buildDSN(kind, cfg) - if err != nil { - return nil, NewInvalidDSNError(err) - } - - db, err := sql.Open(getDriver(kind), dsn) - if err != nil { - return nil, NewConnectionPoolCreationError(err) +// CreatePool implements Factory.CreatePool. +func (f *DefaultPoolFactory) CreatePool(ctx context.Context, deps EngineDeps, entry registry.Entry) (*ConnPool, config.EngineConfig, error) { + eng, ok := engineFor(entry.Kind) + if !ok { + return nil, nil, NewUnsupportedEntryKindError(entry.Kind) } - // Configure pool settings - db.SetMaxOpenConns(cfg.Pool.MaxOpen) - db.SetMaxIdleConns(cfg.Pool.MaxIdle) - db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) - - pool := &ConnPool{ - kind: kind, - db: db, - status: make(chan any, 1), - } - - var cfgAny any = cfg - pool.config.Store(&cfgAny) - - return pool, nil + return createPool(ctx, deps, eng, entry) } -// CreateSQLitePool implements PoolFactoryAPI.CreateSQLitePool -func (f *DefaultPoolFactory) CreateSQLitePool(ctx context.Context, cfg *config.SQLiteConfig) (*ConnPool, error) { - if err := cfg.Validate(); err != nil { - return nil, NewInvalidConfigError(err) +// UpdatePool implements Factory.UpdatePool. +func (f *DefaultPoolFactory) UpdatePool(ctx context.Context, deps EngineDeps, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) { + eng, ok := engineFor(entry.Kind) + if !ok { + return nil, NewUnsupportedEntryKindError(entry.Kind) } - var dsn string - - // Handle in-memory database - if cfg.File == ":memory:" { - dsn = ":memory:" - } else { - // Use the file path directly - dsn = "file:" + cfg.File + "?mode=rwc" - } - - db, err := sql.Open(sqliteDriverName, dsn) - if err != nil { - return nil, NewSQLiteConnectionCreationError(err) - } - - // Enable WAL mode for better concurrency - if _, err := db.ExecContext(ctx, "PRAGMA journal_mode=WAL;"); err != nil { - _ = db.Close() - return nil, NewWALModeError(err) - } - - // SQLite specific settings - db.SetMaxOpenConns(1) // SQLite supports only one writer - db.SetMaxIdleConns(1) - db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) - - pool := &ConnPool{ - kind: config.SQLite, - db: db, - status: make(chan any, 1), - } - - var cfgAny any = cfg - pool.config.Store(&cfgAny) - - return pool, nil + return updatePool(ctx, deps, eng, pool, entry) } diff --git a/service/sql/factory_test.go b/service/sql/factory_test.go index 86d170b76..d8cc20775 100644 --- a/service/sql/factory_test.go +++ b/service/sql/factory_test.go @@ -4,14 +4,18 @@ package sql import ( "context" + "fmt" "testing" "time" _ "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/registry" config "github.com/wippyai/runtime/api/service/sql" + "github.com/wippyai/runtime/api/supervisor" + "go.uber.org/zap" ) func createTestDBConfig() *config.DBConfig { @@ -32,112 +36,45 @@ func createTestDBConfig() *config.DBConfig { } } -// TestDefaultPoolFactory_BuildDSN tests DSN string building without connecting to actual databases -func TestDefaultPoolFactory_BuildDSN(t *testing.T) { - tests := []struct { - name string - kind registry.Kind - cfg *config.DBConfig - expected string - isError bool - }{ - { - name: "PostgreSQL DSN", - kind: config.Postgres, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 5432, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "sslmode": "disable", - }, - }, - expected: "host=localhost port=5432 user=user password=pass dbname=testdb sslmode=disable", - isError: false, - }, - { - name: "PostgreSQL DSN with connect timeout", - kind: config.Postgres, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 5432, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "connect_timeout": "2", - "sslmode": "disable", - }, - }, - expected: "host=localhost port=5432 user=user password=pass dbname=testdb connect_timeout=2 sslmode=disable", - isError: false, - }, - { - name: "MySQL DSN", - kind: config.MySQL, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 3306, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "charset": "utf8mb4", - }, - }, - expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4", - isError: false, - }, - { - name: "MySQL DSN with query options", - kind: config.MySQL, - cfg: &config.DBConfig{ - Host: "localhost", - Port: 3306, - Database: "testdb", - Username: "user", - Password: "pass", - Options: map[string]string{ - "charset": "utf8mb4", - "parseTime": "true", - "timeout": "2s", - }, - }, - expected: "user:pass@tcp(localhost:3306)/testdb?charset=utf8mb4&parseTime=true&timeout=2s", - isError: false, - }, - { - name: "Unsupported database type", - kind: "db.unsupported", - cfg: createTestDBConfig(), - expected: "", - isError: true, - }, - } +// fixedTranscoder decodes registry entries into a preset configuration, letting +// factory tests exercise the engine lifecycle without a real registry payload. +type fixedTranscoder struct{ cfg any } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dsn, err := buildDSN(tt.kind, tt.cfg) +func (f fixedTranscoder) Marshal(v any) (payload.Payload, error) { + return payload.New(v), nil +} - if tt.isError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tt.expected, dsn) - } - }) +func (f fixedTranscoder) Unmarshal(_ payload.Payload, v any) error { + switch target := v.(type) { + case *config.DBConfig: + if c, ok := f.cfg.(*config.DBConfig); ok { + *target = *c + return nil + } + case *config.SQLiteConfig: + if c, ok := f.cfg.(*config.SQLiteConfig); ok { + *target = *c + return nil + } } + return fmt.Errorf("unexpected decode target %T", v) +} + +func (f fixedTranscoder) Transcode(p payload.Payload, format payload.Format) (payload.Payload, error) { + return payload.NewPayload(p.Data(), format), nil } -// TestDefaultPoolFactory_CreateStandardPool tests standard pool factory methods validation -func TestDefaultPoolFactory_CreateStandardPool(t *testing.T) { - // We'll test the validation logic without actually connecting +func depsFor(cfg any) EngineDeps { + return EngineDeps{Transcoder: fixedTranscoder{cfg: cfg}, Env: NewMockEnvRegistry(), Log: zap.NewNop()} +} + +// TestDefaultPoolFactory_CreatePoolValidation tests pool creation validation through +// the registry-backed factory. +func TestDefaultPoolFactory_CreatePoolValidation(t *testing.T) { factory := &DefaultPoolFactory{} tests := []struct { - cfg *config.DBConfig + cfg any name string kind registry.Kind errMsg string @@ -146,50 +83,58 @@ func TestDefaultPoolFactory_CreateStandardPool(t *testing.T) { { name: "Invalid configuration - empty host", kind: config.Postgres, - cfg: &config.DBConfig{Host: "", Port: 5432, Database: "db", Username: "user", Password: "pass"}, + cfg: &config.DBConfig{Host: "", Port: 5432, Database: "db", Username: "user", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - zero port", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 0, Database: "db", Username: "user", Password: "pass"}, + cfg: &config.DBConfig{Host: "localhost", Port: 0, Database: "db", Username: "user", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - empty database", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "", Username: "user", Password: "pass"}, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "", Username: "user", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - empty username", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "", Password: "pass"}, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "", Password: "pass", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { name: "Invalid configuration - empty password", kind: config.Postgres, - cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "user", Password: ""}, + cfg: &config.DBConfig{Host: "localhost", Port: 5432, Database: "db", Username: "user", Password: "", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, isError: true, errMsg: "invalid configuration", }, { - name: "Unsupported database type", + name: "Invalid configuration - SQLite empty file", + kind: config.SQLite, + cfg: &config.SQLiteConfig{File: "", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, + isError: true, + errMsg: "invalid configuration", + }, + { + name: "Unsupported entry kind", kind: "db.unsupported", cfg: createTestDBConfig(), isError: true, - errMsg: "invalid connection config", + errMsg: "unsupported entry kind", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - pool, err := factory.CreateStandardPool(context.Background(), tt.kind, tt.cfg) + entry := registry.Entry{ID: registry.NewID("test", "x"), Kind: tt.kind, Data: payload.New("x")} + pool, _, err := factory.CreatePool(context.Background(), depsFor(tt.cfg), entry) if tt.isError { require.Error(t, err) @@ -200,39 +145,22 @@ func TestDefaultPoolFactory_CreateStandardPool(t *testing.T) { } } -// TestDefaultPoolFactory_CreateSQLitePoolValidation tests SQLite pool validation -func TestDefaultPoolFactory_CreateSQLitePoolValidation(t *testing.T) { - factory := &DefaultPoolFactory{} - - tests := []struct { - cfg *config.SQLiteConfig - name string - errMsg string - isError bool - }{ - { - name: "Invalid configuration - empty file", - cfg: &config.SQLiteConfig{File: "", Pool: config.PoolConfig{MaxLifetime: time.Hour}}, - isError: true, - errMsg: "invalid configuration", - }, - { - name: "Invalid configuration - zero max lifetime", - cfg: &config.SQLiteConfig{File: ":memory:", Pool: config.PoolConfig{MaxLifetime: 0}}, - isError: true, - errMsg: "invalid configuration", - }, +// TestDefaultPoolFactory_CreatePoolSQLiteSuccess exercises the full create lifecycle +// (open, WAL prepare, tune, store) for a SQLite pool. +func TestDefaultPoolFactory_CreatePoolSQLiteSuccess(t *testing.T) { + cfg := &config.SQLiteConfig{ + File: ":memory:", + Lifecycle: supervisor.LifecycleConfig{StartTimeout: time.Minute}, + Pool: config.PoolConfig{MaxLifetime: time.Hour}, } + entry := registry.Entry{ID: registry.NewID("test", "lite"), Kind: config.SQLite, Data: payload.New("x")} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pool, err := factory.CreateSQLitePool(context.Background(), tt.cfg) + pool, ec, err := (&DefaultPoolFactory{}).CreatePool(context.Background(), depsFor(cfg), entry) + require.NoError(t, err) + require.NotNil(t, pool) + assert.Equal(t, config.SQLite, pool.kind) + require.NotNil(t, ec) + assert.Equal(t, time.Minute, ec.LifecycleConfig().StartTimeout) - if tt.isError { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errMsg) - assert.Nil(t, pool) - } - }) - } + require.NoError(t, pool.Stop(context.Background())) } diff --git a/service/sql/manager.go b/service/sql/manager.go index 0a424a7ec..0d66e092c 100644 --- a/service/sql/manager.go +++ b/service/sql/manager.go @@ -4,7 +4,6 @@ package sql import ( "context" - "strconv" "sync" envapi "github.com/wippyai/runtime/api/env" @@ -13,9 +12,7 @@ import ( "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/resource" - config "github.com/wippyai/runtime/api/service/sql" "github.com/wippyai/runtime/api/supervisor" - entryutil "github.com/wippyai/runtime/internal/entry" "go.uber.org/zap" ) @@ -23,7 +20,7 @@ import ( type Manager struct { dtt payload.Transcoder bus event.Bus - factory PoolFactoryAPI + factory Factory env envapi.Registry log *zap.Logger services map[registry.ID]*ConnPool @@ -46,7 +43,7 @@ func NewManagerWithFactory( bus event.Bus, log *zap.Logger, envRegistry envapi.Registry, - factory PoolFactoryAPI, + factory Factory, ) (*Manager, error) { if dtt == nil { return nil, ErrTranscoderRequired @@ -71,138 +68,60 @@ func NewManagerWithFactory( }, nil } -// Add implements registry.EntryListener -func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - - switch entry.Kind { - case config.Postgres, config.MySQL: - return m.handleStandardDBAdd(ctx, entry) - case config.SQLite: - return m.handleSQLiteAdd(ctx, entry) - default: - return NewUnsupportedEntryKindError(entry.Kind) - } +// deps bundles the manager's collaborators for the engine lifecycle. +func (m *Manager) deps() EngineDeps { + return EngineDeps{Transcoder: m.dtt, Env: m.env, Log: m.log} } -// Update implements registry.EntryListener -func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { +// Add implements registry.EntryListener +func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { m.mu.Lock() defer m.mu.Unlock() - switch entry.Kind { - case config.Postgres, config.MySQL: - return m.handleStandardDBUpdate(ctx, entry) - case config.SQLite: - return m.handleSQLiteUpdate(ctx, entry) - default: + if _, ok := engineFor(entry.Kind); !ok { return NewUnsupportedEntryKindError(entry.Kind) } -} - -// Delete implements registry.EntryListener -func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - return m.handleDBDelete(ctx, entry) -} - -func (m *Manager) handleStandardDBAdd(ctx context.Context, entry registry.Entry) error { if _, exists := m.services[entry.ID]; exists { return NewServiceExistsError(entry.ID) } - cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } - - if v := m.resolveEnv(ctx, cfg.HostEnv, "host"); v != "" { - cfg.Host = v - } - if v := m.resolveEnv(ctx, cfg.PortEnv, "port"); v != "" { - cfg.Port, err = strconv.Atoi(v) - if err != nil { - return NewInvalidPortError(cfg.PortEnv, err) - } - } - if v := m.resolveEnv(ctx, cfg.DatabaseEnv, "database"); v != "" { - cfg.Database = v - } - if v := m.resolveEnv(ctx, cfg.UsernameEnv, "username"); v != "" { - cfg.Username = v - } - if v := m.resolveEnv(ctx, cfg.PasswordEnv, "password"); v != "" { - cfg.Password = v - } - - pool, err := m.factory.CreateStandardPool(ctx, entry.Kind, cfg) + pool, cfg, err := m.factory.CreatePool(ctx, m.deps(), entry) if err != nil { - return NewConnectionPoolCreationError(err) + return err } - return m.registerService(ctx, entry, pool, cfg.Lifecycle) + return m.registerService(ctx, entry, pool, cfg.LifecycleConfig()) } -func (m *Manager) handleSQLiteAdd(ctx context.Context, entry registry.Entry) error { - if _, exists := m.services[entry.ID]; exists { - return NewServiceExistsError(entry.ID) - } - - cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } +// Update implements registry.EntryListener +func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() - pool, err := m.factory.CreateSQLitePool(ctx, cfg) - if err != nil { - return NewSQLiteConnectionCreationError(err) + if _, ok := engineFor(entry.Kind); !ok { + return NewUnsupportedEntryKindError(entry.Kind) } - return m.registerService(ctx, entry, pool, cfg.Lifecycle) -} - -func (m *Manager) handleStandardDBUpdate(ctx context.Context, entry registry.Entry) error { pool, exists := m.services[entry.ID] if !exists { return NewServiceNotFoundError(entry.ID) } - cfg, err := entryutil.DecodeEntryConfig[config.DBConfig](ctx, m.dtt, entry) + cfg, err := m.factory.UpdatePool(ctx, m.deps(), pool, entry) if err != nil { - return NewInvalidConfigError(err) + return err } - if err := pool.UpdateConfig(cfg); err != nil { - return NewPoolUpdateError(err) - } - - m.updateService(ctx, entry, cfg.Lifecycle) + m.updateService(ctx, entry, cfg.LifecycleConfig()) return nil } -func (m *Manager) handleSQLiteUpdate(ctx context.Context, entry registry.Entry) error { - pool, exists := m.services[entry.ID] - if !exists { - return NewServiceNotFoundError(entry.ID) - } - - cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } - - if err := pool.UpdateConfig(cfg); err != nil { - return NewSQLiteUpdateError(err) - } - - m.updateService(ctx, entry, cfg.Lifecycle) - return nil -} +// Delete implements registry.EntryListener +func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() -func (m *Manager) handleDBDelete(ctx context.Context, entry registry.Entry) error { _, exists := m.services[entry.ID] if !exists { return NewServiceNotFoundError(entry.ID) @@ -283,21 +202,3 @@ func (m *Manager) unregisterService(ctx context.Context, entry registry.Entry) { m.log.Info("removed database service", zap.String("id", entry.ID.String())) } - -// resolveEnv looks up an environment variable and returns its value. -// Returns empty string if envVar is empty, lookup fails, or var not found. -func (m *Manager) resolveEnv(ctx context.Context, envVar, field string) string { - if envVar == "" || m.env == nil { - return "" - } - val, found, err := m.env.Lookup(ctx, envVar) - if err != nil { - m.log.Warn("failed to lookup env var", zap.String("field", field), zap.String("var", envVar), zap.Error(err)) - return "" - } - if !found { - m.log.Warn("env var not found", zap.String("field", field), zap.String("var", envVar)) - return "" - } - return val -} diff --git a/service/sql/manager_test.go b/service/sql/manager_test.go index ccca5ae1f..d3c553535 100644 --- a/service/sql/manager_test.go +++ b/service/sql/manager_test.go @@ -143,14 +143,9 @@ func NewMockConnPool(kind registry.Kind) *ConnPool { // Mock factory implementation type TestPoolFactory struct { - standardPoolCalls []struct { - Cfg *apiconfig.DBConfig - Kind registry.Kind - } - sqlitePoolCalls []struct { - Cfg *apiconfig.SQLiteConfig - } - shouldFailNext bool + standardPoolCalls []registry.Kind + sqlitePoolCalls []registry.Kind + shouldFailNext bool } func NewTestPoolFactory() *TestPoolFactory { @@ -160,32 +155,39 @@ func NewTestPoolFactory() *TestPoolFactory { } } -func (f *TestPoolFactory) CreateStandardPool(_ context.Context, kind registry.Kind, cfg *apiconfig.DBConfig) (*ConnPool, error) { - f.standardPoolCalls = append(f.standardPoolCalls, struct { - Cfg *apiconfig.DBConfig - Kind registry.Kind - }{ - Kind: kind, - Cfg: cfg, - }) +func mockEngineConfig(kind registry.Kind) apiconfig.EngineConfig { + lifecycle := supervisor.LifecycleConfig{StartTimeout: time.Minute} + if kind == apiconfig.SQLite { + return &apiconfig.SQLiteConfig{ + File: ":memory:", + Lifecycle: lifecycle, + Pool: apiconfig.PoolConfig{MaxLifetime: time.Hour}, + } + } + return &apiconfig.DBConfig{ + Lifecycle: lifecycle, + Pool: apiconfig.PoolConfig{MaxLifetime: time.Hour}, + } +} + +func (f *TestPoolFactory) CreatePool(_ context.Context, _ EngineDeps, entry registry.Entry) (*ConnPool, apiconfig.EngineConfig, error) { + if entry.Kind == apiconfig.SQLite { + f.sqlitePoolCalls = append(f.sqlitePoolCalls, entry.Kind) + } else { + f.standardPoolCalls = append(f.standardPoolCalls, entry.Kind) + } if f.shouldFailNext { - return nil, assert.AnError + return nil, nil, assert.AnError } - return NewMockConnPool(kind), nil + return NewMockConnPool(entry.Kind), mockEngineConfig(entry.Kind), nil } -func (f *TestPoolFactory) CreateSQLitePool(_ context.Context, cfg *apiconfig.SQLiteConfig) (*ConnPool, error) { - f.sqlitePoolCalls = append(f.sqlitePoolCalls, struct { - Cfg *apiconfig.SQLiteConfig - }{ - Cfg: cfg, - }) - +func (f *TestPoolFactory) UpdatePool(_ context.Context, _ EngineDeps, _ *ConnPool, entry registry.Entry) (apiconfig.EngineConfig, error) { if f.shouldFailNext { return nil, assert.AnError } - return NewMockConnPool(apiconfig.SQLite), nil + return mockEngineConfig(entry.Kind), nil } // MockEnvRegistry implements envapi.Registry for testing @@ -340,7 +342,12 @@ func TestManager_Add(t *testing.T) { id registry.ID shouldFail bool expectSuccess bool - }{} + }{ + {name: "add postgres", kind: apiconfig.Postgres, id: registry.NewID("test", "add-pg"), shouldFail: false, expectSuccess: true}, + {name: "add sqlite", kind: apiconfig.SQLite, id: registry.NewID("test", "add-lite"), shouldFail: false, expectSuccess: true}, + {name: "add failure", kind: apiconfig.Postgres, id: registry.NewID("test", "add-fail"), shouldFail: true, expectSuccess: false}, + {name: "unsupported kind", kind: "db.unsupported", id: registry.NewID("test", "add-bad"), shouldFail: false, expectSuccess: false}, + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -376,7 +383,7 @@ func TestManager_Add(t *testing.T) { assert.GreaterOrEqual(t, len(factory.standardPoolCalls), 1) if len(factory.standardPoolCalls) > 0 { lastCall := factory.standardPoolCalls[len(factory.standardPoolCalls)-1] - assert.Equal(t, tt.kind, lastCall.Kind) + assert.Equal(t, tt.kind, lastCall) } } @@ -603,105 +610,3 @@ func TestDecode_NilPayload(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "configuration data is required") } - -func TestManager_ResolveEnv(t *testing.T) { - manager, _, _ := newTestManager(t) - ctx := ctxapi.NewRootContext() - - // Create env registry with test values - envRegistry := NewMockEnvRegistry() - require.NoError(t, envRegistry.Set(ctx, "TEST_HOST", "test-host-value")) - require.NoError(t, envRegistry.Set(ctx, "TEST_PORT", "5432")) - manager.env = envRegistry - - tests := []struct { - name string - envVar string - field string - expected string - }{ - { - name: "Empty env var returns empty", - envVar: "", - field: "host", - expected: "", - }, - { - name: "Found env var returns value", - envVar: "TEST_HOST", - field: "host", - expected: "test-host-value", - }, - { - name: "Not found env var returns empty", - envVar: "NONEXISTENT_VAR", - field: "database", - expected: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := manager.resolveEnv(ctx, tt.envVar, tt.field) - assert.Equal(t, tt.expected, result) - }) - } -} - -func TestManager_AddWithEnvVars(t *testing.T) { - manager, _, _ := newTestManager(t) - ctx := ctxapi.NewRootContext() - - // Create env registry with test values - envRegistry := NewMockEnvRegistry() - require.NoError(t, envRegistry.Set(ctx, "DB_HOST", "env-host")) - require.NoError(t, envRegistry.Set(ctx, "DB_PORT", "9999")) - require.NoError(t, envRegistry.Set(ctx, "DB_NAME", "env-db")) - require.NoError(t, envRegistry.Set(ctx, "DB_USER", "env-user")) - require.NoError(t, envRegistry.Set(ctx, "DB_PASS", "env-pass")) - manager.env = envRegistry - - // Create a custom transcoder that uses env var fields - manager.dtt = &EnvConfigTranscoder{} - - entry := registry.Entry{ - ID: registry.NewID("test", "env-db"), - Kind: apiconfig.Postgres, - Data: payload.New(map[string]string{"test": "data"}), - } - - err := manager.Add(ctx, entry) - assert.NoError(t, err) -} - -// EnvConfigTranscoder returns a config with env var fields set -type EnvConfigTranscoder struct{} - -func (t *EnvConfigTranscoder) Marshal(v any) (payload.Payload, error) { - return payload.New(v), nil -} - -func (t *EnvConfigTranscoder) Unmarshal(_ payload.Payload, v any) error { - switch target := v.(type) { - case *apiconfig.DBConfig: - *target = apiconfig.DBConfig{ - HostEnv: "DB_HOST", - PortEnv: "DB_PORT", - DatabaseEnv: "DB_NAME", - UsernameEnv: "DB_USER", - PasswordEnv: "DB_PASS", - Pool: apiconfig.PoolConfig{ - MaxOpen: 10, - MaxIdle: 5, - MaxLifetime: time.Hour, - }, - } - default: - return fmt.Errorf("unsupported type: %T", v) - } - return nil -} - -func (t *EnvConfigTranscoder) Transcode(p payload.Payload, format payload.Format) (payload.Payload, error) { - return payload.NewPayload(p.Data(), format), nil -} From 8c88237c306bc33bf4655d15c1b2b57d25470d4c Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Wed, 15 Jul 2026 13:22:12 -0300 Subject: [PATCH 03/47] fix(cdc): honest live-observation semantics for db.cdc.sqlite Address the #351 review: the source claimed stronger CDC, snapshot, checkpoint and cursor semantics than it provided. Redesign to be honest and safe, keeping the preupdate hook + row-decode work. - Single capture owner per database via a token registry keyed by the canonical file: a second source is refused, pre-existing hooks are not clobbered, and release is token-guarded so a stopping source never clears the new owner's hooks on the shared single writer connection. - Per-consumer snapshot bootstrap with a real consistency fence (brief writer-connection fence before latching a read transaction), streamed on a helper goroutine; late subscribers get a consistent image then live changes. snapshot/error events bypass a consumer's op filter. - Bounded and non-blocking: the commit hook never blocks; queued transactions and per-transaction rows/bytes are bounded. On overflow the source faults and emits one terminal op=error gap event while the application commit still succeeds. Laggard subscribers get a terminal error and are dropped, never backpressuring the writer. - Session-scoped {epoch, sequence} cursor; drop the wippy_cdc_offsets table and all durable-checkpoint code (zero schema intrusion). - Capture the SQLite database name (schema); writes to temp/attached databases are not reported as main. Invalidate column metadata on PRAGMA schema_version change, with a defensive column-count guard. - Fix a Start/run race on the status channel surfaced by the supervisor path under -race. - Surface engine/file/db_resource/epoch/faulted and the snapshot option and error field through the Lua cdc module. --- api/service/cdc/command.go | 8 +- api/service/cdc/context.go | 3 + runtime/lua/modules/cdc/module.go | 21 +- runtime/lua/modules/cdc/yields.go | 3 + service/cdc/sqlite/cdc_bench_test.go | 194 ++++++++ service/cdc/sqlite/checkpoint.go | 60 --- service/cdc/sqlite/hook.go | 117 +++-- service/cdc/sqlite/hook_registry_test.go | 41 ++ service/cdc/sqlite/integration_lua_test.go | 232 ++++++++++ service/cdc/sqlite/integration_test.go | 44 +- service/cdc/sqlite/manager.go | 30 +- .../cdc/sqlite/redesign_integration_test.go | 225 ++++++++++ service/cdc/sqlite/snapshot.go | 97 +++- service/cdc/sqlite/source.go | 419 ++++++++++++------ service/cdc/sqlite/subscribers.go | 112 +++-- service/cdc/sqlite/subscribers_snapshot.go | 44 ++ service/cdc/sqlite/subscribers_test.go | 18 +- 17 files changed, 1365 insertions(+), 303 deletions(-) create mode 100644 service/cdc/sqlite/cdc_bench_test.go delete mode 100644 service/cdc/sqlite/checkpoint.go create mode 100644 service/cdc/sqlite/hook_registry_test.go create mode 100644 service/cdc/sqlite/integration_lua_test.go create mode 100644 service/cdc/sqlite/redesign_integration_test.go create mode 100644 service/cdc/sqlite/subscribers_snapshot.go diff --git a/api/service/cdc/command.go b/api/service/cdc/command.go index 23e964a25..7eb409180 100644 --- a/api/service/cdc/command.go +++ b/api/service/cdc/command.go @@ -18,9 +18,10 @@ const ( ) type StreamOptions struct { - Tables []string - Ops []string - Buffer int + Tables []string + Ops []string + Buffer int + Snapshot bool } type Change struct { @@ -33,6 +34,7 @@ type Change struct { Relation string `json:"relation"` LSN string `json:"lsn"` CommitLSN string `json:"commit_lsn,omitempty"` + Error string `json:"error,omitempty"` XID uint32 `json:"xid,omitempty"` } diff --git a/api/service/cdc/context.go b/api/service/cdc/context.go index 2ea298b19..3960518fb 100644 --- a/api/service/cdc/context.go +++ b/api/service/cdc/context.go @@ -11,11 +11,14 @@ type SourceInfo struct { Engine string `json:"engine,omitempty"` File string `json:"file,omitempty"` DBResource string `json:"db_resource,omitempty"` + Epoch string `json:"epoch,omitempty"` + Error string `json:"error,omitempty"` Tables []string `json:"tables,omitempty"` Streaming bool `json:"streaming,omitempty"` Failover bool `json:"failover,omitempty"` Temporary bool `json:"temporary,omitempty"` Snapshot bool `json:"snapshot,omitempty"` + Faulted bool `json:"faulted,omitempty"` } type SourceInspector interface { diff --git a/runtime/lua/modules/cdc/module.go b/runtime/lua/modules/cdc/module.go index aef3ade82..892e7a240 100644 --- a/runtime/lua/modules/cdc/module.go +++ b/runtime/lua/modules/cdc/module.go @@ -264,9 +264,24 @@ func (s *Stream) closeWithUnsubscribe(unsubscribe bool) { } func sourceInfoToTable(l *lua.LState, info cdcapi.SourceInfo) *lua.LTable { - t := l.CreateTable(0, 8) + t := l.CreateTable(0, 12) t.RawSetString("name", lua.LString(info.Name)) t.RawSetString("slot", lua.LString(info.Slot)) + if info.Engine != "" { + t.RawSetString("engine", lua.LString(info.Engine)) + } + if info.File != "" { + t.RawSetString("file", lua.LString(info.File)) + } + if info.DBResource != "" { + t.RawSetString("db_resource", lua.LString(info.DBResource)) + } + if info.Epoch != "" { + t.RawSetString("epoch", lua.LString(info.Epoch)) + } + if info.Error != "" { + t.RawSetString("error", lua.LString(info.Error)) + } if info.Publication != "" { t.RawSetString("publication", lua.LString(info.Publication)) } @@ -281,6 +296,7 @@ func sourceInfoToTable(l *lua.LState, info cdcapi.SourceInfo) *lua.LTable { t.RawSetString("failover", lua.LBool(info.Failover)) t.RawSetString("temporary", lua.LBool(info.Temporary)) t.RawSetString("snapshot", lua.LBool(info.Snapshot)) + t.RawSetString("faulted", lua.LBool(info.Faulted)) return t } @@ -312,6 +328,9 @@ func streamOptionsFromLua(l *lua.LState, idx int) (cdcapi.StreamOptions, *lua.Er } opts.Buffer = n } + if v := table.RawGetString("snapshot"); v != lua.LNil { + opts.Snapshot = lua.LVAsBool(v) + } return opts, nil } diff --git a/runtime/lua/modules/cdc/yields.go b/runtime/lua/modules/cdc/yields.go index 476d3bd84..c0b3ac2cc 100644 --- a/runtime/lua/modules/cdc/yields.go +++ b/runtime/lua/modules/cdc/yields.go @@ -155,6 +155,9 @@ func changeToLua(l *lua.LState, change cdcapi.Change) (lua.LValue, error) { if change.CommitLSN != "" { tbl.RawSetString("commit_lsn", lua.LString(change.CommitLSN)) } + if change.Error != "" { + tbl.RawSetString("error", lua.LString(change.Error)) + } if change.XID != 0 { tbl.RawSetString("xid", lua.LInteger(change.XID)) } diff --git a/service/cdc/sqlite/cdc_bench_test.go b/service/cdc/sqlite/cdc_bench_test.go new file mode 100644 index 000000000..47ad687cf --- /dev/null +++ b/service/cdc/sqlite/cdc_bench_test.go @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + config "github.com/wippyai/runtime/api/service/cdc" + sqlconfig "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +var benchCtx = context.Background() + +type benchRegistry struct{ db *sql.DB } + +func (r *benchRegistry) Acquire(context.Context, registry.ID, resource.AccessMode) (resource.Resource[any], error) { + return &benchResource{res: sqlservice.DBResource{DB: r.db, Type: sqlconfig.SQLite}}, nil +} +func (r *benchRegistry) List() ([]registry.ID, error) { return nil, nil } +func (r *benchRegistry) Exists(registry.ID) bool { return true } + +type benchResource struct{ res sqlservice.DBResource } + +func (b *benchResource) Get() (any, error) { return b.res, nil } +func (b *benchResource) Release() {} + +func benchPool(b *testing.B, driver string) *sql.DB { + b.Helper() + file := filepath.Join(b.TempDir(), "bench.db") + db, err := sql.Open(driver, "file:"+file+"?mode=rwc") + if err != nil { + b.Fatal(err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + if _, err := db.ExecContext(benchCtx, "PRAGMA journal_mode=WAL"); err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = db.Close() }) + return db +} + +func benchStartSource(b *testing.B, db *sql.DB) *Source { + b.Helper() + h, err := buildSource(sourceOptions{ + res: &benchRegistry{db: db}, + dbResource: registry.NewID("app", "db"), + name: "bench-src", + statusInterval: "1h", + }) + if err != nil { + b.Fatal(err) + } + src := h.(*Source) + if _, err := src.Start(context.Background()); err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = src.Stop(context.Background()) }) + return src +} + +func drainStream(stream config.ChangeStream) func() { + stop := make(chan struct{}) + go func() { + for { + select { + case <-stop: + return + case _, ok := <-stream.Changes(): + if !ok { + return + } + } + } + }() + return func() { close(stop) } +} + +func reportLatencies(b *testing.B, lat []time.Duration) { + if len(lat) == 0 { + return + } + sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] }) + p := func(q float64) time.Duration { + idx := int(q * float64(len(lat)-1)) + return lat[idx] + } + b.ReportMetric(float64(p(0.50).Nanoseconds()), "p50-ns/commit") + b.ReportMetric(float64(p(0.99).Nanoseconds()), "p99-ns/commit") + b.ReportMetric(float64(p(0.999).Nanoseconds()), "p999-ns/commit") +} + +func benchWrite(b *testing.B, db *sql.DB, payload string) { + if _, err := db.ExecContext(benchCtx, "CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY, v TEXT)"); err != nil { + b.Fatal(err) + } + lat := make([]time.Duration, 0, b.N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + if _, err := db.ExecContext(benchCtx, "INSERT INTO t (v) VALUES (?)", payload); err != nil { + b.Fatal(err) + } + lat = append(lat, time.Since(start)) + } + b.StopTimer() + reportLatencies(b, lat) +} + +func BenchmarkWriteHooksDisabled(b *testing.B) { + db := benchPool(b, "sqlite3") + benchWrite(b, db, "payload") +} + +func BenchmarkWriteHooksEnabledNoSubscribers(b *testing.B) { + db := benchPool(b, sqliteCDCDriver) + benchStartSource(b, db) + benchWrite(b, db, "payload") +} + +func BenchmarkWriteOneSubscriber(b *testing.B) { + db := benchPool(b, sqliteCDCDriver) + src := benchStartSource(b, db) + stop := drainStream(src.Subscribe(config.StreamOptions{Buffer: 1024})) + defer stop() + benchWrite(b, db, "payload") +} + +func BenchmarkWriteLargeBlobOneSubscriber(b *testing.B) { + db := benchPool(b, sqliteCDCDriver) + src := benchStartSource(b, db) + stop := drainStream(src.Subscribe(config.StreamOptions{Buffer: 1024})) + defer stop() + blob := make([]byte, 64*1024) + if _, err := db.ExecContext(benchCtx, "CREATE TABLE t (id INTEGER PRIMARY KEY, v BLOB)"); err != nil { + b.Fatal(err) + } + lat := make([]time.Duration, 0, b.N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + if _, err := db.ExecContext(benchCtx, "INSERT INTO t (v) VALUES (?)", blob); err != nil { + b.Fatal(err) + } + lat = append(lat, time.Since(start)) + } + b.StopTimer() + reportLatencies(b, lat) +} + +func BenchmarkWriteSaturatedSubscriber(b *testing.B) { + db := benchPool(b, sqliteCDCDriver) + src := benchStartSource(b, db) + _ = src.Subscribe(config.StreamOptions{Buffer: 1}) + benchWrite(b, db, "payload") +} + +func BenchmarkLargeTransaction(b *testing.B) { + db := benchPool(b, sqliteCDCDriver) + src := benchStartSource(b, db) + stop := drainStream(src.Subscribe(config.StreamOptions{Buffer: 4096})) + defer stop() + if _, err := db.ExecContext(benchCtx, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); err != nil { + b.Fatal(err) + } + const rowsPerTxn = 1000 + b.ResetTimer() + for i := 0; i < b.N; i++ { + tx, err := db.BeginTx(benchCtx, nil) + if err != nil { + b.Fatal(err) + } + for j := 0; j < rowsPerTxn; j++ { + if _, err := tx.ExecContext(benchCtx, "INSERT INTO t (v) VALUES (?)", "payload"); err != nil { + b.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.ReportMetric(float64(rowsPerTxn), "rows/txn") +} diff --git a/service/cdc/sqlite/checkpoint.go b/service/cdc/sqlite/checkpoint.go deleted file mode 100644 index bc1a7c3aa..000000000 --- a/service/cdc/sqlite/checkpoint.go +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build sqlite_preupdate_hook - -package sqlite - -import ( - "context" - "database/sql" -) - -const createOffsetsSQL = `CREATE TABLE IF NOT EXISTS ` + offsetsTable + ` ( - source TEXT PRIMARY KEY, - last_seq INTEGER NOT NULL DEFAULT 0, - snapshot_done INTEGER NOT NULL DEFAULT 0, - updated_at TEXT NOT NULL DEFAULT (datetime('now')) -)` - -func ensureOffsets(ctx context.Context, db *sql.DB) error { - _, err := db.ExecContext(ctx, createOffsetsSQL) - return err -} - -func loadOffset(ctx context.Context, db *sql.DB, source string) (snapshotDone bool, lastSeq uint64, err error) { - row := db.QueryRowContext(ctx, "SELECT snapshot_done, last_seq FROM "+offsetsTable+" WHERE source = ?", source) - var done int - var seq int64 - switch scanErr := row.Scan(&done, &seq); scanErr { - case nil: - return done != 0, uint64(seq), nil - case sql.ErrNoRows: - return false, 0, nil - default: - return false, 0, scanErr - } -} - -func saveSnapshotDone(ctx context.Context, db *sql.DB, source string) error { - _, err := db.ExecContext(ctx, - "INSERT INTO "+offsetsTable+" (source, snapshot_done, updated_at) VALUES (?, 1, datetime('now')) "+ - "ON CONFLICT(source) DO UPDATE SET snapshot_done = 1, updated_at = datetime('now')", - source) - return err -} - -func saveOffset(ctx context.Context, db *sql.DB, source string, seq uint64) error { - if db == nil { - return nil - } - _, err := db.ExecContext(ctx, - "INSERT INTO "+offsetsTable+" (source, last_seq, updated_at) VALUES (?, ?, datetime('now')) "+ - "ON CONFLICT(source) DO UPDATE SET last_seq = MAX(last_seq, excluded.last_seq), updated_at = datetime('now')", - source, int64(seq)) - return err -} - -func deleteOffset(ctx context.Context, db *sql.DB, source string) error { - _, err := db.ExecContext(ctx, "DELETE FROM "+offsetsTable+" WHERE source = ?", source) - return err -} diff --git a/service/cdc/sqlite/hook.go b/service/cdc/sqlite/hook.go index a8a2a5f27..389fe212b 100644 --- a/service/cdc/sqlite/hook.go +++ b/service/cdc/sqlite/hook.go @@ -17,8 +17,6 @@ import ( sqlservice "github.com/wippyai/runtime/service/sql" ) -// sqliteCDCDriver is a SQLite driver variant whose ConnectHook rebinds preupdate -// hooks on every connection the pool opens to a file with a registered sink. const sqliteCDCDriver = "sqlite3_wippy" const ( @@ -30,18 +28,24 @@ const ( var ( errNotSQLiteConn = apierror.New(apierror.Invalid, "underlying connection is not a SQLite connection").WithRetryable(apierror.False) errCDCMemoryUnsupported = apierror.New(apierror.Invalid, "sqlite cdc requires a file-backed database").WithRetryable(apierror.False) + errCaptureOwned = apierror.New(apierror.Conflict, "sqlite cdc capture already owned for this database").WithRetryable(apierror.True) ) -// cdcSink receives row-level changes observed on the writer connection. type cdcSink interface { - PreUpdate(op int, table string, rowid int64, old, new []any) + PreUpdate(op int, schema, table string, rowid int64, ncols int, old, new []any, scanErr error) Commit() Rollback() } +type captureOwner struct { + sink cdcSink + token uint64 +} + var ( - cdcMu sync.RWMutex - cdcSinks = make(map[string]cdcSink) + cdcMu sync.Mutex + captures = make(map[string]captureOwner) + captureNo uint64 ) func init() { @@ -54,59 +58,95 @@ func cdcConnectHook(conn *sqlite3.SQLiteConn) error { if file == "" { return nil } - cdcMu.RLock() - sink, ok := cdcSinks[file] - cdcMu.RUnlock() - if ok { - bindCDCHooks(conn, sink) + + cdcMu.Lock() + defer cdcMu.Unlock() + if owner, ok := captures[file]; ok { + bindCDCHooks(conn, owner.sink) } + return nil } -func registerSink(file string, sink cdcSink) { +func claimCapture(file string, sink cdcSink) (uint64, error) { cdcMu.Lock() - cdcSinks[file] = sink - cdcMu.Unlock() + defer cdcMu.Unlock() + if _, ok := captures[file]; ok { + return 0, errCaptureOwned + } + + captureNo++ + captures[file] = captureOwner{sink: sink, token: captureNo} + + return captureNo, nil } -func unregisterSink(file string) { +func releaseCapture(file string, token uint64) { cdcMu.Lock() - delete(cdcSinks, file) - cdcMu.Unlock() + defer cdcMu.Unlock() + if owner, ok := captures[file]; ok && owner.token == token { + delete(captures, file) + } } -func installHooksOnRaw(raw any, sink cdcSink) (string, error) { +func installHooksOnRaw(raw any, sink cdcSink) (string, uint64, error) { conn, ok := raw.(*sqlite3.SQLiteConn) if !ok { - return "", errNotSQLiteConn + return "", 0, errNotSQLiteConn } + file := normalizeCDCPath(conn.GetFilename("main")) if file == "" { - return "", errCDCMemoryUnsupported + return "", 0, errCDCMemoryUnsupported } + + token, err := claimCapture(file, sink) + if err != nil { + return file, 0, err + } + bindCDCHooks(conn, sink) - return file, nil + + return file, token, nil } -func clearHooksOnRaw(raw any) error { +func applyOwnerOnRaw(raw any, file string) error { conn, ok := raw.(*sqlite3.SQLiteConn) if !ok { return errNotSQLiteConn } + + cdcMu.Lock() + defer cdcMu.Unlock() + if owner, ok := captures[file]; ok { + bindCDCHooks(conn, owner.sink) + } else { + clearHooks(conn) + } + + return nil +} + +func clearHooks(conn *sqlite3.SQLiteConn) { conn.RegisterPreUpdateHook(nil) conn.RegisterCommitHook(nil) conn.RegisterRollbackHook(nil) - return nil } func normalizeCDCPath(path string) string { if path == "" { return "" } + + if resolved, err := filepath.EvalSymlinks(path); err == nil { + path = resolved + } + abs, err := filepath.Abs(path) if err != nil { return filepath.Clean(path) } + return abs } @@ -115,22 +155,27 @@ func bindCDCHooks(conn *sqlite3.SQLiteConn, sink cdcSink) { count := d.Count() var oldRow, newRow []any var rowid int64 + var scanErr error switch d.Op { case sqlite3.SQLITE_INSERT: - newRow = scanPreUpdateRow(&d, count, true) + newRow, scanErr = scanPreUpdateRow(&d, count, true) rowid = d.NewRowID case sqlite3.SQLITE_DELETE: - oldRow = scanPreUpdateRow(&d, count, false) + oldRow, scanErr = scanPreUpdateRow(&d, count, false) rowid = d.OldRowID case sqlite3.SQLITE_UPDATE: - oldRow = scanPreUpdateRow(&d, count, false) - newRow = scanPreUpdateRow(&d, count, true) + oldRow, scanErr = scanPreUpdateRow(&d, count, false) + if scanErr == nil { + newRow, scanErr = scanPreUpdateRow(&d, count, true) + } rowid = d.NewRowID } - sink.PreUpdate(d.Op, d.TableName, rowid, oldRow, newRow) + + sink.PreUpdate(d.Op, d.DatabaseName, d.TableName, rowid, count, oldRow, newRow, scanErr) }) conn.RegisterCommitHook(func() int { sink.Commit() + return 0 }) conn.RegisterRollbackHook(func() { @@ -138,15 +183,21 @@ func bindCDCHooks(conn *sqlite3.SQLiteConn, sink cdcSink) { }) } -func scanPreUpdateRow(d *sqlite3.SQLitePreUpdateData, count int, isNew bool) []any { +func scanPreUpdateRow(d *sqlite3.SQLitePreUpdateData, count int, isNew bool) ([]any, error) { if count <= 0 { - return nil + return nil, nil } + vals := make([]any, count) + var err error if isNew { - _ = d.New(vals...) + err = d.New(vals...) } else { - _ = d.Old(vals...) + err = d.Old(vals...) + } + if err != nil { + return nil, err } - return vals + + return vals, nil } diff --git a/service/cdc/sqlite/hook_registry_test.go b/service/cdc/sqlite/hook_registry_test.go new file mode 100644 index 000000000..320e46624 --- /dev/null +++ b/service/cdc/sqlite/hook_registry_test.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type fakeSink struct{} + +func (fakeSink) PreUpdate(int, string, string, int64, int, []any, []any, error) {} +func (fakeSink) Commit() {} +func (fakeSink) Rollback() {} + +func TestCaptureRegistrySingleOwnerAndTokenGuard(t *testing.T) { + const file = "/tmp/wippy-cdc-registry-test.db" + + a := fakeSink{} + b := fakeSink{} + + tok, err := claimCapture(file, a) + require.NoError(t, err) + t.Cleanup(func() { releaseCapture(file, tok) }) + + _, err = claimCapture(file, b) + require.ErrorIs(t, err, errCaptureOwned, "a second owner must be refused") + + releaseCapture(file, tok+1000) + _, err = claimCapture(file, b) + require.ErrorIs(t, err, errCaptureOwned, "release with a stale token must not evict the owner") + + releaseCapture(file, tok) + tok2, err := claimCapture(file, b) + require.NoError(t, err, "after the real owner releases, a new owner can claim") + require.NotEqual(t, tok, tok2) + releaseCapture(file, tok2) +} diff --git a/service/cdc/sqlite/integration_lua_test.go b/service/cdc/sqlite/integration_lua_test.go new file mode 100644 index 000000000..c471125d8 --- /dev/null +++ b/service/cdc/sqlite/integration_lua_test.go @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build integration && sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + lua "github.com/wippyai/go-lua" + "go.uber.org/zap" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/event" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" + "github.com/wippyai/runtime/api/security" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + apisup "github.com/wippyai/runtime/api/supervisor" + "github.com/wippyai/runtime/runtime/lua/engine" + luapayload "github.com/wippyai/runtime/runtime/lua/engine/payload" + cdcmod "github.com/wippyai/runtime/runtime/lua/modules/cdc" + pgcdc "github.com/wippyai/runtime/service/cdc/postgres" + "github.com/wippyai/runtime/system/eventbus" + systempayload "github.com/wippyai/runtime/system/payload" + sysrelay "github.com/wippyai/runtime/system/relay" + "github.com/wippyai/runtime/system/scheduler" + "github.com/wippyai/runtime/system/scheduler/pool/inline" + syssup "github.com/wippyai/runtime/system/supervisor" +) + +type signalingStreamer struct { + inner cdcapi.SourceStreamer + ready chan struct{} + once sync.Once +} + +func (s *signalingStreamer) Stream(ctx context.Context, source string, opts cdcapi.StreamOptions) (cdcapi.ChangeStream, cdcapi.SourceInfo, error) { + stream, info, err := s.inner.Stream(ctx, source, opts) + if err == nil { + s.once.Do(func() { close(s.ready) }) + } + return stream, info, err +} + +func TestLuaSeesRealRunningSQLiteSourceAndItsChanges(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) + require.NoError(t, err) + + bus := eventbus.NewBus() + sup := syssup.NewSupervisor(bus, zap.NewNop()) + supCtx, supCancel := context.WithCancel(context.Background()) + defer supCancel() + require.NoError(t, sup.Start(supCtx)) + + transcoder := systempayload.NewTranscoder() + luapayload.Register(transcoder) + + manager, err := NewManager(transcoder, bus, zap.NewNop(), &fakeRegistry{db: db}) + require.NoError(t, err) + + entryID := registry.NewID("test", "cdc-lua-e2e") + src, err := buildSource(sourceOptions{ + res: &fakeRegistry{db: db}, + dbResource: registry.NewID("app", "db"), + name: entryID.String(), + statusInterval: "1h", + }) + require.NoError(t, err) + srcImpl := src.(*Source) + manager.sources[entryID] = src + manager.storeInfo(registry.Entry{ID: entryID, Kind: cdcapi.SQLite}, &cdcapi.SQLiteConfig{DBResource: "app:db"}) + + lc := apisup.LifecycleConfig{AutoStart: true} + lc.InitDefaults() + bus.Send(supCtx, event.Event{System: registry.System, Kind: registry.TxBegin, Path: "tx"}) + bus.Send(supCtx, event.Event{ + System: apisup.System, + Kind: apisup.ServiceRegister, + Path: entryID.String(), + Data: &apisup.Entry{Service: src, Config: lc}, + }) + bus.Send(supCtx, event.Event{System: registry.System, Kind: registry.TxCommit, Path: "tx"}) + + require.Eventually(t, func() bool { + return srcImpl.Epoch() != "" + }, 15*time.Second, 50*time.Millisecond, "supervisor must auto-start the registered source") + defer func() { + stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = srcImpl.Stop(stopCtx) + stopCancel() + }() + + streamer := &signalingStreamer{inner: manager, ready: make(chan struct{})} + root := security.SetStrictMode(ctxapi.NewRootContext(), false) + payload.WithTranscoder(root, transcoder) + root = cdcapi.WithSourceInspector(root, manager) + root = cdcapi.WithSourceStreamer(root, streamer) + + node := sysrelay.NewNode("cdc-lua-sqlite-node") + root = relay.WithNode(root, node) + + runCtx, runCancel := context.WithTimeout(root, 30*time.Second) + defer runCancel() + + dispReg := scheduler.NewRegistry() + cdcDisp := pgcdc.NewDispatcher(pgcdc.WithWorkers(1)) + require.NoError(t, cdcDisp.Start(runCtx)) + defer func() { require.NoError(t, cdcDisp.Stop(context.Background())) }() + cdcDisp.RegisterAll(func(id dispatcher.CommandID, h dispatcher.Handler) { + dispReg.Register(id, h) + }) + + const expectedEmail = "lua-sqlite-e2e@wippy.ai" + const hostID = "test.cdc.sqlite.lua" + factory := func() (process.Process, error) { + cfg := engine.FactoryConfig{ + ScriptName: "cdc_sqlite_lua_e2e", + Script: ` +local cdc = require("cdc") + +local function main() + local rows, err = cdc.list_sources() + if err ~= nil then return nil, "list_sources error: " .. tostring(err) end + if #rows ~= 1 then return nil, "expected 1 source, got " .. tostring(#rows) end + + local r = rows[1] + if r.engine ~= "sqlite" then return nil, "wrong engine: " .. tostring(r.engine) end + + local stream, stream_err = cdc.stream("test:cdc-lua-e2e", { + tables = {"users"}, + ops = {"insert"}, + buffer = 8, + }) + if stream_err ~= nil then return nil, "stream error: " .. tostring(stream_err) end + + local ch = stream:channel() + local change, ok = ch:receive() + stream:release() + if ok ~= true then return nil, "stream closed before change" end + if change.op ~= "insert" then return nil, "wrong op: " .. tostring(change.op) end + if change.source ~= "test:cdc-lua-e2e" then return nil, "wrong source: " .. tostring(change.source) end + if change.schema ~= "main" then return nil, "wrong schema: " .. tostring(change.schema) end + if change.table ~= "users" then return nil, "wrong table: " .. tostring(change.table) end + if change.after == nil then return nil, "missing after table" end + if change.after.email ~= "` + expectedEmail + `" then + return nil, "wrong email: " .. tostring(change.after.email) + end + + return change.after.email +end + +return { main = main } +`, + ModuleBinders: append(engine.CoreBinders(), func(l *lua.LState) error { + engine.LoadModuleDef(l, cdcmod.Module) + return nil + }), + } + return engine.NewFactory(cfg)() + } + + pool, err := inline.New(factory, dispReg) + require.NoError(t, err) + defer pool.Stop() + require.NoError(t, node.RegisterHost(hostID, pool)) + + frameCtx, frame := ctxapi.OpenFrameContext(runCtx) + defer ctxapi.ReleaseFrameContext(frame) + testPID := pid.PID{Host: hostID, UniqID: "cdc-lua-e2e"} + testPID = testPID.Precomputed() + require.NoError(t, apiruntime.SetFramePID(frameCtx, testPID)) + + resultCh := make(chan *apiruntime.Result, 1) + errCh := make(chan error, 1) + go func() { + result, err := pool.Call(frameCtx, "main", nil) + if err != nil { + errCh <- err + return + } + resultCh <- result + }() + + select { + case <-streamer.ready: + case err := <-errCh: + require.NoError(t, err) + case result := <-resultCh: + t.Fatalf("Lua returned before subscribing: value=%v err=%v", func() any { + if result != nil && result.Value != nil { + return result.Value.Data() + } + return nil + }(), func() any { + if result != nil { + return result.Error + } + return nil + }()) + case <-runCtx.Done(): + t.Fatal("timed out waiting for Lua CDC stream subscription") + } + + _, err = db.Exec(`INSERT INTO users (email) VALUES (?)`, expectedEmail) + require.NoError(t, err) + + var result *apiruntime.Result + select { + case result = <-resultCh: + case err := <-errCh: + require.NoError(t, err) + case <-runCtx.Done(): + t.Fatal("timed out waiting for Lua to receive CDC change") + } + require.NotNil(t, result) + require.NoError(t, result.Error) + require.NotNil(t, result.Value) + got, ok := result.Value.Data().(lua.LString) + require.True(t, ok, "expected Lua string result, got %T", result.Value.Data()) + require.Equal(t, expectedEmail, string(got)) +} diff --git a/service/cdc/sqlite/integration_test.go b/service/cdc/sqlite/integration_test.go index 0c1a7718a..a0d3eab2b 100644 --- a/service/cdc/sqlite/integration_test.go +++ b/service/cdc/sqlite/integration_test.go @@ -169,14 +169,14 @@ func TestIntegrationSnapshotBootstrap(t *testing.T) { _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) require.NoError(t, err) - src := newSource(t, db, sourceOptions{snapshot: true}) - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - + src := newSource(t, db, sourceOptions{}) _, err = src.Start(context.Background()) require.NoError(t, err) defer func() { _ = src.Stop(context.Background()) }() + stream := src.Subscribe(config.StreamOptions{Snapshot: true}) + defer stream.Close() + snap := waitChange(t, stream.Changes()) assert.Equal(t, "snapshot", snap.Op) assert.Equal(t, "existing@b.com", snap.After["email"]) @@ -188,35 +188,37 @@ func TestIntegrationSnapshotBootstrap(t *testing.T) { assert.Equal(t, "new@b.com", live.After["email"]) } -func TestIntegrationRestartKeepsCheckpoint(t *testing.T) { +func TestIntegrationRestartResnapshots(t *testing.T) { db, _ := openPool(t) _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) require.NoError(t, err) _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) require.NoError(t, err) - first := newSource(t, db, sourceOptions{snapshot: true, name: "src"}) - s1 := first.Subscribe(config.StreamOptions{}) + first := newSource(t, db, sourceOptions{name: "src"}) _, err = first.Start(context.Background()) require.NoError(t, err) + s1 := first.Subscribe(config.StreamOptions{Snapshot: true}) snap := waitChange(t, s1.Changes()) require.Equal(t, "snapshot", snap.Op) s1.Close() require.NoError(t, first.Stop(context.Background())) - second := newSource(t, db, sourceOptions{snapshot: true, name: "src"}) - s2 := second.Subscribe(config.StreamOptions{}) - defer s2.Close() + epoch1 := first.Epoch() + + second := newSource(t, db, sourceOptions{name: "src"}) _, err = second.Start(context.Background()) require.NoError(t, err) defer func() { _ = second.Stop(context.Background()) }() - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (2, 'new@b.com')`) - require.NoError(t, err) + assert.NotEqual(t, epoch1, second.Epoch(), "each Start must mint a fresh session epoch") + + s2 := second.Subscribe(config.StreamOptions{Snapshot: true}) + defer s2.Close() got := waitChange(t, s2.Changes()) - assert.Equal(t, "insert", got.Op, "restart must not re-snapshot; first event should be the live insert") - assert.Equal(t, "new@b.com", got.After["email"]) + assert.Equal(t, "snapshot", got.Op, "honest v1: no durable checkpoint, so a fresh snapshot subscriber re-sees existing state") + assert.Equal(t, "existing@b.com", got.After["email"]) } func TestIntegrationLaggardDoesNotStallWrites(t *testing.T) { @@ -255,9 +257,17 @@ func TestIntegrationLaggardDoesNotStallWrites(t *testing.T) { _, err = db.Exec(`INSERT INTO t (v) VALUES ('final')`) require.NoError(t, err) - got := waitChange(t, reader.Changes()) - assert.Equal(t, "insert", got.Op) - assert.Equal(t, "final", got.After["v"]) + deadline := time.After(10 * time.Second) + for { + select { + case got := <-reader.Changes(): + if got.Op == "insert" && got.After["v"] == "final" { + return + } + case <-deadline: + t.Fatal("did not observe the 'final' insert on a fresh subscriber") + } + } } func TestIntegrationTableAllowlist(t *testing.T) { diff --git a/service/cdc/sqlite/manager.go b/service/cdc/sqlite/manager.go index 58d9f8828..9df617838 100644 --- a/service/cdc/sqlite/manager.go +++ b/service/cdc/sqlite/manager.go @@ -20,7 +20,8 @@ type sourceHandle interface { supervisor.Service Subscribe(opts config.StreamOptions) config.ChangeStream closeSubscriptions() - markDrop() + Epoch() string + Faulted() (bool, string) } type sourceOptions struct { @@ -142,7 +143,6 @@ func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { if !exists { return NewServiceNotFoundError(entry.ID) } - src.markDrop() src.closeSubscriptions() m.removeInfo(entry.ID) m.unregister(ctx, entry) @@ -188,8 +188,8 @@ func (m *Manager) List() []config.SourceInfo { defer m.mu.Unlock() out := make([]config.SourceInfo, 0, len(m.infos)) - for _, info := range m.infos { - out = append(out, info) + for id, info := range m.infos { + out = append(out, m.enrich(id, info)) } return out } @@ -200,12 +200,26 @@ func (m *Manager) Get(name string) (config.SourceInfo, bool) { if id, ok := m.infosByName[name]; ok { if info, present := m.infos[id]; present { - return info, true + return m.enrich(id, info), true } } return config.SourceInfo{}, false } +func (m *Manager) enrich(id registry.ID, info config.SourceInfo) config.SourceInfo { + src := m.sources[id] + if src == nil { + return info + } + + info.Epoch = src.Epoch() + if faulted, reason := src.Faulted(); faulted { + info.Faulted = true + info.Error = reason + } + return info +} + func (m *Manager) Stream(_ context.Context, name string, opts config.StreamOptions) (config.ChangeStream, config.SourceInfo, error) { m.mu.Lock() src, info, ok := m.lookupSourceLocked(name) @@ -213,6 +227,12 @@ func (m *Manager) Stream(_ context.Context, name string, opts config.StreamOptio if !ok { return nil, config.SourceInfo{}, NewServiceNotFoundError(registry.ParseID(name)) } + + info.Epoch = src.Epoch() + if faulted, reason := src.Faulted(); faulted { + info.Faulted = true + info.Error = reason + } return src.Subscribe(opts), info, nil } diff --git a/service/cdc/sqlite/redesign_integration_test.go b/service/cdc/sqlite/redesign_integration_test.go new file mode 100644 index 000000000..17a67f1db --- /dev/null +++ b/service/cdc/sqlite/redesign_integration_test.go @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build integration && sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +func TestIntegrationLateSubscriberGetsSnapshotThenLive(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{Snapshot: true}) + defer stream.Close() + + snap := waitChange(t, stream.Changes()) + assert.Equal(t, "snapshot", snap.Op) + assert.Equal(t, "main", snap.Schema) + assert.Equal(t, "existing@b.com", snap.After["email"]) + + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (2, 'live@b.com')`) + require.NoError(t, err) + + live := waitChange(t, stream.Changes()) + assert.Equal(t, "insert", live.Op) + assert.Equal(t, "live@b.com", live.After["email"]) +} + +func TestIntegrationOpFilteredSubscriberStillGetsSnapshot(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{Snapshot: true, Ops: []string{"insert"}}) + defer stream.Close() + + snap := waitChange(t, stream.Changes()) + assert.Equal(t, "snapshot", snap.Op, "op filter must not drop snapshot rows") + assert.Equal(t, "existing@b.com", snap.After["email"]) +} + +func TestIntegrationSecondSourceRefused(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src1 := newSource(t, db, sourceOptions{name: "src-1"}) + _, err = src1.Start(context.Background()) + require.NoError(t, err) + + src2 := newSource(t, db, sourceOptions{name: "src-2"}) + ctx2, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + _, err = src2.Start(ctx2) + require.Error(t, err, "a second capture owner on the same database must be refused") + + require.NoError(t, src1.Stop(context.Background())) + + src3 := newSource(t, db, sourceOptions{name: "src-3"}) + _, err = src3.Start(context.Background()) + require.NoError(t, err, "after the first owner stops, a new source may claim capture") + require.NoError(t, src3.Stop(context.Background())) +} + +func TestIntegrationOverflowFaultsWithoutStallingWriter(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + src.maxRows = 5 + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + tx, err := db.Begin() + require.NoError(t, err) + for i := 0; i < 20; i++ { + _, err = tx.Exec(`INSERT INTO t (v) VALUES ('x')`) + require.NoError(t, err) + } + require.NoError(t, tx.Commit(), "the application commit must succeed even when CDC overflows") + + c := waitChange(t, stream.Changes()) + assert.Equal(t, "error", c.Op) + assert.NotEmpty(t, c.Error) + + faulted, reason := src.Faulted() + assert.True(t, faulted) + assert.NotEmpty(t, reason) + + var n int + require.NoError(t, db.QueryRow(`SELECT count(*) FROM t`).Scan(&n)) + assert.Equal(t, 20, n, "all application rows must be durably written despite the CDC fault") +} + +func TestIntegrationSubscribeAfterFaultGetsTerminalError(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + src.maxRows = 2 + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + tx, err := db.Begin() + require.NoError(t, err) + for i := 0; i < 5; i++ { + _, err = tx.Exec(`INSERT INTO t (v) VALUES ('x')`) + require.NoError(t, err) + } + require.NoError(t, tx.Commit()) + + require.Eventually(t, func() bool { + faulted, _ := src.Faulted() + return faulted + }, 5*time.Second, 10*time.Millisecond) + + late := src.Subscribe(config.StreamOptions{}) + defer late.Close() + c := waitChange(t, late.Changes()) + assert.Equal(t, "error", c.Op, "a subscriber joining a faulted source must receive a terminal error") +} + +func TestIntegrationAlterTableColumnsNotStale(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + _, err = db.Exec(`INSERT INTO t (id, a) VALUES (1, 'first')`) + require.NoError(t, err) + c1 := waitChange(t, stream.Changes()) + assert.Equal(t, "first", c1.After["a"]) + + _, err = db.Exec(`ALTER TABLE t ADD COLUMN b TEXT`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO t (id, a, b) VALUES (2, 'second', 'added')`) + require.NoError(t, err) + + c2 := waitChange(t, stream.Changes()) + assert.Equal(t, "second", c2.After["a"]) + assert.Equal(t, "added", c2.After["b"], "column cache must be invalidated after ALTER TABLE") +} + +func TestIntegrationTempTableNotCapturedAsMain(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE main_t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + stream := src.Subscribe(config.StreamOptions{}) + defer stream.Close() + + _, err = db.Exec(`CREATE TEMP TABLE tmp_t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO tmp_t (id, v) VALUES (1, 'temp-only')`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO main_t (id, v) VALUES (1, 'main-row')`) + require.NoError(t, err) + + c := waitChange(t, stream.Changes()) + assert.Equal(t, "main", c.Schema) + assert.Equal(t, "main_t", c.Table) + assert.Equal(t, "main-row", c.After["v"], "writes to the temp database must not be reported as main") +} + +func TestIntegrationStopCleansUpWithExpiredContext(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{name: "src-a"}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + + expired, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, src.Stop(expired), "Stop must complete cleanup even when the caller context is already cancelled") + + src2 := newSource(t, db, sourceOptions{name: "src-b"}) + _, err = src2.Start(context.Background()) + require.NoError(t, err, "hooks must be released after Stop so a new source can claim capture") + require.NoError(t, src2.Stop(context.Background())) +} diff --git a/service/cdc/sqlite/snapshot.go b/service/cdc/sqlite/snapshot.go index 132b49705..e2c4faf0c 100644 --- a/service/cdc/sqlite/snapshot.go +++ b/service/cdc/sqlite/snapshot.go @@ -7,27 +7,85 @@ package sqlite import ( "context" "database/sql" - "strconv" "strings" config "github.com/wippyai/runtime/api/service/cdc" ) -func (s *Source) runSnapshot(ctx context.Context, conn *sql.Conn) error { - tables, err := s.snapshotTables(ctx, conn) +func (s *Source) bootstrapSubscription(ctx context.Context, sub *subscription) { + defer sub.finishSnapshot() + + snapDB, err := openSnapshotConn(s.file) if err != nil { - return err + sub.fail("snapshot open: " + err.Error()) + return + } + defer func() { _ = snapDB.Close() }() + + tx, err := s.fenceAndBegin(ctx, snapDB) + if err != nil { + if ctx.Err() != nil { + return + } + sub.fail("snapshot begin: " + err.Error()) + return + } + defer func() { _ = tx.Rollback() }() + + tables, err := s.snapshotTables(ctx, tx, sub) + if err != nil { + if ctx.Err() != nil { + return + } + sub.fail("snapshot tables: " + err.Error()) + return } + for _, table := range tables { - if err := s.snapshotTable(ctx, conn, table); err != nil { - return err + if err := s.streamSnapshotTable(ctx, tx, sub, table); err != nil { + if ctx.Err() != nil || sub.isClosed() { + return + } + sub.fail("snapshot table " + table + ": " + err.Error()) + return } } - return nil } -func (s *Source) snapshotTables(ctx context.Context, conn *sql.Conn) ([]string, error) { - rows, err := conn.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") +func (s *Source) fenceAndBegin(ctx context.Context, snapDB *sql.DB) (*sql.Tx, error) { + s.mu.Lock() + writerDB := s.writerDB + s.mu.Unlock() + if writerDB == nil { + return nil, ErrSourceClosed + } + + wc, err := writerDB.Conn(ctx) + if err != nil { + return nil, err + } + defer func() { _ = wc.Close() }() + + if err := wc.PingContext(ctx); err != nil { + return nil, err + } + + tx, err := snapDB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, err + } + + var count int64 + if err := tx.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master").Scan(&count); err != nil { + _ = tx.Rollback() + return nil, err + } + + return tx, nil +} + +func (s *Source) snapshotTables(ctx context.Context, tx *sql.Tx, sub *subscription) ([]string, error) { + rows, err := tx.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") if err != nil { return nil, err } @@ -39,17 +97,18 @@ func (s *Source) snapshotTables(ctx context.Context, conn *sql.Conn) ([]string, if err := rows.Scan(&name); err != nil { return nil, err } - if s.tableAllowed(name) { + if s.tableAllowed(name) && sub.tableAllowed(name) { tables = append(tables, name) } } + return tables, rows.Err() } -func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, table string) error { +func (s *Source) streamSnapshotTable(ctx context.Context, tx *sql.Tx, sub *subscription, table string) error { cols := s.columnsFor(ctx, table) - rows, err := conn.QueryContext(ctx, "SELECT * FROM "+quoteIdent(table)) //nolint:gosec // quoted identifier from sqlite_master; SQLite cannot bind table names as parameters + rows, err := tx.QueryContext(ctx, "SELECT * FROM "+quoteIdent(table)) //nolint:gosec // quoted identifier sourced from sqlite_master; SQLite cannot bind table names if err != nil { return err } @@ -72,16 +131,20 @@ func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, table string if err := rows.Scan(ptrs...); err != nil { return err } - seq := s.seq.Add(1) - s.subs.publish(ctx, config.Change{ + + change := config.Change{ Source: s.name, Op: "snapshot", + Schema: "main", Table: table, Relation: table, After: mapRow(cols, vals), - LSN: strconv.FormatUint(seq, 10), - }) + } + if !sub.sendSnapshot(ctx, change) { + return nil + } } + return rows.Err() } @@ -102,6 +165,7 @@ func resolveColumns(ctx context.Context, db *sql.DB, table string) ([]columnInfo } cols = append(cols, columnInfo{name: name, text: textAffinity(declType)}) } + return cols, rows.Err() } @@ -110,6 +174,7 @@ func columnsFromNames(names []string) []columnInfo { for i, n := range names { cols[i] = columnInfo{name: n} } + return cols } diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index aaa1aa27e..4fb1d3b2f 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -7,6 +7,7 @@ package sqlite import ( "context" "database/sql" + "errors" "fmt" "os" "strconv" @@ -26,47 +27,62 @@ import ( ) const ( - offsetsTable = "wippy_cdc_offsets" changesCounter = "wippy_cdc_changes_total" walGauge = "wippy_cdc_wal_size_bytes" defaultStatusInterval = 30 * time.Second commitQueueSize = 256 + maxTxnRows = 200_000 + maxTxnBytes = 128 << 20 auxBusyTimeoutMillisec = 5000 + claimAttempts = 40 + claimRetryDelay = 50 * time.Millisecond + cleanupTimeout = 5 * time.Second ) type capturedChange struct { - table string - old []any - new []any - op int - rowid int64 + schema string + table string + old []any + new []any + op int + rowid int64 + ncols int } type Source struct { - poolRes resource.Resource[any] res resource.Registry + poolRes resource.Resource[any] readDB *sql.DB - checkpointDB *sql.DB + writerDB *sql.DB + runCtx context.Context + cancel context.CancelFunc runDone chan struct{} commits chan []capturedChange + faultCh chan struct{} subs *subscribers - writerDB *sql.DB cols map[string][]columnInfo - cancel context.CancelFunc tables map[string]struct{} log *zap.Logger - dbResID registry.ID - file string + faultMsg atomic.Pointer[string] name string + file string + epoch string + dbResID registry.ID pending []capturedChange statusInterval time.Duration + token uint64 + pendingBytes int + maxRows int + maxBytes int seq atomic.Uint64 + schemaVer atomic.Int64 colMu sync.RWMutex mu sync.Mutex pendMu sync.Mutex + faultOnce sync.Once stopped atomic.Bool - dropCP atomic.Bool - snap bool + faulted atomic.Bool + defaultSnap bool } func buildSource(opts sourceOptions) (sourceHandle, error) { @@ -74,6 +90,7 @@ func buildSource(opts sourceOptions) (sourceHandle, error) { if log == nil { log = zap.NewNop() } + interval := defaultStatusInterval if opts.statusInterval != "" { d, err := time.ParseDuration(opts.statusInterval) @@ -84,6 +101,7 @@ func buildSource(opts sourceOptions) (sourceHandle, error) { interval = d } } + return &Source{ log: log, res: opts.res, @@ -93,60 +111,138 @@ func buildSource(opts sourceOptions) (sourceHandle, error) { dbResID: opts.dbResource, tables: filterSet(opts.tables), cols: make(map[string][]columnInfo), - snap: opts.snapshot, + faultCh: make(chan struct{}), + maxRows: maxTxnRows, + maxBytes: maxTxnBytes, + defaultSnap: opts.snapshot, }, nil } func (s *Source) Subscribe(opts config.StreamOptions) config.ChangeStream { - return s.subs.subscribe(opts) + wantSnapshot := opts.Snapshot || s.defaultSnap + sub := s.subs.subscribe(s.name, opts, wantSnapshot) + if faulted, reason := s.Faulted(); faulted { + sub.fail(reason) + return sub + } + if !wantSnapshot { + return sub + } + + s.mu.Lock() + ctx := s.runCtx + ready := s.readDB != nil && !s.stopped.Load() + s.mu.Unlock() + + if ready && ctx != nil { + go s.bootstrapSubscription(ctx, sub) + } else { + sub.finishSnapshot() + } + + return sub } func (s *Source) closeSubscriptions() { s.subs.closeAll() } -func (s *Source) markDrop() { - s.dropCP.Store(true) +func (s *Source) Epoch() string { + s.mu.Lock() + defer s.mu.Unlock() + + return s.epoch +} + +func (s *Source) Faulted() (bool, string) { + if !s.faulted.Load() { + return false, "" + } + + msg := "" + if p := s.faultMsg.Load(); p != nil { + msg = *p + } + + return true, msg } -func (s *Source) PreUpdate(op int, table string, rowid int64, old, new []any) { - if !s.tableAllowed(table) { +func (s *Source) fault(reason string) { + s.faultOnce.Do(func() { + s.faulted.Store(true) + r := reason + s.faultMsg.Store(&r) + s.resetPending() + close(s.faultCh) + }) +} + +func (s *Source) resetPending() { + s.pendMu.Lock() + s.pending = nil + s.pendingBytes = 0 + s.pendMu.Unlock() +} + +func (s *Source) PreUpdate(op int, schema, table string, rowid int64, ncols int, old, new []any, scanErr error) { + if s.faulted.Load() { + return + } + if !schemaAllowed(schema) || !s.tableAllowed(table) { + return + } + if scanErr != nil { + s.fault("read preupdate row: " + scanErr.Error()) return } + s.pendMu.Lock() - s.pending = append(s.pending, capturedChange{op: op, table: table, rowid: rowid, old: old, new: new}) + if len(s.pending) >= s.maxRows || s.pendingBytes >= s.maxBytes { + s.pendMu.Unlock() + s.fault(ErrChangeBacklogOverflow.Error()) + return + } + s.pending = append(s.pending, capturedChange{op: op, schema: schema, table: table, rowid: rowid, ncols: ncols, old: old, new: new}) + s.pendingBytes += approxRowSize(old) + approxRowSize(new) s.pendMu.Unlock() } func (s *Source) Commit() { + if s.faulted.Load() { + s.resetPending() + return + } + s.pendMu.Lock() batch := s.pending s.pending = nil + s.pendingBytes = 0 s.pendMu.Unlock() if len(batch) == 0 { return } + select { case s.commits <- batch: - case <-s.runDone: + default: + s.fault(ErrChangeBacklogOverflow.Error()) } } func (s *Source) Rollback() { - s.pendMu.Lock() - s.pending = nil - s.pendMu.Unlock() + s.resetPending() +} + +func schemaAllowed(schema string) bool { + return schema == "" || strings.EqualFold(schema, "main") } func (s *Source) tableAllowed(table string) bool { - lower := strings.ToLower(table) - if lower == offsetsTable { - return false - } if len(s.tables) == 0 { return true } - _, ok := s.tables[lower] + _, ok := s.tables[strings.ToLower(table)] + return ok } @@ -167,46 +263,21 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { return nil, fmt.Errorf("acquire writer conn: %w", err) } - var file string - if rawErr := conn.Raw(func(dc any) error { - f, e := installHooksOnRaw(dc, s) - file = f - return e - }); rawErr != nil { + file, token, err := s.installWithRetry(ctx, conn) + if err != nil { _ = conn.Close() res.Release() - return nil, rawErr - } - registerSink(file, s) - - s.mu.Lock() - s.poolRes = res - s.writerDB = writerDB - s.file = file - s.mu.Unlock() - - readDB, checkpointDB, err := openAuxConns(file) - if err != nil { - s.abortStart(ctx, conn, writerDB, file) return nil, err } - s.mu.Lock() - s.readDB = readDB - s.checkpointDB = checkpointDB - s.mu.Unlock() - - if err := ensureOffsets(ctx, checkpointDB); err != nil { - s.abortStart(ctx, conn, writerDB, file) + readDB, err := openReadConn(file) + if err != nil { + _ = conn.Close() + s.detachHooks(ctx, writerDB, file, token) + res.Release() return nil, err } - snapDone, lastSeq, loadErr := loadOffset(ctx, checkpointDB, s.name) - if loadErr != nil { - s.log.Warn("load cdc offset failed; treating as fresh", zap.Error(loadErr)) - } - if lastSeq > s.seq.Load() { - s.seq.Store(lastSeq) - } + _ = conn.Close() runCtx, cancel := context.WithCancel(ctx) status := make(chan any, 8) @@ -217,44 +288,61 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { if s.stopped.Load() { s.mu.Unlock() cancel() - s.abortStart(ctx, conn, writerDB, file) + _ = readDB.Close() + s.detachHooks(ctx, writerDB, file, token) + res.Release() return nil, ErrSourceClosed } + epoch := strconv.FormatInt(time.Now().UnixNano(), 10) + s.poolRes = res + s.writerDB = writerDB + s.readDB = readDB + s.file = file + s.token = token + s.epoch = epoch s.cancel = cancel + s.runCtx = runCtx s.runDone = runDone s.commits = commits s.mu.Unlock() - doSnapshot := s.snap && !snapDone - if doSnapshot { - if err := s.runSnapshot(runCtx, conn); err != nil { - cancel() - s.abortStart(ctx, conn, writerDB, file) - return nil, fmt.Errorf("snapshot: %w", err) - } - if serr := saveSnapshotDone(runCtx, checkpointDB, s.name); serr != nil { - s.log.Warn("persist snapshot completion failed; restart may re-snapshot", zap.Error(serr)) - } + select { + case status <- "sqlite cdc started": + default: } go s.run(runCtx, status, runDone, metrics.GetCollector(ctx)) - _ = conn.Close() + s.log.Info("sqlite cdc source started", zap.String("file", file), zap.String("epoch", epoch)) - select { - case status <- "sqlite cdc started": - default: - } - s.log.Info("sqlite cdc source started", - zap.String("file", s.file), - zap.Bool("snapshot", doSnapshot)) return status, nil } -func (s *Source) abortStart(ctx context.Context, conn *sql.Conn, writerDB *sql.DB, file string) { - _ = conn.Close() - s.detachHooks(ctx, writerDB, file) - s.releaseResources(ctx) +func (s *Source) installWithRetry(ctx context.Context, conn *sql.Conn) (string, uint64, error) { + var file string + var token uint64 + for attempt := 0; attempt < claimAttempts; attempt++ { + err := conn.Raw(func(dc any) error { + f, t, e := installHooksOnRaw(dc, s) + file, token = f, t + + return e + }) + if err == nil { + return file, token, nil + } + if !errors.Is(err, errCaptureOwned) { + return "", 0, err + } + + select { + case <-ctx.Done(): + return "", 0, ctx.Err() + case <-time.After(claimRetryDelay): + } + } + + return "", 0, errCaptureOwned } func (s *Source) acquirePool(ctx context.Context) (sqlservice.DBResource, resource.Resource[any], error) { @@ -262,11 +350,13 @@ func (s *Source) acquirePool(ctx context.Context) (sqlservice.DBResource, resour if err != nil { return sqlservice.DBResource{}, nil, fmt.Errorf("acquire db resource: %w", err) } + dbAny, err := res.Get() if err != nil { res.Release() return sqlservice.DBResource{}, nil, fmt.Errorf("get db resource: %w", err) } + dbRes, ok := dbAny.(sqlservice.DBResource) if !ok { res.Release() @@ -276,6 +366,7 @@ func (s *Source) acquirePool(ctx context.Context) (sqlservice.DBResource, resour res.Release() return sqlservice.DBResource{}, nil, fmt.Errorf("resource %s is not a sqlite database (kind %s)", s.name, dbRes.Type) } + return dbRes, res, nil } @@ -290,6 +381,7 @@ func (s *Source) Stop(ctx context.Context) error { runDone := s.runDone writerDB := s.writerDB file := s.file + token := s.token s.mu.Unlock() if cancel != nil { @@ -299,51 +391,43 @@ func (s *Source) Stop(ctx context.Context) error { select { case <-runDone: case <-ctx.Done(): - return ctx.Err() + <-runDone } } if writerDB != nil { - s.detachHooks(ctx, writerDB, file) + s.detachHooks(ctx, writerDB, file, token) } - if s.dropCP.Load() { - s.mu.Lock() - cpDB := s.checkpointDB - s.mu.Unlock() - if cpDB != nil { - _ = deleteOffset(ctx, cpDB, s.name) - } - } - s.releaseResources(ctx) + s.releaseResources() + return nil } -func (s *Source) detachHooks(ctx context.Context, writerDB *sql.DB, file string) { - unregisterSink(file) - conn, err := writerDB.Conn(ctx) +func (s *Source) detachHooks(ctx context.Context, writerDB *sql.DB, file string, token uint64) { + releaseCapture(file, token) + + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + + conn, err := writerDB.Conn(cleanupCtx) if err != nil { return } - _ = conn.Raw(clearHooksOnRaw) + _ = conn.Raw(func(dc any) error { return applyOwnerOnRaw(dc, file) }) _ = conn.Close() } -func (s *Source) releaseResources(_ context.Context) { +func (s *Source) releaseResources() { s.mu.Lock() readDB := s.readDB - cpDB := s.checkpointDB res := s.poolRes s.readDB = nil - s.checkpointDB = nil s.poolRes = nil s.mu.Unlock() if readDB != nil { _ = readDB.Close() } - if cpDB != nil { - _ = cpDB.Close() - } if res != nil { res.Release() } @@ -356,14 +440,23 @@ func (s *Source) run(ctx context.Context, status chan any, runDone chan struct{} ticker := time.NewTicker(s.statusInterval) defer ticker.Stop() + faultCh := s.faultCh for { select { case batch := <-s.commits: - s.process(ctx, batch, mc) + if !s.faulted.Load() { + s.process(ctx, batch, mc) + } + case <-faultCh: + s.emitFault(ctx) + faultCh = nil case <-ticker.C: - s.onTick(ctx, mc) + s.onTick(mc) case <-ctx.Done(): - s.drainRemaining(ctx, mc) + if !s.faulted.Load() { + s.drainRemaining(ctx, mc) + } + return } } @@ -380,14 +473,31 @@ func (s *Source) drainRemaining(ctx context.Context, mc metrics.Collector) { } } +func (s *Source) emitFault(ctx context.Context) { + msg := "sqlite cdc source faulted" + if p := s.faultMsg.Load(); p != nil { + msg = *p + } + + s.log.Error("sqlite cdc source faulted", zap.String("source", s.name), zap.String("reason", msg)) + s.subs.publish(ctx, config.Change{Source: s.name, Op: "error", Error: msg}) +} + func (s *Source) process(ctx context.Context, batch []capturedChange, mc metrics.Collector) { + s.refreshSchemaVersion(ctx) for _, ch := range batch { cols := s.columnsFor(ctx, ch.table) + if ch.ncols > 0 && len(cols) > 0 && len(cols) != ch.ncols { + s.invalidateColumns(ch.table) + cols = s.columnsFor(ctx, ch.table) + } + op := opString(ch.op) seq := s.seq.Add(1) change := config.Change{ Source: s.name, Op: op, + Schema: normalizeSchema(ch.schema), Table: ch.table, Relation: ch.table, Before: mapRow(cols, ch.old), @@ -401,16 +511,32 @@ func (s *Source) process(ctx context.Context, batch []capturedChange, mc metrics } } -func (s *Source) onTick(ctx context.Context, mc metrics.Collector) { - if mc != nil { - if info, err := os.Stat(s.file + "-wal"); err == nil { - mc.GaugeSet(walGauge, float64(info.Size()), metrics.Labels{"source": s.name}) - } +func (s *Source) refreshSchemaVersion(ctx context.Context) { + var ver int64 + if err := s.readDB.QueryRowContext(ctx, "PRAGMA schema_version").Scan(&ver); err != nil { + return } - if seq := s.seq.Load(); seq > 0 { - if err := saveOffset(ctx, s.checkpointDB, s.name, seq); err != nil { - s.log.Warn("persist cdc offset failed", zap.Error(err)) - } + + prev := s.schemaVer.Swap(ver) + if prev != 0 && prev != ver { + s.colMu.Lock() + s.cols = make(map[string][]columnInfo) + s.colMu.Unlock() + } +} + +func (s *Source) invalidateColumns(table string) { + s.colMu.Lock() + delete(s.cols, table) + s.colMu.Unlock() +} + +func (s *Source) onTick(mc metrics.Collector) { + if mc == nil { + return + } + if info, err := os.Stat(s.file + "-wal"); err == nil { + mc.GaugeSet(walGauge, float64(info.Size()), metrics.Labels{"source": s.name}) } } @@ -426,14 +552,25 @@ func (s *Source) columnsFor(ctx context.Context, table string) []columnInfo { if err != nil { s.log.Warn("resolve columns failed; emitting positional column names", zap.String("table", table), zap.Error(err)) + return nil } + s.colMu.Lock() s.cols[table] = cols s.colMu.Unlock() + return cols } +func normalizeSchema(schema string) string { + if schema == "" { + return "main" + } + + return schema +} + func opString(op int) string { switch op { case cdcInsert: @@ -447,20 +584,42 @@ func opString(op int) string { } } -func openAuxConns(file string) (read, checkpoint *sql.DB, err error) { - read, err = sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)+"&_query_only=ON") +func openReadConn(file string) (*sql.DB, error) { + db, err := sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)+"&_query_only=ON") if err != nil { - return nil, nil, fmt.Errorf("open read connection: %w", err) + return nil, fmt.Errorf("open read connection: %w", err) + } + + db.SetMaxOpenConns(2) + db.SetMaxIdleConns(2) + + return db, nil +} + +func approxRowSize(vals []any) int { + size := 0 + for _, v := range vals { + switch t := v.(type) { + case []byte: + size += len(t) + case string: + size += len(t) + default: + size += 8 + } } - read.SetMaxOpenConns(1) - read.SetMaxIdleConns(1) - checkpoint, err = sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)) + return size +} + +func openSnapshotConn(file string) (*sql.DB, error) { + db, err := sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)+"&_query_only=ON") if err != nil { - _ = read.Close() - return nil, nil, fmt.Errorf("open checkpoint connection: %w", err) + return nil, fmt.Errorf("open snapshot connection: %w", err) } - checkpoint.SetMaxOpenConns(1) - checkpoint.SetMaxIdleConns(1) - return read, checkpoint, nil + + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + return db, nil } diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go index 05c451505..13721da50 100644 --- a/service/cdc/sqlite/subscribers.go +++ b/service/cdc/sqlite/subscribers.go @@ -26,7 +26,7 @@ func newSubscribers() *subscribers { return &subscribers{m: make(map[uint64]*subscription)} } -func (s *subscribers) subscribe(opts config.StreamOptions) config.ChangeStream { +func (s *subscribers) subscribe(sourceName string, opts config.StreamOptions, wantSnapshot bool) *subscription { buffer := opts.Buffer if buffer <= 0 { buffer = defaultStreamBuffer @@ -38,22 +38,29 @@ func (s *subscribers) subscribe(opts config.StreamOptions) config.ChangeStream { s.mu.Lock() s.next++ sub := &subscription{ - parent: s, - id: s.next, - in: make(chan config.Change, buffer), - out: make(chan config.Change, buffer), - done: make(chan struct{}), - tables: filterSet(opts.Tables), - ops: filterSet(opts.Ops), + parent: s, + id: s.next, + sourceName: sourceName, + in: make(chan config.Change, buffer), + out: make(chan config.Change, buffer), + done: make(chan struct{}), + termCh: make(chan struct{}), + tables: filterSet(opts.Tables), + ops: filterSet(opts.Ops), + wantSnapshot: wantSnapshot, + } + if wantSnapshot { + sub.snap = make(chan config.Change) } s.m[sub.id] = sub s.mu.Unlock() go sub.run() + return sub } -func (s *subscribers) publish(ctx context.Context, change config.Change) { +func (s *subscribers) publish(_ context.Context, change config.Change) { s.mu.RLock() matched := make([]*subscription, 0, len(s.m)) for _, sub := range s.m { @@ -64,7 +71,7 @@ func (s *subscribers) publish(ctx context.Context, change config.Change) { s.mu.RUnlock() for _, sub := range matched { - sub.send(ctx, change) + sub.send(change) } } @@ -89,15 +96,21 @@ func (s *subscribers) closeAll() { } type subscription struct { - parent *subscribers - in chan config.Change - out chan config.Change - done chan struct{} - tables map[string]struct{} - ops map[string]struct{} - id uint64 - once sync.Once - closed atomic.Bool + parent *subscribers + in chan config.Change + out chan config.Change + snap chan config.Change + done chan struct{} + termCh chan struct{} + tables map[string]struct{} + ops map[string]struct{} + term atomic.Pointer[config.Change] + sourceName string + id uint64 + closeOnce sync.Once + failOnce sync.Once + closed atomic.Bool + wantSnapshot bool } func (s *subscription) Changes() <-chan config.Change { @@ -105,7 +118,7 @@ func (s *subscription) Changes() <-chan config.Change { } func (s *subscription) Close() { - s.once.Do(func() { + s.closeOnce.Do(func() { s.closed.Store(true) if s.parent != nil { s.parent.remove(s.id) @@ -114,39 +127,76 @@ func (s *subscription) Close() { }) } +func (s *subscription) fail(reason string) { + s.failOnce.Do(func() { + c := config.Change{Source: s.sourceName, Op: "error", Error: reason} + s.term.Store(&c) + s.closed.Store(true) + if s.parent != nil { + s.parent.remove(s.id) + } + close(s.termCh) + }) +} + func (s *subscription) run() { defer close(s.out) + + if s.wantSnapshot && !s.pump(s.snap, true) { + s.flushTerm() + return + } + if !s.pump(s.in, false) { + s.flushTerm() + } +} + +func (s *subscription) pump(src <-chan config.Change, snapshotPhase bool) bool { for { select { case <-s.done: - return - default: - } - select { - case change := <-s.in: + return false + case <-s.termCh: + return false + case change, ok := <-src: + if !ok { + return snapshotPhase + } select { case <-s.done: - return + return false + case <-s.termCh: + return false case s.out <- change: } + } + } +} + +func (s *subscription) flushTerm() { + if t := s.term.Load(); t != nil { + select { + case s.out <- *t: case <-s.done: - return } } } -func (s *subscription) send(_ context.Context, change config.Change) { +func (s *subscription) send(change config.Change) { if s.closed.Load() { return } select { case s.in <- change: default: - s.Close() + s.fail("sqlite cdc subscriber backlog overflow") } } func (s *subscription) matches(change config.Change) bool { + if change.Op == "error" || change.Op == "snapshot" { + return true + } if len(s.ops) > 0 { if _, ok := s.ops[strings.ToLower(change.Op)]; !ok { return false @@ -159,8 +209,10 @@ func (s *subscription) matches(change config.Change) bool { if _, ok := s.tables[strings.ToLower(change.Table)]; ok { return true } + return false } + return true } @@ -168,6 +220,7 @@ func filterSet(values []string) map[string]struct{} { if len(values) == 0 { return nil } + out := make(map[string]struct{}, len(values)) for _, v := range values { v = strings.ToLower(strings.TrimSpace(v)) @@ -178,5 +231,6 @@ func filterSet(values []string) map[string]struct{} { if len(out) == 0 { return nil } + return out } diff --git a/service/cdc/sqlite/subscribers_snapshot.go b/service/cdc/sqlite/subscribers_snapshot.go new file mode 100644 index 000000000..aff4e47cf --- /dev/null +++ b/service/cdc/sqlite/subscribers_snapshot.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "strings" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +func (s *subscription) isClosed() bool { + return s.closed.Load() +} + +func (s *subscription) finishSnapshot() { + if s.snap != nil { + close(s.snap) + } +} + +func (s *subscription) sendSnapshot(ctx context.Context, change config.Change) bool { + select { + case s.snap <- change: + return true + case <-s.done: + return false + case <-s.termCh: + return false + case <-ctx.Done(): + return false + } +} + +func (s *subscription) tableAllowed(name string) bool { + if len(s.tables) == 0 { + return true + } + _, ok := s.tables[strings.ToLower(name)] + + return ok +} diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go index 20e192a91..973600ec2 100644 --- a/service/cdc/sqlite/subscribers_test.go +++ b/service/cdc/sqlite/subscribers_test.go @@ -39,7 +39,7 @@ func TestSubscriptionMatches(t *testing.T) { func TestSubscribersPublishAndClose(t *testing.T) { subs := newSubscribers() - stream := subs.subscribe(config.StreamOptions{}) + stream := subs.subscribe("s", config.StreamOptions{}, false) subs.publish(context.Background(), config.Change{Op: "insert", Table: "users", Source: "s"}) @@ -63,30 +63,30 @@ func TestSubscribersPublishAndClose(t *testing.T) { func TestSubscribeBufferClamp(t *testing.T) { subs := newSubscribers() - def := subs.subscribe(config.StreamOptions{Buffer: 0}).(*subscription) + def := subs.subscribe("s", config.StreamOptions{Buffer: 0}, false) assert.Equal(t, defaultStreamBuffer, cap(def.in)) - neg := subs.subscribe(config.StreamOptions{Buffer: -5}).(*subscription) + neg := subs.subscribe("s", config.StreamOptions{Buffer: -5}, false) assert.Equal(t, defaultStreamBuffer, cap(neg.in)) - exact := subs.subscribe(config.StreamOptions{Buffer: 7}).(*subscription) + exact := subs.subscribe("s", config.StreamOptions{Buffer: 7}, false) assert.Equal(t, 7, cap(exact.in)) - huge := subs.subscribe(config.StreamOptions{Buffer: maxStreamBuffer + 100}).(*subscription) + huge := subs.subscribe("s", config.StreamOptions{Buffer: maxStreamBuffer + 100}, false) assert.Equal(t, maxStreamBuffer, cap(huge.in)) } func TestSubscribeAssignsUniqueIncreasingIDs(t *testing.T) { subs := newSubscribers() - a := subs.subscribe(config.StreamOptions{}).(*subscription) - b := subs.subscribe(config.StreamOptions{}).(*subscription) + a := subs.subscribe("s", config.StreamOptions{}, false) + b := subs.subscribe("s", config.StreamOptions{}, false) assert.Equal(t, uint64(1), a.id) assert.Equal(t, uint64(2), b.id) } func TestPublishNeverBlocksAndClosesLaggard(t *testing.T) { subs := newSubscribers() - stream := subs.subscribe(config.StreamOptions{Buffer: 1}) + stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}, false) done := make(chan struct{}) go func() { @@ -116,7 +116,7 @@ func TestPublishNeverBlocksAndClosesLaggard(t *testing.T) { func TestSubscribersFilterByOp(t *testing.T) { subs := newSubscribers() - stream := subs.subscribe(config.StreamOptions{Ops: []string{"delete"}}) + stream := subs.subscribe("s", config.StreamOptions{Ops: []string{"delete"}}, false) defer stream.Close() subs.publish(context.Background(), config.Change{Op: "insert", Table: "users"}) From ddf2b505b39af4468bade4ec4840dc9ee98d0624 Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Wed, 15 Jul 2026 13:31:26 -0300 Subject: [PATCH 04/47] test(cdc): unit helpers + multi-subscriber, multi-table & no-gap concurrent-write snapshot coverage - Unit: schemaAllowed/normalizeSchema/opString/approxRowSize and the control-event (snapshot/error) filter-bypass in subscription.matches. - Integration: per-subscriber table filtering fan-out; snapshot covering multiple tables; snapshot with concurrent live writes observes every row with no gap (convergence guarantee). --- service/cdc/sqlite/helpers_test.go | 41 +++++++ .../cdc/sqlite/redesign_integration_test.go | 104 ++++++++++++++++++ service/cdc/sqlite/subscribers_test.go | 12 ++ 3 files changed, 157 insertions(+) create mode 100644 service/cdc/sqlite/helpers_test.go diff --git a/service/cdc/sqlite/helpers_test.go b/service/cdc/sqlite/helpers_test.go new file mode 100644 index 000000000..cd0c325e6 --- /dev/null +++ b/service/cdc/sqlite/helpers_test.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSchemaAllowed(t *testing.T) { + assert.True(t, schemaAllowed("")) + assert.True(t, schemaAllowed("main")) + assert.True(t, schemaAllowed("MAIN")) + assert.False(t, schemaAllowed("temp")) + assert.False(t, schemaAllowed("attached")) +} + +func TestNormalizeSchema(t *testing.T) { + assert.Equal(t, "main", normalizeSchema("")) + assert.Equal(t, "temp", normalizeSchema("temp")) + assert.Equal(t, "audit", normalizeSchema("audit")) +} + +func TestOpString(t *testing.T) { + assert.Equal(t, "insert", opString(cdcInsert)) + assert.Equal(t, "update", opString(cdcUpdate)) + assert.Equal(t, "delete", opString(cdcDelete)) + assert.Equal(t, "unknown", opString(9999)) +} + +func TestApproxRowSize(t *testing.T) { + assert.Equal(t, 0, approxRowSize(nil)) + assert.Equal(t, 8, approxRowSize([]any{int64(1)})) + assert.Equal(t, 3, approxRowSize([]any{[]byte{1, 2, 3}})) + assert.Equal(t, 5, approxRowSize([]any{"hello"})) + assert.Equal(t, 8+3+5, approxRowSize([]any{1.5, []byte("abc"), "hello"})) + assert.Equal(t, 8, approxRowSize([]any{nil})) +} diff --git a/service/cdc/sqlite/redesign_integration_test.go b/service/cdc/sqlite/redesign_integration_test.go index 17a67f1db..ac9a46405 100644 --- a/service/cdc/sqlite/redesign_integration_test.go +++ b/service/cdc/sqlite/redesign_integration_test.go @@ -223,3 +223,107 @@ func TestIntegrationStopCleansUpWithExpiredContext(t *testing.T) { require.NoError(t, err, "hooks must be released after Stop so a new source can claim capture") require.NoError(t, src2.Stop(context.Background())) } + +func TestIntegrationMultipleSubscribersFilters(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + subAll := src.Subscribe(config.StreamOptions{}) + defer subAll.Close() + subUsers := src.Subscribe(config.StreamOptions{Tables: []string{"users"}}) + defer subUsers.Close() + + _, err = db.Exec(`INSERT INTO orders (id, v) VALUES (1, 'o')`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO users (id, v) VALUES (1, 'u')`) + require.NoError(t, err) + + only := waitChange(t, subUsers.Changes()) + assert.Equal(t, "users", only.Table, "the users-filtered subscriber must skip the orders write") + assert.Equal(t, "u", only.After["v"]) + + first := waitChange(t, subAll.Changes()) + assert.Equal(t, "orders", first.Table) + second := waitChange(t, subAll.Changes()) + assert.Equal(t, "users", second.Table) +} + +func TestIntegrationSnapshotCoversMultipleTables(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) + require.NoError(t, err) + _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, total REAL)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'a@b.com')`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO orders (id, total) VALUES (1, 99.5)`) + require.NoError(t, err) + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + sub := src.Subscribe(config.StreamOptions{Snapshot: true}) + defer sub.Close() + + seen := map[string]bool{} + for i := 0; i < 2; i++ { + c := waitChange(t, sub.Changes()) + require.Equal(t, "snapshot", c.Op) + seen[c.Table] = true + } + assert.True(t, seen["users"], "snapshot must cover the users table") + assert.True(t, seen["orders"], "snapshot must cover the orders table") +} + +func TestIntegrationSnapshotWithConcurrentWritesNoGap(t *testing.T) { + db, _ := openPool(t) + _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(t, err) + + const preRows = 20 + const liveRows = 20 + for i := 1; i <= preRows; i++ { + _, err = db.Exec(`INSERT INTO t (id, v) VALUES (?, 'pre')`, i) + require.NoError(t, err) + } + + src := newSource(t, db, sourceOptions{}) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + sub := src.Subscribe(config.StreamOptions{Snapshot: true}) + defer sub.Close() + + go func() { + for i := preRows + 1; i <= preRows+liveRows; i++ { + _, _ = db.Exec(`INSERT INTO t (id, v) VALUES (?, 'live')`, i) + } + }() + + seen := map[int64]bool{} + deadline := time.After(15 * time.Second) + for len(seen) < preRows+liveRows { + select { + case c := <-sub.Changes(): + if id, ok := c.After["id"].(int64); ok { + seen[id] = true + } + case <-deadline: + t.Fatalf("did not observe all rows without a gap; saw %d/%d", len(seen), preRows+liveRows) + } + } + for i := int64(1); i <= preRows+liveRows; i++ { + assert.Truef(t, seen[i], "row %d missing: snapshot+live must cover every row with no gap", i) + } +} diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go index 973600ec2..c081f43ce 100644 --- a/service/cdc/sqlite/subscribers_test.go +++ b/service/cdc/sqlite/subscribers_test.go @@ -37,6 +37,18 @@ func TestSubscriptionMatches(t *testing.T) { assert.False(t, byTable.matches(config.Change{Op: "insert", Table: "orders"})) } +func TestSubscriptionMatchesBypassesFiltersForControlEvents(t *testing.T) { + sub := &subscription{ + ops: map[string]struct{}{"insert": {}}, + tables: map[string]struct{}{"users": {}}, + } + + assert.True(t, sub.matches(config.Change{Op: "error", Table: "orders"}), "terminal error must reach every subscriber") + assert.True(t, sub.matches(config.Change{Op: "snapshot", Table: "orders"}), "snapshot rows must bypass op/table filters") + assert.False(t, sub.matches(config.Change{Op: "delete", Table: "users"}), "op filter still applies to normal changes") + assert.False(t, sub.matches(config.Change{Op: "insert", Table: "orders"}), "table filter still applies to normal changes") +} + func TestSubscribersPublishAndClose(t *testing.T) { subs := newSubscribers() stream := subs.subscribe("s", config.StreamOptions{}, false) From f40446b6c7643992468901da15008ffa22f87b68 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 8 Aug 2026 23:29:42 -0400 Subject: [PATCH 05/47] feat(cdc): add driver-neutral source registry and manager --- api/service/cdc/command.go | 32 +- api/service/cdc/context.go | 102 ++++- api/service/cdc/context_test.go | 24 ++ api/service/cdc/errors.go | 1 + boot/components/service/storage/cdc.go | 31 +- boot/components/system/all.go | 1 + boot/components/system/cdc.go | 25 ++ boot/components/system/constants.go | 1 + service/cdc/manager.go | 293 ++++++++++++++ service/cdc/manager_test.go | 485 +++++++++++++++++++++++ service/cdc/postgres/driver.go | 238 ++++++++++++ service/cdc/slot.go | 507 +++++++++++++++++++++++++ service/cdc/stream.go | 90 +++++ system/cdc/registry.go | 152 ++++++++ system/cdc/registry_test.go | 99 +++++ 15 files changed, 2057 insertions(+), 24 deletions(-) create mode 100644 boot/components/system/cdc.go create mode 100644 service/cdc/manager.go create mode 100644 service/cdc/manager_test.go create mode 100644 service/cdc/postgres/driver.go create mode 100644 service/cdc/slot.go create mode 100644 service/cdc/stream.go create mode 100644 system/cdc/registry.go create mode 100644 system/cdc/registry_test.go diff --git a/api/service/cdc/command.go b/api/service/cdc/command.go index 7eb409180..4715809ab 100644 --- a/api/service/cdc/command.go +++ b/api/service/cdc/command.go @@ -7,6 +7,7 @@ import ( "github.com/wippyai/runtime/api/dispatcher" "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" ) func init() { @@ -22,20 +23,29 @@ type StreamOptions struct { Ops []string Buffer int Snapshot bool + // After is an opaque source cursor. A driver that cannot resume from a + // cursor must return ErrUnsupported rather than silently ignore it. + After string } type Change struct { - Before map[string]any `json:"before,omitempty"` - After map[string]any `json:"after,omitempty"` - Source string `json:"source"` - Op string `json:"op"` - Schema string `json:"schema"` - Table string `json:"table"` - Relation string `json:"relation"` - LSN string `json:"lsn"` - CommitLSN string `json:"commit_lsn,omitempty"` - Error string `json:"error,omitempty"` - XID uint32 `json:"xid,omitempty"` + Before map[string]any `json:"before,omitempty"` + After map[string]any `json:"after,omitempty"` + Source string `json:"source"` + // SourceID is the canonical registry identity. Source is retained as the + // legacy wire representation used by existing Lua consumers. + SourceID registry.ID `json:"source_id,omitempty"` + Op string `json:"op"` + Schema string `json:"schema"` + Table string `json:"table"` + Relation string `json:"relation"` + LSN string `json:"lsn"` + CommitLSN string `json:"commit_lsn,omitempty"` + Cursor string `json:"cursor,omitempty"` + Generation string `json:"generation,omitempty"` + Transaction string `json:"transaction,omitempty"` + Error string `json:"error,omitempty"` + XID uint32 `json:"xid,omitempty"` } type SubscribeCmd struct { diff --git a/api/service/cdc/context.go b/api/service/cdc/context.go index 3960518fb..234f138da 100644 --- a/api/service/cdc/context.go +++ b/api/service/cdc/context.go @@ -2,9 +2,80 @@ package cdc -import "context" +import ( + "context" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/registry" +) + +// SourceState is the driver-neutral lifecycle state exposed by a CDC source. +// Driver-specific health details remain in SourceInfo.Error and the legacy +// compatibility fields below. +type SourceState string + +const ( + SourceStateUnknown SourceState = "unknown" + SourceStateStarting SourceState = "starting" + SourceStateRunning SourceState = "running" + SourceStateFaulted SourceState = "faulted" + SourceStateStopped SourceState = "stopped" +) + +// Capabilities describes guarantees provided by a source. The common API does +// not infer PostgreSQL or SQLite semantics from the source kind. +type Capabilities struct { + Snapshot bool `json:"snapshot,omitempty"` + Durable bool `json:"durable,omitempty"` + Replayable bool `json:"replayable,omitempty"` + CapturesExternalWrites bool `json:"captures_external_writes,omitempty"` + BeforeImages bool `json:"before_images,omitempty"` + Coalesced bool `json:"coalesced,omitempty"` +} + +// Stream is the driver-neutral event stream. Err is intentionally optional so +// existing stream implementations remain source-compatible; new sources may +// implement ErrStream to expose a terminal cause without synthesizing an +// error-valued row. +type Stream interface { + Changes() <-chan Change + Close() +} + +// ErrStream exposes a terminal stream error. A consumer should type-assert +// this interface after Changes is closed. +type ErrStream interface { + Stream + Err() error +} + +// Source is the common source contract implemented by every CDC driver. +// Subscribe receives a context so a source can reject subscriptions while it +// is not ready and can bind snapshot work to the caller's lifetime. +type Source interface { + Info() SourceInfo + Subscribe(context.Context, StreamOptions) (Stream, error) +} + +// Registry is the read-only system-level CDC registry exposed to services and +// runtimes. Registry IDs, rather than driver aliases such as PostgreSQL slots, +// are the only global identity. +type Registry interface { + List() []SourceInfo + Get(registry.ID) (Source, bool) +} type SourceInfo struct { + ID registry.ID `json:"id,omitempty"` + Kind registry.Kind `json:"kind,omitempty"` + State SourceState `json:"state,omitempty"` + Capabilities Capabilities `json:"capabilities,omitempty"` + Generation string `json:"generation,omitempty"` + + // The fields below are retained for wire compatibility with existing Lua + // and API consumers. New code must use ID, Kind, State, Capabilities and + // Generation; driver-specific metadata should not be added to the common + // contract. Name string `json:"name"` Slot string `json:"slot"` Publication string `json:"publication,omitempty"` @@ -52,3 +123,32 @@ func GetSourceStreamer(ctx context.Context) SourceStreamer { v, _ := ctx.Value(sourceStreamerKey{}).(SourceStreamer) return v } + +var registryKey = &ctxapi.Key{Name: "cdc.registry"} + +// WithRegistry attaches the driver-neutral CDC registry to the application +// context. Like the network and resource APIs, this is a write-once boot +// dependency and is safe to read after the application context is sealed. +func WithRegistry(ctx context.Context, registry Registry) context.Context { + if registry == nil { + return ctx + } + ac := ctxapi.AppFromContext(ctx) + if ac == nil { + return ctx + } + if ac.Get(registryKey) == nil { + ac.With(registryKey, registry) + } + return ctx +} + +// GetRegistry retrieves the system CDC registry from the application context. +func GetRegistry(ctx context.Context) Registry { + ac := ctxapi.AppFromContext(ctx) + if ac == nil { + return nil + } + registry, _ := ac.Get(registryKey).(Registry) + return registry +} diff --git a/api/service/cdc/context_test.go b/api/service/cdc/context_test.go index 933ae8da8..4f9b5d0a0 100644 --- a/api/service/cdc/context_test.go +++ b/api/service/cdc/context_test.go @@ -7,16 +7,21 @@ import ( "testing" "github.com/stretchr/testify/assert" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/registry" ) type stubInspector struct{} type stubStreamer struct{} +type stubRegistry struct{} func (stubInspector) List() []SourceInfo { return nil } func (stubInspector) Get(string) (SourceInfo, bool) { return SourceInfo{}, false } func (stubStreamer) Stream(context.Context, string, StreamOptions) (ChangeStream, SourceInfo, error) { return nil, SourceInfo{}, nil } +func (stubRegistry) List() []SourceInfo { return nil } +func (stubRegistry) Get(registry.ID) (Source, bool) { return nil, false } func TestWithSourceInspectorRoundTrip(t *testing.T) { ctx := WithSourceInspector(context.Background(), stubInspector{}) @@ -47,3 +52,22 @@ func TestWithSourceStreamerNilDoesNotAttach(t *testing.T) { func TestGetSourceStreamerEmptyCtx(t *testing.T) { assert.Nil(t, GetSourceStreamer(context.Background())) } + +func TestWithRegistryRoundTripOnApplicationContext(t *testing.T) { + ctx := ctxapi.NewRootContext() + reg := stubRegistry{} + ctx = WithRegistry(ctx, reg) + assert.Equal(t, reg, GetRegistry(ctx)) +} + +func TestWithRegistryDoesNotAttachToPlainContext(t *testing.T) { + reg := stubRegistry{} + ctx := WithRegistry(context.Background(), reg) + assert.Nil(t, GetRegistry(ctx)) +} + +func TestWithRegistryNilDoesNotAttach(t *testing.T) { + ctx := ctxapi.NewRootContext() + ctx = WithRegistry(ctx, nil) + assert.Nil(t, GetRegistry(ctx)) +} diff --git a/api/service/cdc/errors.go b/api/service/cdc/errors.go index f26752037..87c20fd75 100644 --- a/api/service/cdc/errors.go +++ b/api/service/cdc/errors.go @@ -17,4 +17,5 @@ var ( ErrInvalidSnapshotFetchSize = apierror.New(apierror.Invalid, "snapshot_fetch_size must be non-negative").WithRetryable(apierror.False) ErrDBResourceRequired = apierror.New(apierror.Invalid, "db_resource is required").WithRetryable(apierror.False) ErrSourceNotFound = apierror.New(apierror.NotFound, "cdc source not found").WithRetryable(apierror.False) + ErrUnsupported = apierror.New(apierror.Invalid, "cdc operation is not supported by this source").WithRetryable(apierror.False) ) diff --git a/boot/components/service/storage/cdc.go b/boot/components/service/storage/cdc.go index 196020b21..19e7f83f9 100644 --- a/boot/components/service/storage/cdc.go +++ b/boot/components/service/storage/cdc.go @@ -4,6 +4,7 @@ package storage import ( "context" + "errors" "github.com/wippyai/runtime/api/boot" "github.com/wippyai/runtime/api/event" @@ -13,6 +14,7 @@ import ( cdcapi "github.com/wippyai/runtime/api/service/cdc" bootpkg "github.com/wippyai/runtime/boot" bootsystem "github.com/wippyai/runtime/boot/components/system" + cdcservice "github.com/wippyai/runtime/service/cdc" pgcdc "github.com/wippyai/runtime/service/cdc/postgres" sqlitecdc "github.com/wippyai/runtime/service/cdc/sqlite" ) @@ -20,30 +22,35 @@ import ( func CDC() boot.Component { return boot.New(boot.P{ Name: CDCName, - DependsOn: []boot.Name{bootsystem.EnvironmentName, bootsystem.ResourcesName}, + DependsOn: []boot.Name{bootsystem.EnvironmentName, bootsystem.ResourcesName, bootsystem.CDCRegistryName}, Load: func(ctx context.Context) (context.Context, error) { logger := logapi.GetLogger(ctx) dtt := payload.GetTranscoder(ctx) bus := event.GetBus(ctx) resReg := resourceapi.GetRegistry(ctx) handlers := bootpkg.GetHandlerRegistry(ctx) - - pgManager, err := pgcdc.NewManager(dtt, bus, logger.Named("cdc.postgres")) - if err != nil { - return ctx, NewCDCManagerError(err) + if cdcapi.GetRegistry(ctx) == nil { + return ctx, NewCDCManagerError(errors.New("cdc system registry not available")) } - sqliteManager, err := sqlitecdc.NewManager(dtt, bus, logger.Named("cdc.sqlite"), resReg) + cdcRegistry, ok := cdcapi.GetRegistry(ctx).(cdcservice.Registry) + if !ok { + return ctx, NewCDCManagerError(errors.New("cdc system registry has an unsupported implementation")) + } + manager, err := cdcservice.NewManager( + cdcRegistry, + dtt, + bus, + resReg, + logger.Named("cdc"), + cdcservice.WithDriver(pgcdc.NewDriver(), sqlitecdc.NewDriver()), + ) if err != nil { return ctx, NewCDCManagerError(err) } - handlers.RegisterListener("db.cdc.postgres", pgManager) - handlers.RegisterListener("db.cdc.sqlite", sqliteManager) - - composite := cdcapi.NewComposite(pgManager, sqliteManager) - ctx = cdcapi.WithSourceInspector(ctx, composite) - ctx = cdcapi.WithSourceStreamer(ctx, composite) + handlers.RegisterListener("db.cdc.postgres", manager) + handlers.RegisterListener("db.cdc.sqlite", manager) return ctx, nil }, }) diff --git a/boot/components/system/all.go b/boot/components/system/all.go index ed389fcd0..b5410cfc0 100644 --- a/boot/components/system/all.go +++ b/boot/components/system/all.go @@ -21,6 +21,7 @@ func All() []boot.Component { Network(), SocketDispatcher(), Resources(), + CDC(), Factory(), ProcessManager(), Interceptor(), diff --git a/boot/components/system/cdc.go b/boot/components/system/cdc.go new file mode 100644 index 000000000..31b84e5c2 --- /dev/null +++ b/boot/components/system/cdc.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MPL-2.0 + +package system + +import ( + "context" + + "github.com/wippyai/runtime/api/boot" + logapi "github.com/wippyai/runtime/api/logs" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + cdcsystem "github.com/wippyai/runtime/system/cdc" +) + +// CDC creates the driver-neutral source registry. Concrete drivers are +// injected by the service layer after this component has published the +// registry into the application context. +func CDC() boot.Component { + return boot.New(boot.P{ + Name: CDCRegistryName, + Load: func(ctx context.Context) (context.Context, error) { + logger := logapi.GetLogger(ctx) + return cdcapi.WithRegistry(ctx, cdcsystem.NewRegistry(logger.Named("cdc"))), nil + }, + }) +} diff --git a/boot/components/system/constants.go b/boot/components/system/constants.go index 3723d2d73..bd3bacc3a 100644 --- a/boot/components/system/constants.go +++ b/boot/components/system/constants.go @@ -10,6 +10,7 @@ const ( EnvironmentName boot.Name = "env" NetworkName boot.Name = "network" ResourcesName boot.Name = "resources" + CDCRegistryName boot.Name = "cdc.registry" InterceptorName boot.Name = "interceptor" FrameResolversName boot.Name = "frame_resolvers" FunctionsName boot.Name = "functions" diff --git a/service/cdc/manager.go b/service/cdc/manager.go new file mode 100644 index 000000000..0db607ecc --- /dev/null +++ b/service/cdc/manager.go @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package cdc contains the driver router and lifecycle owner for CDC sources. +// Database-specific packages implement Driver; this package never imports a +// concrete database implementation. +package cdc + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/wippyai/runtime/api/event" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + api "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + "go.uber.org/zap" +) + +var ( + ErrRegistryRequired = errors.New("cdc manager: registry is required") + ErrEventBusRequired = errors.New("cdc manager: event bus is required") + ErrDriverRequired = errors.New("cdc manager: driver is required") + ErrUnsupportedKind = errors.New("cdc manager: unsupported source kind") + ErrSourceExists = errors.New("cdc manager: source already exists") + ErrSourceNotFound = errors.New("cdc manager: source not found") +) + +// Dependencies are the shared collaborators available to concrete drivers. +// The manager owns lifecycle event emission; drivers only construct sources. +type Dependencies struct { + Transcoder payload.Transcoder + Resources resource.Registry + Logger *zap.Logger +} + +// ManagedSource is the internal source returned by a driver. Source +// construction and lifecycle remain separate from the system registry, while +// one value is used by both the public source API and supervisor. +type ManagedSource interface { + api.Source + supervisor.Service +} + +// Registry is the mutable capability the manager needs from the system CDC +// registry. Keeping the concrete system implementation behind this interface +// lets boot and tests inject the canonical registry without coupling the +// service package to a particular registry implementation. +type Registry interface { + api.Registry + Register(registry.ID, api.Source, registry.Kind) error + Unregister(registry.ID) (api.Source, bool) +} + +// Disposable is an optional destructive-delete hook. It is invoked only for +// manager.Delete, while the source remains as a non-subscribable tombstone; +// registry removal commits only after disposal succeeds. Ordinary +// Stop/replacement never calls it, which lets drivers retain durable resources +// such as PostgreSQL replication slots across restart and update while still +// cleaning them up on dynamic uninstall. +type Disposable interface { + Dispose(context.Context) error +} + +// ExclusiveResource identifies a resource that cannot be held by two source +// generations at once (for example, a persistent PostgreSQL replication slot). +// Sources with different keys can be started before the registry swap. Sources +// with the same non-empty key use the slot's stop-start-restart handoff. +type ExclusiveResource interface { + ExclusiveResourceKey() string +} + +// Driver constructs a source for one registry kind. Drivers are injected when +// the manager is built; package initialization must not mutate global routing. +type Driver interface { + Kind() registry.Kind + Create(context.Context, registry.Entry, Dependencies) (ManagedSource, error) +} + +// Option configures a Manager. +type Option func(*Manager) + +// WithDriver injects one or more concrete source drivers. A later driver for +// the same kind intentionally replaces an earlier test or extension driver. +func WithDriver(drivers ...Driver) Option { + return func(m *Manager) { + for _, driver := range drivers { + if driver == nil { + continue + } + m.drivers[driver.Kind()] = driver + } + } +} + +// Manager routes registry entries to injected drivers and owns their +// supervisor/system-registry lifecycle. +type Manager struct { + registry Registry + bus event.Bus + deps Dependencies + log *zap.Logger + drivers map[registry.Kind]Driver + mu sync.Mutex +} + +func NewManager( + reg Registry, + dtt payload.Transcoder, + bus event.Bus, + resources resource.Registry, + log *zap.Logger, + opts ...Option, +) (*Manager, error) { + if reg == nil { + return nil, ErrRegistryRequired + } + if bus == nil { + return nil, ErrEventBusRequired + } + if log == nil { + log = zap.NewNop() + } + m := &Manager{ + registry: reg, + bus: bus, + deps: Dependencies{ + Transcoder: dtt, + Resources: resources, + Logger: log, + }, + log: log, + drivers: make(map[registry.Kind]Driver), + } + for _, opt := range opts { + if opt != nil { + opt(m) + } + } + return m, nil +} + +func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + + driver, ok := m.drivers[entry.Kind] + if !ok { + return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) + } + id := canonicalID(entry.ID) + if _, exists := m.registry.Get(id); exists { + return fmt.Errorf("%w: %s", ErrSourceExists, id.String()) + } + + source, err := driver.Create(ctx, entry, m.deps) + if err != nil { + return err + } + if source == nil { + return ErrDriverRequired + } + slot := newSourceSlot(id, entry.Kind, source, m.log.With(zap.String("id", id.String()))) + if err := m.registry.Register(id, slot, entry.Kind); err != nil { + _ = source.Stop(ctx) + return err + } + m.registerSupervisor(ctx, id, slot) + m.log.Info("added cdc source", zap.String("id", id.String()), zap.String("kind", entry.Kind)) + return nil +} + +func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + + driver, ok := m.drivers[entry.Kind] + if !ok { + return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) + } + id := canonicalID(entry.ID) + _, exists := m.registry.Get(id) + if !exists { + return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) + } + + // Build the replacement before changing visibility. A malformed entry or + // failed dependency acquisition leaves the old source untouched. + replacement, err := driver.Create(ctx, entry, m.deps) + if err != nil { + return err + } + if replacement == nil { + return ErrDriverRequired + } + slot, ok := m.registry.Get(id) + if !ok { + _ = replacement.Stop(ctx) + return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) + } + managedSlot, ok := slot.(*sourceSlot) + if !ok { + _ = replacement.Stop(ctx) + return errors.New("cdc manager: source is not managed by a stable slot") + } + if err := managedSlot.Replace(ctx, replacement); err != nil { + return err + } + m.log.Info("updated cdc source", zap.String("id", id.String()), zap.String("kind", entry.Kind)) + return nil +} + +func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + + id := canonicalID(entry.ID) + source, ok := m.registry.Get(id) + if !ok { + return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) + } + var err error + if disposable, ok := source.(Disposable); ok { + err = disposable.Dispose(ctx) + } else { + err = stopSource(ctx, source) + } + if err != nil { + m.log.Warn("cdc source failed to stop during delete", + zap.String("id", id.String()), zap.Error(err)) + return err + } + if _, ok := m.registry.Unregister(id); !ok { + return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) + } + m.unregisterSupervisor(ctx, id) + m.log.Info("removed cdc source", zap.String("id", id.String())) + return nil +} + +func (m *Manager) List() []api.SourceInfo { + return m.registry.List() +} + +func (m *Manager) Get(id registry.ID) (api.Source, bool) { + return m.registry.Get(id) +} + +func (m *Manager) registerSupervisor(ctx context.Context, id registry.ID, source ManagedSource) { + cfg := supervisor.LifecycleConfig{} + if configured, ok := source.(interface { + LifecycleConfig() supervisor.LifecycleConfig + }); ok { + cfg = configured.LifecycleConfig() + } + cfg.InitDefaults() + m.bus.Send(ctx, event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRegister, + Path: id.String(), + Data: &supervisor.Entry{ + Service: source, + Config: cfg, + }, + }) +} + +func (m *Manager) unregisterSupervisor(ctx context.Context, id registry.ID) { + m.bus.Send(ctx, event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRemove, + Path: id.String(), + }) +} + +func stopSource(ctx context.Context, source api.Source) error { + if managed, ok := source.(supervisor.Service); ok { + return managed.Stop(ctx) + } + return nil +} + +func canonicalID(id registry.ID) registry.ID { + return registry.ParseID(id.String()) +} + +var ( + _ registry.EntryListener = (*Manager)(nil) + _ api.Registry = (*Manager)(nil) +) diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go new file mode 100644 index 000000000..b8a3c98a4 --- /dev/null +++ b/service/cdc/manager_test.go @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/event" + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + cdcsystem "github.com/wippyai/runtime/system/cdc" + "github.com/wippyai/runtime/system/eventbus" +) + +type testStream struct { + changes chan api.Change +} + +func (s *testStream) Changes() <-chan api.Change { return s.changes } +func (s *testStream) Close() { close(s.changes) } + +type managedTestSource struct { + info api.SourceInfo + startErr error + stopErr error + exclusive string + stream *testStream + startCount atomic.Int32 + stopCount atomic.Int32 + active atomic.Int32 + maxActive atomic.Int32 +} + +type disposableTestSource struct { + *managedTestSource + disposeCount atomic.Int32 + disposeErr error + failedOnce atomic.Bool +} + +func (s *disposableTestSource) Dispose(ctx context.Context) error { + s.disposeCount.Add(1) + if s.disposeErr != nil && s.failedOnce.CompareAndSwap(false, true) { + return s.disposeErr + } + return s.Stop(ctx) +} + +func (s *managedTestSource) Info() api.SourceInfo { return s.info } + +func (s *managedTestSource) Subscribe(context.Context, api.StreamOptions) (api.Stream, error) { + if s.stream != nil { + return s.stream, nil + } + return &testStream{changes: make(chan api.Change)}, nil +} + +func (s *managedTestSource) Start(context.Context) (<-chan any, error) { + s.startCount.Add(1) + if s.startErr == nil { + active := s.active.Add(1) + for { + max := s.maxActive.Load() + if active <= max || s.maxActive.CompareAndSwap(max, active) { + break + } + } + } + return nil, s.startErr +} + +func (s *managedTestSource) Stop(context.Context) error { + s.stopCount.Add(1) + if s.active.Load() > 0 { + s.active.Add(-1) + } + return s.stopErr +} + +func (s *managedTestSource) LifecycleConfig() supervisor.LifecycleConfig { + return supervisor.LifecycleConfig{AutoStart: true} +} + +func (s *managedTestSource) ExclusiveResourceKey() string { return s.exclusive } + +type testDriver struct { + kind registry.Kind + create func(registry.Entry) (ManagedSource, error) +} + +func (d testDriver) Kind() registry.Kind { return d.kind } + +func (d testDriver) Create(_ context.Context, entry registry.Entry, _ Dependencies) (ManagedSource, error) { + return d.create(entry) +} + +type recordingBus struct { + mu sync.Mutex + events []event.Event +} + +func (b *recordingBus) Subscribe(context.Context, event.System, chan<- event.Event) (event.SubscriberID, error) { + return "", nil +} + +func (b *recordingBus) SubscribeP(context.Context, event.System, event.Kind, chan<- event.Event) (event.SubscriberID, error) { + return "", nil +} + +func (*recordingBus) Unsubscribe(context.Context, event.SubscriberID) {} + +func (b *recordingBus) Send(_ context.Context, e event.Event) { + b.mu.Lock() + b.events = append(b.events, e) + b.mu.Unlock() +} + +func (b *recordingBus) snapshot() []event.Event { + b.mu.Lock() + defer b.mu.Unlock() + return append([]event.Event(nil), b.events...) +} + +func newManagerTest(t *testing.T, drivers ...Driver) (*Manager, *eventbus.Bus) { + t.Helper() + bus := eventbus.NewBus() + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(drivers...)) + require.NoError(t, err) + t.Cleanup(bus.Stop) + return m, bus +} + +func TestNewManagerRequiresRegistryAndBus(t *testing.T) { + bus := eventbus.NewBus() + t.Cleanup(bus.Stop) + _, err := NewManager(nil, nil, bus, nil, nil) + require.ErrorIs(t, err, ErrRegistryRequired) + _, err = NewManager(cdcsystem.NewRegistry(nil), nil, nil, nil, nil) + require.ErrorIs(t, err, ErrEventBusRequired) +} + +func TestManagerRoutesCanonicalIDsAndOwnsLifecycle(t *testing.T) { + var created []*managedTestSource + driver := testDriver{ + kind: "db.cdc.test", + create: func(entry registry.Entry) (ManagedSource, error) { + source := &managedTestSource{info: api.SourceInfo{Name: "driver-name", Generation: entry.ID.String()}} + created = append(created, source) + return source, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.ID{NS: "app", Name: "events"} + entry := registry.Entry{ID: id, Kind: driver.kind} + + require.NoError(t, m.Add(context.Background(), entry)) + got, ok := m.Get(registry.ParseID("app:events")) + require.True(t, ok) + slot, ok := got.(*sourceSlot) + require.True(t, ok) + slot.mu.RLock() + require.Same(t, created[0], slot.current) + slot.mu.RUnlock() + require.Equal(t, "app:events", m.List()[0].ID.String()) + require.Equal(t, registry.Kind(driver.kind), m.List()[0].Kind) + require.ErrorIs(t, m.Add(context.Background(), entry), ErrSourceExists) + + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: registry.ParseID("app:events")})) + require.EqualValues(t, 1, created[0].stopCount.Load()) + _, ok = m.Get(id) + require.False(t, ok) +} + +func TestManagerDeleteInvokesDisposeOnlyAfterUnregister(t *testing.T) { + source := &disposableTestSource{managedTestSource: &managedTestSource{info: api.SourceInfo{Name: "source"}}} + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { return source, nil }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Delete(context.Background(), entry)) + + require.EqualValues(t, 1, source.disposeCount.Load()) + require.EqualValues(t, 1, source.stopCount.Load()) + _, ok := m.Get(id) + require.False(t, ok) + events := bus.snapshot() + require.Len(t, events, 2) + require.Equal(t, supervisor.ServiceRegister, events[0].Kind) + require.Equal(t, supervisor.ServiceRemove, events[1].Kind) +} + +func TestManagerDeleteRetainsTombstoneForDisposeRetry(t *testing.T) { + source := &disposableTestSource{ + managedTestSource: &managedTestSource{info: api.SourceInfo{Name: "source"}}, + disposeErr: errors.New("cleanup failed"), + } + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { return source, nil }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + + require.EqualError(t, m.Delete(context.Background(), entry), "cleanup failed") + _, ok := m.Get(id) + require.True(t, ok, "failed disposal must leave a retryable tombstone") + require.Len(t, bus.snapshot(), 1, "supervisor removal waits for successful disposal") + + require.NoError(t, m.Delete(context.Background(), entry)) + _, ok = m.Get(id) + require.False(t, ok) + require.EqualValues(t, 2, source.disposeCount.Load()) + require.EqualValues(t, 1, source.stopCount.Load()) + events := bus.snapshot() + require.Len(t, events, 2) + require.Equal(t, supervisor.ServiceRemove, events[1].Kind) +} + +func TestManagerUpdateBuildFailureLeavesOldSource(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}} + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + return nil, errors.New("candidate rejected") + }, + } + m, _ := newManagerTest(t, driver) + // Seed the system registry through the same manager with a temporary + // successful driver, then switch the injected driver to the failing one. + seed := testDriver{kind: driver.kind, create: func(registry.Entry) (ManagedSource, error) { return old, nil }} + m.drivers[driver.kind] = seed + id := registry.NewID("app", "events") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: id, Kind: driver.kind})) + m.drivers[driver.kind] = driver + + err := m.Update(context.Background(), registry.Entry{ID: id, Kind: driver.kind}) + require.EqualError(t, err, "candidate rejected") + got, ok := m.Get(id) + require.True(t, ok) + slot, ok := got.(*sourceSlot) + require.True(t, ok) + slot.mu.RLock() + require.Same(t, old, slot.current) + slot.mu.RUnlock() + require.EqualValues(t, 0, old.stopCount.Load()) +} + +func TestManagerUpdateAtomicallyReplacesAndStopsOld(t *testing.T) { + var next int + var created []*managedTestSource + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + source := &managedTestSource{info: api.SourceInfo{Name: string(rune('a' + next))}} + created = append(created, source) + return source, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + require.Len(t, created, 2) + got, ok := m.Get(id) + require.True(t, ok) + slot, ok := got.(*sourceSlot) + require.True(t, ok) + slot.mu.RLock() + require.Same(t, created[1], slot.current) + slot.mu.RUnlock() + require.EqualValues(t, 1, created[0].stopCount.Load()) +} + +func TestManagerUpdateDoesNotFailAfterNewGenerationCommits(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, stopErr: errors.New("old cleanup failed")} + newSource := &managedTestSource{info: api.SourceInfo{Name: "new"}} + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return newSource, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + require.Same(t, newSource, mustSlot(t, m, id).currentSource()) + require.EqualValues(t, 1, old.stopCount.Load()) +} + +func TestManagerUpdateKeepsStableSupervisorRegistration(t *testing.T) { + var next int + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + return &managedTestSource{info: api.SourceInfo{Name: string(rune('a' + next))}}, nil + }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + + events := bus.snapshot() + require.Len(t, events, 1) + require.Equal(t, supervisor.ServiceRegister, events[0].Kind) + require.Equal(t, id.String(), events[0].Path) + registered := events[0].Data.(*supervisor.Entry) + current, ok := m.Get(id) + require.True(t, ok) + require.Same(t, current, registered.Service) +} + +func TestManagerUpdateFailedStartRetainsRunningGeneration(t *testing.T) { + var next int + old := &managedTestSource{info: api.SourceInfo{Name: "old"}} + failed := &managedTestSource{info: api.SourceInfo{Name: "failed"}, startErr: errors.New("candidate start failed")} + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return failed, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + source, ok := m.Get(id) + require.True(t, ok) + slot := source.(*sourceSlot) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "candidate start failed") + slot.mu.RLock() + require.Same(t, old, slot.current) + require.Equal(t, slotRunning, slot.state) + slot.mu.RUnlock() + require.EqualValues(t, 0, old.stopCount.Load()) +} + +func TestManagerUpdateSameExclusiveKeyStopsAndRestores(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-1"} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-1", startErr: errors.New("candidate start failed")} + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "candidate start failed") + slot.mu.RLock() + require.Same(t, old, slot.current) + require.Equal(t, slotRunning, slot.state) + slot.mu.RUnlock() + require.EqualValues(t, 1, old.stopCount.Load()) + require.EqualValues(t, 2, old.startCount.Load(), "old generation must be restored after its initial start") + require.EqualValues(t, 1, old.maxActive.Load(), "exclusive generations must never overlap") +} + +func TestManagerUpdateSameExclusiveKeyStopFailureFaultsSlot(t *testing.T) { + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-1", stopErr: errors.New("old stop failed")} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-1"} + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "old stop failed") + require.Same(t, old, slot.currentSource()) + slot.mu.RLock() + require.Equal(t, slotFaulted, slot.state) + slot.mu.RUnlock() + require.EqualValues(t, 0, candidate.startCount.Load(), "candidate must not start while old ownership is uncertain") +} + +func TestSourceSlotStampsCanonicalIdentityAndGeneration(t *testing.T) { + id := registry.NewID("app", "events") + upstream := &testStream{changes: make(chan api.Change, 1)} + source := &managedTestSource{stream: upstream} + slot := newSourceSlot(id, "db.cdc.test", source) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + stream, err := slot.Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + upstream.changes <- api.Change{Source: "driver-alias", Generation: "driver-generation"} + + select { + case change := <-stream.Changes(): + require.Equal(t, id, change.SourceID) + require.Equal(t, id.String(), change.Source) + require.Equal(t, "1", change.Generation) + case <-time.After(time.Second): + t.Fatal("timed out waiting for stamped change") + } + stream.Close() + require.NoError(t, slot.Stop(context.Background())) +} + +func mustSlot(t *testing.T, m *Manager, id registry.ID) *sourceSlot { + t.Helper() + source, ok := m.Get(id) + require.True(t, ok) + slot, ok := source.(*sourceSlot) + require.True(t, ok) + return slot +} + +func (s *sourceSlot) currentSource() ManagedSource { + s.mu.RLock() + defer s.mu.RUnlock() + return s.current +} + +func TestManagerRejectsUnsupportedAndMissingSources(t *testing.T) { + m, _ := newManagerTest(t) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: "db.cdc.unknown"} + require.ErrorIs(t, m.Add(context.Background(), entry), ErrUnsupportedKind) + require.ErrorIs(t, m.Update(context.Background(), entry), ErrUnsupportedKind) + require.ErrorIs(t, m.Delete(context.Background(), entry), ErrSourceNotFound) +} + +var _ event.Bus = (*eventbus.Bus)(nil) diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go new file mode 100644 index 000000000..964fbb474 --- /dev/null +++ b/service/cdc/postgres/driver.go @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" + + "github.com/wippyai/runtime/api/registry" + config "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + cdcservice "github.com/wippyai/runtime/service/cdc" + entryutil "github.com/wippyai/runtime/system/entry" + "go.uber.org/zap" +) + +// Driver wires PostgreSQL CDC into the driver-neutral CDC manager. It only +// constructs a source; registry visibility, replacement, and supervisor +// lifecycle remain owned by service/cdc. +type Driver struct{} + +func NewDriver() cdcservice.Driver { return Driver{} } + +func (Driver) Kind() registry.Kind { return config.Postgres } + +func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice.Dependencies) (cdcservice.ManagedSource, error) { + if deps.Transcoder == nil { + return nil, ErrTranscoderRequired + } + cfg, err := entryutil.DecodeEntryConfig[config.Config](ctx, deps.Transcoder, entry) + if err != nil { + return nil, NewInvalidConfigError(err) + } + if err := cfg.Validate(); err != nil { + return nil, NewInvalidConfigError(err) + } + standby, _ := cfg.StandbyDuration() + status, _ := cfg.StatusDuration() + replDSN, adminDSN, err := buildDSNs(cfg) + if err != nil { + return nil, err + } + log := deps.Logger + if log == nil { + log = zap.NewNop() + } + opts := SourceOptions{ + ReplDSN: replDSN, + AdminDSN: adminDSN, + Name: entry.ID.String(), + Slot: cfg.SlotName, + Publication: cfg.Publication, + Tables: cfg.Tables, + Temporary: cfg.Temporary, + Snapshot: cfg.Snapshot, + Streaming: cfg.Streaming, + Failover: cfg.Failover, + StandbyInterval: standby, + StatusInterval: status, + SnapshotFetchSize: cfg.SnapshotFetchSize, + MaxTransactionChanges: cfg.MaxTransactionChanges, + MaxTransactionBytes: cfg.MaxTransactionBytes, + Log: log.With(zap.String("id", entry.ID.String())), + } + return &sourceAdapter{ + source: NewSource(opts), + opts: opts, + lifecycle: cfg.Lifecycle, + exclusiveKey: postgresExclusiveKey(cfg), + }, nil +} + +// sourceAdapter preserves the existing PostgreSQL source implementation while +// exposing the common context-aware CDC contract. The old source stream API +// remains private to this adapter, so no driver-specific shape leaks into the +// common manager or dispatcher. +type sourceAdapter struct { + mu sync.RWMutex + source *Source + opts SourceOptions + lifecycle supervisor.LifecycleConfig + exclusiveKey string +} + +func (s *sourceAdapter) Info() config.SourceInfo { + s.mu.RLock() + source := s.source + s.mu.RUnlock() + source.mu.Lock() + state := source.state + source.mu.Unlock() + + info := config.SourceInfo{ + Kind: config.Postgres, + Name: source.name, + Slot: source.slot, + Publication: source.publication, + Tables: append([]string(nil), source.tables...), + Streaming: state == sourceRunning, + Failover: source.failover, + Temporary: source.temporary, + Snapshot: source.snapshot, + State: postgresSourceState(state), + Capabilities: config.Capabilities{ + Snapshot: source.snapshot, + Durable: true, + Replayable: true, + CapturesExternalWrites: true, + BeforeImages: false, + }, + } + if state == sourceFailed { + info.Faulted = true + } + return info +} + +func (s *sourceAdapter) Subscribe(ctx context.Context, opts config.StreamOptions) (config.Stream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if opts.After != "" { + return nil, config.ErrUnsupported + } + if opts.Snapshot { + return nil, fmt.Errorf("%w: snapshot is configured on the source", config.ErrUnsupported) + } + + s.mu.RLock() + source := s.source + source.mu.Lock() + state := source.state + source.mu.Unlock() + if state != sourceRunning { + s.mu.RUnlock() + return nil, config.ErrSourceNotReady + } + stream := source.Subscribe(opts) + s.mu.RUnlock() + return stream, nil +} + +func (s *sourceAdapter) Start(ctx context.Context) (<-chan any, error) { + s.mu.RLock() + source := s.source + opts := s.opts + s.mu.RUnlock() + status, err := source.Start(ctx) + if !errors.Is(err, ErrSourceClosed) { + return status, err + } + + // Source deliberately makes a stopped generation terminal so a stale + // replication connection can never be reused. The stable manager slot can + // still restart the logical generation by constructing a fresh source with + // the same immutable configuration and checkpoint identity. + fresh := NewSource(opts) + s.mu.Lock() + if s.source == source { + s.source = fresh + } + source = s.source + s.mu.Unlock() + return source.Start(ctx) +} + +func (s *sourceAdapter) Stop(ctx context.Context) error { + s.mu.RLock() + source := s.source + s.mu.RUnlock() + return source.Stop(ctx) +} + +// Dispose is used only for a committed dynamic delete. The generic manager +// keeps a non-subscribable tombstone until this completes, so retries can +// finish cleanup. Ordinary Stop/replacement never drops the slot. +func (s *sourceAdapter) Dispose(ctx context.Context) error { + s.mu.RLock() + source := s.source + s.mu.RUnlock() + source.MarkForSlotDrop() + stopErr := source.Stop(ctx) + source.mu.Lock() + temporary := source.temporary + source.mu.Unlock() + if temporary { + return stopErr + } + cleanupErr := source.dropSlotAndCheckpoint(ctx) + return errors.Join(stopErr, cleanupErr) +} + +func (s *sourceAdapter) LifecycleConfig() supervisor.LifecycleConfig { + return s.lifecycle +} + +// PostgreSQL replication slots are exclusive resources. The stable manager +// slot uses this key to perform a stop/start handoff for updates that retain +// the same slot, avoiding a concurrent replication-slot ownership error. +func (s *sourceAdapter) ExclusiveResourceKey() string { + s.mu.RLock() + key := s.exclusiveKey + s.mu.RUnlock() + return key +} + +func postgresExclusiveKey(cfg *config.Config) string { + host := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(cfg.Host), ".")) + endpoint := net.JoinHostPort(host, strconv.Itoa(cfg.Port)) + return "postgres/" + endpoint + "/" + cfg.Database + "/slot/" + cfg.SlotName +} + +func postgresSourceState(state sourceState) config.SourceState { + switch state { + case sourceStarting: + return config.SourceStateStarting + case sourceRunning: + return config.SourceStateRunning + case sourceFailed: + return config.SourceStateFaulted + case sourceStopped: + return config.SourceStateStopped + default: + return config.SourceStateUnknown + } +} + +var _ cdcservice.ManagedSource = (*sourceAdapter)(nil) +var _ cdcservice.ExclusiveResource = (*sourceAdapter)(nil) +var _ cdcservice.Disposable = (*sourceAdapter)(nil) diff --git a/service/cdc/slot.go b/service/cdc/slot.go new file mode 100644 index 000000000..d15aaeb9d --- /dev/null +++ b/service/cdc/slot.go @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "errors" + "strconv" + "sync" + + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + "go.uber.org/zap" +) + +var ( + ErrSourceClosed = errors.New("cdc source slot is closed") + ErrSourceBusy = errors.New("cdc source slot is stopping") +) + +type slotState uint8 + +const ( + slotIdle slotState = iota + slotStarting + slotRunning + slotStopping + slotFaulted + slotStopped +) + +// sourceSlot is the stable object placed in both the system registry and the +// supervisor. A driver replacement changes the delegated generation, never the +// supervisor object or registry pointer. +type sourceSlot struct { + id registry.ID + kind registry.Kind + + opMu sync.Mutex + mu sync.RWMutex + + current ManagedSource + log *zap.Logger + generation uint64 + state slotState + runCtx context.Context + runCancel context.CancelFunc + status chan any + statusDone bool + replacing bool + disposing bool +} + +func newSourceSlot(id registry.ID, kind registry.Kind, source ManagedSource, logs ...*zap.Logger) *sourceSlot { + log := zap.NewNop() + if len(logs) > 0 && logs[0] != nil { + log = logs[0] + } + return &sourceSlot{ + id: canonicalID(id), + kind: kind, + current: source, + log: log, + generation: 1, + state: slotIdle, + } +} + +func (s *sourceSlot) Info() api.SourceInfo { + s.mu.RLock() + current := s.current + generation := s.generation + state := s.state + s.mu.RUnlock() + if current == nil { + return api.SourceInfo{ + ID: s.id, + Kind: s.kind, + Generation: generationString(generation), + State: sourceState(state), + } + } + info := current.Info() + info.ID = s.id + info.Kind = s.kind + info.Generation = generationString(generation) + info.State = sourceState(state) + return info +} + +func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api.Stream, error) { + s.mu.RLock() + if s.state != slotRunning || s.current == nil || s.disposing { + s.mu.RUnlock() + return nil, api.ErrSourceNotReady + } + current := s.current + generation := s.generation + s.mu.RUnlock() + stream, err := current.Subscribe(ctx, opts) + if err != nil { + return nil, err + } + if stream == nil { + return nil, errors.New("cdc source returned a nil stream") + } + + s.mu.RLock() + stillCurrent := s.state == slotRunning && s.current == current && s.generation == generation + s.mu.RUnlock() + if !stillCurrent { + stream.Close() + return nil, api.ErrSourceNotReady + } + return newStampedStream(s.id, generation, opts.Buffer, stream), nil +} + +// Start is idempotent while the active generation is running. This is what +// permits an update to synchronously start a candidate before the supervisor +// receives its unchanged stable slot pointer. +func (s *sourceSlot) Start(ctx context.Context) (<-chan any, error) { + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.Lock() + if s.disposing { + s.mu.Unlock() + return nil, ErrSourceBusy + } + if s.state == slotRunning { + status := s.status + s.mu.Unlock() + return status, nil + } + if s.state == slotStopping { + s.mu.Unlock() + return nil, ErrSourceBusy + } + if s.current == nil { + s.mu.Unlock() + return nil, ErrSourceClosed + } + current := s.current + status := make(chan any, 8) + s.status = status + s.statusDone = false + s.state = slotStarting + runCtx, runCancel := detachedContext(ctx) + s.runCtx = runCtx + s.runCancel = runCancel + s.replacing = false + s.mu.Unlock() + + underlying, err := startSource(ctx, runCtx, current) + if err != nil { + _ = stopSource(ctx, current) + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.mu.Unlock() + return nil, err + } + + s.mu.Lock() + s.state = slotRunning + generation := s.generation + s.mu.Unlock() + s.watchStatus(current, generation, underlying) + return status, nil +} + +func (s *sourceSlot) Stop(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.Lock() + if s.disposing { + if s.state == slotStopped || s.state == slotFaulted { + s.mu.Unlock() + return nil + } + s.mu.Unlock() + return ErrSourceBusy + } + if s.state == slotStopped { + s.mu.Unlock() + return nil + } + s.state = slotStopping + current := s.current + cancel := s.runCancel + s.mu.Unlock() + + if cancel != nil { + cancel() + } + err := stopSource(ctx, current) + + s.mu.Lock() + if err != nil { + s.state = slotFaulted + } else { + s.state = slotStopped + } + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.replacing = false + s.mu.Unlock() + return err +} + +// Dispose performs a committed delete. It mirrors Stop's stable-slot state +// transition, but delegates the destructive hook to the active driver only +// on this path. Updates and supervisor restarts always use Stop instead. +func (s *sourceSlot) Dispose(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.Lock() + s.disposing = true + s.state = slotStopping + current := s.current + cancel := s.runCancel + s.mu.Unlock() + + if cancel != nil { + cancel() + } + var err error + if disposable, ok := current.(Disposable); ok { + err = disposable.Dispose(ctx) + } else { + err = stopSource(ctx, current) + } + + s.mu.Lock() + if err != nil { + s.state = slotFaulted + } else { + s.state = slotStopped + } + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.replacing = false + s.mu.Unlock() + return err +} + +// Replace starts a candidate before changing visibility whenever the slot is +// running or the candidate is configured for auto-start. Failure leaves the +// old generation current and untouched. +func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error { + if candidate == nil { + return ErrDriverRequired + } + if ctx == nil { + ctx = context.Background() + } + s.opMu.Lock() + defer s.opMu.Unlock() + + s.mu.RLock() + old := s.current + state := s.state + runCtx := s.runCtx + runCancel := s.runCancel + disposing := s.disposing + s.mu.RUnlock() + if disposing { + return ErrSourceBusy + } + + startCandidate := state == slotRunning || lifecycleAutoStart(candidate) + sameExclusive := state == slotRunning && exclusiveResourceKey(old) != "" && + exclusiveResourceKey(old) == exclusiveResourceKey(candidate) + var underlying <-chan any + var err error + oldStopped := false + createdRunContext := false + keepRunContext := false + defer func() { + if createdRunContext && !keepRunContext && runCancel != nil { + runCancel() + } + }() + if startCandidate && state == slotRunning { + s.mu.Lock() + s.replacing = true + s.mu.Unlock() + } + if sameExclusive { + // A source such as PostgreSQL may not start a second generation while + // the old generation owns the same slot. Stop the old generation first; + // if the candidate fails, restore the old generation before returning. + if err := stopSource(ctx, old); err != nil { + s.mu.Lock() + s.replacing = false + s.state = slotFaulted + s.closeStatusLocked() + s.mu.Unlock() + return err + } + oldStopped = true + } + if startCandidate { + if runCtx == nil { + runCtx, runCancel = detachedContext(ctx) + createdRunContext = true + } + underlying, err = startSource(ctx, runCtx, candidate) + if err != nil { + _ = stopSource(ctx, candidate) + if sameExclusive { + var restartErr error + underlying, restartErr = startSource(ctx, runCtx, old) + if restartErr == nil { + s.mu.Lock() + s.state = slotRunning + s.replacing = false + generation := s.generation + s.mu.Unlock() + s.watchStatus(old, generation, underlying) + return err + } + s.mu.Lock() + s.state = slotFaulted + s.replacing = false + s.closeStatusLocked() + s.mu.Unlock() + return errors.Join(err, restartErr) + } + s.mu.Lock() + s.replacing = false + s.mu.Unlock() + return err + } + } + + s.mu.Lock() + if s.state == slotStopping { + s.mu.Unlock() + _ = stopSource(ctx, candidate) + return ErrSourceBusy + } + s.current = candidate + s.generation++ + if startCandidate { + keepRunContext = true + if s.status == nil || s.statusDone { + s.status = make(chan any, 8) + s.statusDone = false + } + s.state = slotRunning + s.runCtx = runCtx + s.runCancel = runCancel + s.replacing = false + generation := s.generation + s.mu.Unlock() + s.watchStatus(candidate, generation, underlying) + } else { + s.mu.Unlock() + } + + if old != nil && !oldStopped { + if err := stopSource(ctx, old); err != nil { + s.log.Warn("old cdc source failed to stop after replacement", + zap.String("id", s.id.String()), zap.Error(err)) + } + } + return nil +} + +func (s *sourceSlot) LifecycleConfig() supervisor.LifecycleConfig { + s.mu.RLock() + current := s.current + s.mu.RUnlock() + if configured, ok := current.(interface { + LifecycleConfig() supervisor.LifecycleConfig + }); ok { + return configured.LifecycleConfig() + } + return supervisor.LifecycleConfig{} +} + +func (s *sourceSlot) watchStatus(source ManagedSource, generation uint64, updates <-chan any) { + if updates == nil { + return + } + go func() { + for detail := range updates { + s.mu.RLock() + current := s.current == source && s.generation == generation && s.state == slotRunning && !s.replacing + status := s.status + s.mu.RUnlock() + if !current || status == nil { + continue + } + select { + case status <- detail: + default: + } + } + + s.mu.Lock() + if s.current == source && s.generation == generation && s.state == slotRunning && !s.replacing { + s.state = slotFaulted + s.closeStatusLocked() + } + s.mu.Unlock() + }() +} + +func exclusiveResourceKey(source ManagedSource) string { + if keyed, ok := source.(ExclusiveResource); ok { + return keyed.ExclusiveResourceKey() + } + return "" +} + +func (s *sourceSlot) currentGeneration() uint64 { + s.mu.RLock() + generation := s.generation + s.mu.RUnlock() + return generation +} + +func (s *sourceSlot) closeStatusLocked() { + if s.status != nil && !s.statusDone { + close(s.status) + s.statusDone = true + } +} + +func lifecycleAutoStart(source ManagedSource) bool { + configured, ok := source.(interface { + LifecycleConfig() supervisor.LifecycleConfig + }) + return ok && configured.LifecycleConfig().AutoStart +} + +func sourceState(state slotState) api.SourceState { + switch state { + case slotStarting: + return api.SourceStateStarting + case slotRunning: + return api.SourceStateRunning + case slotFaulted: + return api.SourceStateFaulted + case slotStopped: + return api.SourceStateStopped + default: + return api.SourceStateUnknown + } +} + +// startSource gives the startup operation its caller's cancellation while +// retaining a detached run context after successful startup. Supervisor +// start timeouts must still interrupt a blocked driver handshake, but a +// dynamic registry event must not cancel a source that it has just started. +func startSource(ctx context.Context, runCtx context.Context, source ManagedSource) (<-chan any, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if runCtx == nil { + runCtx = context.WithoutCancel(ctx) + } + startCtx, cancelStart := context.WithCancel(runCtx) + stopPropagation := context.AfterFunc(ctx, cancelStart) + updates, err := source.Start(startCtx) + stopPropagation() + if err == nil { + err = ctx.Err() + } + if err != nil { + cancelStart() + } + return updates, err +} + +func detachedContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithCancel(context.WithoutCancel(ctx)) +} + +func generationString(generation uint64) string { + if generation == 0 { + return "" + } + return strconv.FormatUint(generation, 10) +} + +var _ ManagedSource = (*sourceSlot)(nil) +var _ Disposable = (*sourceSlot)(nil) diff --git a/service/cdc/stream.go b/service/cdc/stream.go new file mode 100644 index 000000000..febf488d0 --- /dev/null +++ b/service/cdc/stream.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "sync" + + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" +) + +const ( + defaultStreamBuffer = 128 + maxStreamBuffer = 65536 +) + +// stampedStream is the boundary between a driver stream and the common CDC +// API. Drivers own transport-specific change decoding; the stable source slot +// owns the process identity and generation that consumers use for routing and +// resume diagnostics. +type stampedStream struct { + upstream api.Stream + sourceID registry.ID + generation string + out chan api.Change + done chan struct{} + once sync.Once +} + +func newStampedStream(id registry.ID, generation uint64, requestedBuffer int, upstream api.Stream) *stampedStream { + buffer := requestedBuffer + if buffer <= 0 { + buffer = defaultStreamBuffer + } + if buffer > maxStreamBuffer { + buffer = maxStreamBuffer + } + + stream := &stampedStream{ + upstream: upstream, + sourceID: registry.ParseID(id.String()), + generation: generationString(generation), + out: make(chan api.Change, buffer), + done: make(chan struct{}), + } + go stream.run() + return stream +} + +func (s *stampedStream) Changes() <-chan api.Change { return s.out } + +func (s *stampedStream) Close() { + s.once.Do(func() { + close(s.done) + s.upstream.Close() + }) +} + +func (s *stampedStream) Err() error { + if withError, ok := s.upstream.(interface{ Err() error }); ok { + return withError.Err() + } + return nil +} + +func (s *stampedStream) run() { + defer close(s.out) + changes := s.upstream.Changes() + for { + select { + case <-s.done: + return + case change, ok := <-changes: + if !ok { + return + } + change.SourceID = s.sourceID + change.Source = s.sourceID.String() + change.Generation = s.generation + select { + case <-s.done: + return + case s.out <- change: + } + } + } +} + +var _ api.Stream = (*stampedStream)(nil) +var _ api.ErrStream = (*stampedStream)(nil) diff --git a/system/cdc/registry.go b/system/cdc/registry.go new file mode 100644 index 000000000..232766966 --- /dev/null +++ b/system/cdc/registry.go @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package cdc contains the process-local registry of configured CDC sources. +// It deliberately knows nothing about source construction or any particular +// database driver; service/cdc owns that responsibility. +package cdc + +import ( + "errors" + "sort" + "sync" + + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" + "go.uber.org/zap" +) + +var ( + ErrSourceExists = errors.New("cdc source already registered") + ErrSourceMissing = errors.New("cdc source not registered") +) + +type entry struct { + source api.Source + kind registry.Kind +} + +// Registry is a concurrency-safe, driver-neutral source registry. The map is +// keyed only by canonical registry.ID values. Drivers must not introduce +// process-wide aliases such as PostgreSQL slot names into this layer. +type Registry struct { + log *zap.Logger + mu sync.RWMutex + sources map[registry.ID]entry +} + +func NewRegistry(log *zap.Logger) *Registry { + if log == nil { + log = zap.NewNop() + } + return &Registry{ + log: log, + sources: make(map[registry.ID]entry), + } +} + +// Register adds a source. It does not replace an existing source; callers +// must use Replace for an update so an accidental duplicate cannot orphan a +// running source. +func (r *Registry) Register(id registry.ID, source api.Source, kind registry.Kind) error { + if source == nil { + return errors.New("cdc source is nil") + } + id = canonicalID(id) + + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.sources[id]; exists { + return ErrSourceExists + } + r.sources[id] = entry{source: source, kind: kind} + r.log.Debug("cdc source registered", zap.String("id", id.String()), zap.String("kind", kind)) + return nil +} + +// Replace atomically makes source visible under id and returns the previously +// visible source. The old source is not stopped here: lifecycle ownership stays +// with service/cdc, which can stop it after the visibility swap. +func (r *Registry) Replace(id registry.ID, source api.Source, kind registry.Kind) (api.Source, bool, error) { + if source == nil { + return nil, false, errors.New("cdc source is nil") + } + id = canonicalID(id) + + r.mu.Lock() + old, exists := r.sources[id] + if !exists { + r.mu.Unlock() + return nil, false, ErrSourceMissing + } + r.sources[id] = entry{source: source, kind: kind} + r.mu.Unlock() + + r.log.Info("cdc source replaced", zap.String("id", id.String()), zap.String("kind", kind)) + return old.source, true, nil +} + +// Unregister removes a source and returns it to the lifecycle owner. +func (r *Registry) Unregister(id registry.ID) (api.Source, bool) { + id = canonicalID(id) + r.mu.Lock() + old, exists := r.sources[id] + if exists { + delete(r.sources, id) + } + r.mu.Unlock() + if exists { + r.log.Debug("cdc source unregistered", zap.String("id", id.String())) + return old.source, true + } + return nil, false +} + +func (r *Registry) Get(id registry.ID) (api.Source, bool) { + id = canonicalID(id) + r.mu.RLock() + item, ok := r.sources[id] + r.mu.RUnlock() + if !ok { + return nil, false + } + return item.source, true +} + +// List returns a deterministic snapshot. Metadata from the registry is merged +// over the source's own Info so a source cannot accidentally publish a +// different global identity or kind. +func (r *Registry) List() []api.SourceInfo { + r.mu.RLock() + items := make([]struct { + id registry.ID + kind registry.Kind + source api.Source + }, 0, len(r.sources)) + for id, item := range r.sources { + items = append(items, struct { + id registry.ID + kind registry.Kind + source api.Source + }{id: id, kind: item.kind, source: item.source}) + } + r.mu.RUnlock() + + sort.Slice(items, func(i, j int) bool { + return items[i].id.String() < items[j].id.String() + }) + out := make([]api.SourceInfo, 0, len(items)) + for _, item := range items { + info := item.source.Info() + info.ID = item.id + info.Kind = item.kind + info.Name = item.id.String() + out = append(out, info) + } + return out +} + +func canonicalID(id registry.ID) registry.ID { + return registry.ParseID(id.String()) +} + +var _ api.Registry = (*Registry)(nil) diff --git a/system/cdc/registry_test.go b/system/cdc/registry_test.go new file mode 100644 index 000000000..babe22617 --- /dev/null +++ b/system/cdc/registry_test.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" +) + +type testSource struct { + info api.SourceInfo +} + +func (s *testSource) Info() api.SourceInfo { return s.info } + +func (s *testSource) Subscribe(context.Context, api.StreamOptions) (api.Stream, error) { + return nil, nil +} + +func newTestSource(name string) *testSource { + return &testSource{info: api.SourceInfo{Name: name}} +} + +func TestRegistryCanonicalIDAndDuplicateProtection(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + representation := registry.ID{NS: "app", Name: "events"} + source := newTestSource("wrong-name") + + require.NoError(t, r.Register(id, source, "db.cdc.test")) + got, ok := r.Get(representation) + require.True(t, ok) + require.Same(t, source, got) + require.ErrorIs(t, r.Register(id, newTestSource("other"), "db.cdc.test"), ErrSourceExists) +} + +func TestRegistryReplaceIsAtomicAndReturnsOld(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + old := newTestSource("old") + newSource := newTestSource("new") + require.NoError(t, r.Register(id, old, "db.cdc.old")) + + previous, ok, err := r.Replace(registry.ParseID(id.String()), newSource, "db.cdc.new") + require.NoError(t, err) + require.True(t, ok) + require.Same(t, old, previous) + current, ok := r.Get(id) + require.True(t, ok) + require.Same(t, newSource, current) + + missing := registry.NewID("app", "missing") + _, replaced, err := r.Replace(missing, newTestSource("candidate"), "db.cdc.test") + require.ErrorIs(t, err, ErrSourceMissing) + require.False(t, replaced) + _, exists := r.Get(missing) + require.False(t, exists) +} + +func TestRegistryListIsSortedAndOverlaysIdentity(t *testing.T) { + r := NewRegistry(nil) + require.NoError(t, r.Register(registry.NewID("app", "z"), newTestSource("z"), "db.cdc.z")) + require.NoError(t, r.Register(registry.NewID("app", "a"), newTestSource("a"), "db.cdc.a")) + + infos := r.List() + require.Len(t, infos, 2) + require.Equal(t, "app:a", infos[0].ID.String()) + require.Equal(t, registry.Kind("db.cdc.a"), infos[0].Kind) + require.Equal(t, "app:a", infos[0].Name) + require.Equal(t, "app:z", infos[1].ID.String()) +} + +func TestRegistryConcurrentReplaceAndGet(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + require.NoError(t, r.Register(id, newTestSource("initial"), "db.cdc.test")) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + for j := 0; j < 100; j++ { + candidate := newTestSource("candidate") + _, _, _ = r.Replace(id, candidate, registry.Kind("db.cdc.test")) + _, _ = r.Get(id) + _ = i + } + }(i) + } + wg.Wait() + _, ok := r.Get(id) + require.True(t, ok) +} From d0347bd13434878a77a513b6ea8f0a6cbdda26c6 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:08:41 -0400 Subject: [PATCH 06/47] fix(cdc): bind subscriptions to dispatcher lifecycle --- service/cdc/dispatcher.go | 498 +++++++++++++++++++++++++++++++++ service/cdc/dispatcher_test.go | 323 +++++++++++++++++++++ 2 files changed, 821 insertions(+) create mode 100644 service/cdc/dispatcher.go create mode 100644 service/cdc/dispatcher_test.go diff --git a/service/cdc/dispatcher.go b/service/cdc/dispatcher.go new file mode 100644 index 000000000..99afd1d5a --- /dev/null +++ b/service/cdc/dispatcher.go @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: MPL-2.0 + +// Package cdc provides the command dispatcher shared by all CDC drivers. +package cdc + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + "go.uber.org/zap" +) + +const defaultWorkers = 4 + +var ( + // ErrDispatcherNotStarted is returned to a command submitted before Start. + ErrDispatcherNotStarted = errors.New("cdc dispatcher is not started") + // ErrDispatcherStopping is returned to a command submitted during Stop. + ErrDispatcherStopping = errors.New("cdc dispatcher is stopping") + // ErrDispatcherStarted is returned when Start is called for an active run. + ErrDispatcherStarted = errors.New("cdc dispatcher is already started") + // ErrUnknownCommand identifies a command that was not registered by this dispatcher. + ErrUnknownCommand = errors.New("unknown cdc dispatcher command") + // ErrNoSourceStreamer indicates that CDC sources were not installed in the context. + ErrNoSourceStreamer = errors.New("cdc source streamer not available") + // ErrNilSource indicates a corrupt registry entry that claims to exist but + // does not provide a source implementation. + ErrNilSource = errors.New("cdc registry returned a nil source") + // ErrNoRelayNode indicates that the process has no relay transport. + ErrNoRelayNode = errors.New("cdc relay node not available") +) + +type dispatcherState uint8 + +const ( + stateNew dispatcherState = iota + stateRunning + stateStopping + stateStopped +) + +// Dispatcher routes CDC subscriptions from the process dispatcher to the +// configured source manager. The dispatcher owns subscription relays; a +// driver owns the source and its stream implementation. +type Dispatcher struct { + workers int + log *zap.Logger + + mu sync.Mutex + state dispatcherState + ctx context.Context + cancel context.CancelFunc + jobs chan dispatchJob + sessions map[uint64]*relaySession + nextID uint64 + stopDone chan struct{} + + workersWG sync.WaitGroup + relaysWG sync.WaitGroup +} + +type dispatchJob struct { + ctx context.Context + cmd dispatcher.Command + receiver dispatcher.ResultReceiver + tag uint64 +} + +// DispatcherOption configures a Dispatcher. +type DispatcherOption func(*Dispatcher) + +// WithWorkers sets the number of command workers. Values less than one are +// ignored and leave the default (or previously configured) value unchanged. +func WithWorkers(n int) DispatcherOption { + return func(d *Dispatcher) { + if n > 0 { + d.workers = n + } + } +} + +// WithLogger sets the dispatcher logger. +func WithLogger(log *zap.Logger) DispatcherOption { + return func(d *Dispatcher) { + if log != nil { + d.log = log + } + } +} + +// NewDispatcher creates a CDC dispatcher. +func NewDispatcher(opts ...DispatcherOption) *Dispatcher { + d := &Dispatcher{ + workers: defaultWorkers, + state: stateNew, + sessions: make(map[uint64]*relaySession), + log: zap.NewNop(), + } + for _, opt := range opts { + if opt != nil { + opt(d) + } + } + return d +} + +// Start starts the command workers. A dispatcher can be started again after a +// completed Stop, but cannot be started concurrently with an active run. +func (d *Dispatcher) Start(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + + d.mu.Lock() + if d.state == stateRunning { + d.mu.Unlock() + return ErrDispatcherStarted + } + if d.state == stateStopping { + d.mu.Unlock() + return ErrDispatcherStopping + } + + d.ctx, d.cancel = context.WithCancel(ctx) + d.jobs = make(chan dispatchJob, d.workers*2) + d.sessions = make(map[uint64]*relaySession) + d.stopDone = make(chan struct{}) + d.state = stateRunning + for i := 0; i < d.workers; i++ { + d.workersWG.Add(1) + } + runCtx := d.ctx + d.mu.Unlock() + + for i := 0; i < d.workers; i++ { + go d.worker(runCtx) + } + return nil +} + +// Stop stops workers and all active relays. It is safe to call concurrently +// with Handle and can be called more than once. If ctx expires, cleanup +// continues in the background and a later Stop observes its final result. +func (d *Dispatcher) Stop(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + + d.mu.Lock() + switch d.state { + case stateNew, stateStopped: + d.mu.Unlock() + return nil + case stateStopping: + done := d.stopDone + d.mu.Unlock() + return waitForStop(ctx, done) + case stateRunning: + d.state = stateStopping + done := d.stopDone + cancel := d.cancel + sessions := make([]*relaySession, 0, len(d.sessions)) + for _, session := range d.sessions { + sessions = append(sessions, session) + } + d.mu.Unlock() + + if cancel != nil { + cancel() + } + for _, session := range sessions { + session.stop() + } + go d.finishStop(done) + return waitForStop(ctx, done) + default: + d.mu.Unlock() + return nil + } +} + +func (d *Dispatcher) finishStop(done chan struct{}) { + d.workersWG.Wait() + d.relaysWG.Wait() + + d.mu.Lock() + if d.state == stateStopping && d.stopDone == done { + d.state = stateStopped + d.ctx = nil + d.cancel = nil + d.jobs = nil + close(done) + } + d.mu.Unlock() +} + +func waitForStop(ctx context.Context, done <-chan struct{}) error { + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (d *Dispatcher) worker(ctx context.Context) { + defer d.workersWG.Done() + + for { + select { + case job := <-d.jobs: + if ctx.Err() != nil { + complete(job.receiver, job.tag, nil, ErrDispatcherStopping) + continue + } + d.execute(ctx, job) + case <-ctx.Done(): + d.drainJobs() + return + } + } +} + +// drainJobs completes commands accepted before cancellation. Jobs are never +// dropped silently, which prevents a process yield from remaining pending +// when the dispatcher is stopped. +func (d *Dispatcher) drainJobs() { + for { + select { + case job := <-d.jobs: + complete(job.receiver, job.tag, nil, ErrDispatcherStopping) + default: + return + } + } +} + +func (d *Dispatcher) execute(dispatchCtx context.Context, job dispatchJob) { + switch cmd := job.cmd.(type) { + case cdcapi.SubscribeCmd: + d.executeSubscribe(dispatchCtx, job.ctx, cmd, job.tag, job.receiver) + case *cdcapi.SubscribeCmd: + if cmd == nil { + complete(job.receiver, job.tag, nil, fmt.Errorf("%w: nil subscribe command", ErrUnknownCommand)) + return + } + d.executeSubscribe(dispatchCtx, job.ctx, *cmd, job.tag, job.receiver) + default: + complete(job.receiver, job.tag, nil, fmt.Errorf("%w: %T", ErrUnknownCommand, job.cmd)) + } +} + +func (d *Dispatcher) executeSubscribe(dispatchCtx, requestCtx context.Context, cmd cdcapi.SubscribeCmd, tag uint64, receiver dispatcher.ResultReceiver) { + ctx, cancelContext := linkedContext(dispatchCtx, requestCtx) + node := relay.GetNode(ctx) + if node == nil { + cancelContext() + complete(receiver, tag, nil, ErrNoRelayNode) + return + } + + stream, err := d.openStream(ctx, cmd) + if err != nil { + cancelContext() + if dispatchCtx != nil && dispatchCtx.Err() != nil { + err = ErrDispatcherStopping + } + complete(receiver, tag, nil, err) + return + } + if stream == nil { + cancelContext() + complete(receiver, tag, nil, errors.New("cdc source returned a nil stream")) + return + } + + loopCtx, cancelLoop := context.WithCancel(ctx) + session := &relaySession{ + cancel: func() { + cancelLoop() + cancelContext() + }, + close: stream.Close, + } + if !d.addSession(session) { + session.stop() + complete(receiver, tag, nil, ErrDispatcherStopping) + return + } + + changes := stream.Changes() + go d.relay(loopCtx, session, changes, stream, node, cmd.PID, cmd.Topic, cmd.Source) + + complete(receiver, tag, cdcapi.Subscription{ + Source: cmd.Source, + Topic: cmd.Topic, + Stop: session.stop, + }, nil) +} + +// openStream resolves the canonical system registry first. The legacy +// SourceStreamer path is retained temporarily for callers that predate the +// driver-neutral registry; boot uses the registry path for every driver. +func (d *Dispatcher) openStream(ctx context.Context, cmd cdcapi.SubscribeCmd) (changeStream, error) { + if reg := cdcapi.GetRegistry(ctx); reg != nil { + source, ok := reg.Get(registry.ParseID(cmd.Source)) + if !ok { + return nil, fmt.Errorf("%w: %s", cdcapi.ErrSourceNotFound, cmd.Source) + } + if source == nil { + return nil, fmt.Errorf("%w: %s", ErrNilSource, cmd.Source) + } + return source.Subscribe(ctx, cmd.Options) + } + + streamer := cdcapi.GetSourceStreamer(ctx) + if streamer == nil { + return nil, ErrNoSourceStreamer + } + stream, _, err := streamer.Stream(ctx, cmd.Source, cmd.Options) + return stream, err +} + +// linkedContext preserves the request context's values (including relay +// routing) while making dispatcher shutdown a second cancellation parent. The +// returned cleanup must be held by the relay session until the stream ends. +func linkedContext(dispatchCtx, requestCtx context.Context) (context.Context, context.CancelFunc) { + if requestCtx == nil { + requestCtx = context.Background() + } + if dispatchCtx == nil { + return context.WithCancel(requestCtx) + } + ctx, cancel := context.WithCancel(requestCtx) + stopPropagation := context.AfterFunc(dispatchCtx, cancel) + return ctx, func() { + stopPropagation() + cancel() + } +} + +func (d *Dispatcher) addSession(session *relaySession) bool { + d.mu.Lock() + defer d.mu.Unlock() + + if d.state != stateRunning { + return false + } + d.nextID++ + session.id = d.nextID + d.sessions[session.id] = session + d.relaysWG.Add(1) + return true +} + +func (d *Dispatcher) relay(ctx context.Context, session *relaySession, changes <-chan cdcapi.Change, stream changeStream, node relay.Node, target pid.PID, topic, source string) { + defer func() { + // Closing a naturally exhausted stream is still the dispatcher's + // ownership responsibility. stop is idempotent and suppresses any + // duplicate terminal caused by the close. + session.stop() + d.relayDone(session.id) + d.relaysWG.Done() + }() + + for { + select { + case change, ok := <-changes: + if !ok { + if err := streamError(stream); err != nil { + d.sendTerminal(node, target, topic, err) + } else { + d.sendTerminal(node, target, topic, nil) + } + return + } + + pkg := relay.NewPackage(pid.Zero(), target, topic, payload.New(change)) + if err := node.Send(pkg); err != nil { + d.log.Debug("failed to relay cdc change", + zap.String("source", source), + zap.Error(err)) + // A failed relay cannot make progress. Close the source stream + // and cancel this relay so it cannot retain a worker or source. + session.stop() + return + } + case <-ctx.Done(): + return + } + } +} + +func (d *Dispatcher) relayDone(id uint64) { + d.mu.Lock() + delete(d.sessions, id) + d.mu.Unlock() +} + +func (d *Dispatcher) sendTerminal(node relay.Node, target pid.PID, topic string, err error) { + var terminal payload.Payloads + if err != nil { + terminal = append(terminal, payload.NewError(err)) + } + terminal = append(terminal, payload.NewTerminal()) + pkg := relay.NewPackage(pid.Zero(), target, topic, terminal...) + if sendErr := node.Send(pkg); sendErr != nil { + d.log.Debug("failed to send cdc terminal", + zap.String("topic", topic), + zap.Error(sendErr)) + } +} + +// streamError is an optional extension implemented by streams that can +// report a typed terminal error after their change channel closes. Keeping it +// optional preserves compatibility with the original stream interface while +// allowing all drivers to expose terminal failures consistently. +func streamError(stream changeStream) error { + if s, ok := stream.(interface{ Err() error }); ok { + return s.Err() + } + return nil +} + +type changeStream interface { + Changes() <-chan cdcapi.Change + Close() +} + +func complete(receiver dispatcher.ResultReceiver, tag uint64, data any, err error) { + if receiver != nil { + receiver.CompleteYield(tag, data, err) + } +} + +type relaySession struct { + id uint64 + cancel context.CancelFunc + close func() + once sync.Once +} + +func (s *relaySession) stop() { + s.once.Do(func() { + if s.cancel != nil { + s.cancel() + } + if s.close != nil { + s.close() + } + }) +} + +// Handle queues a command for execution by the dispatcher worker pool. +func (d *Dispatcher) Handle(ctx context.Context, cmd dispatcher.Command, tag uint64, receiver dispatcher.ResultReceiver) error { + if ctx == nil { + ctx = context.Background() + } + + d.mu.Lock() + if d.state != stateRunning { + err := ErrDispatcherNotStarted + if d.state == stateStopping { + err = ErrDispatcherStopping + } + d.mu.Unlock() + complete(receiver, tag, nil, err) + return nil + } + jobs := d.jobs + runCtx := d.ctx + d.mu.Unlock() + + job := dispatchJob{ctx: ctx, cmd: cmd, tag: tag, receiver: receiver} + select { + case jobs <- job: + case <-runCtx.Done(): + complete(receiver, tag, nil, ErrDispatcherStopping) + case <-ctx.Done(): + complete(receiver, tag, nil, ctx.Err()) + } + return nil +} + +// RegisterAll registers all CDC command handlers with the process dispatcher. +func (d *Dispatcher) RegisterAll(register func(id dispatcher.CommandID, h dispatcher.Handler)) { + if register != nil { + register(cdcapi.Subscribe, dispatcher.HandlerFunc(d.Handle)) + } +} diff --git a/service/cdc/dispatcher_test.go b/service/cdc/dispatcher_test.go new file mode 100644 index 000000000..e14eacd27 --- /dev/null +++ b/service/cdc/dispatcher_test.go @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + cdcsystem "github.com/wippyai/runtime/system/cdc" +) + +type dispatcherTestStream struct { + changes chan cdcapi.Change + err error + closed atomic.Bool + once sync.Once +} + +func (s *dispatcherTestStream) Changes() <-chan cdcapi.Change { return s.changes } + +func (s *dispatcherTestStream) Close() { + s.once.Do(func() { + s.closed.Store(true) + }) +} + +func (s *dispatcherTestStream) Err() error { return s.err } + +type dispatcherTestSource struct { + stream *dispatcherTestStream + info cdcapi.SourceInfo +} + +func (s *dispatcherTestSource) Info() cdcapi.SourceInfo { return s.info } + +func (s *dispatcherTestSource) Subscribe(context.Context, cdcapi.StreamOptions) (cdcapi.Stream, error) { + return s.stream, nil +} + +type blockingSubscribeSource struct { + started chan struct{} + once sync.Once +} + +func (s *blockingSubscribeSource) Info() cdcapi.SourceInfo { + return cdcapi.SourceInfo{ID: registry.NewID("test", "blocking")} +} + +func (s *blockingSubscribeSource) Subscribe(ctx context.Context, _ cdcapi.StreamOptions) (cdcapi.Stream, error) { + s.once.Do(func() { close(s.started) }) + <-ctx.Done() + return nil, ctx.Err() +} + +type nilSourceRegistry struct{} + +func (nilSourceRegistry) List() []cdcapi.SourceInfo { return nil } + +func (nilSourceRegistry) Get(registry.ID) (cdcapi.Source, bool) { return nil, true } + +type dispatcherTestNode struct { + packages chan *relay.Package + send func(*relay.Package) error +} + +func (n *dispatcherTestNode) ID() pid.NodeID { return "cdc-dispatcher-test" } + +func (n *dispatcherTestNode) Send(pkg *relay.Package) error { + if n.send != nil { + return n.send(pkg) + } + n.packages <- pkg + return nil +} + +func (n *dispatcherTestNode) RegisterHost(pid.HostID, relay.Receiver) error { return nil } +func (n *dispatcherTestNode) UnregisterHost(pid.HostID) {} +func (n *dispatcherTestNode) GetHost(pid.HostID) (relay.Receiver, bool) { return nil, false } +func (n *dispatcherTestNode) Attach(pid.PID, chan *relay.Package) (context.CancelFunc, error) { + return func() {}, nil +} +func (n *dispatcherTestNode) Detach(pid.PID) {} + +type dispatcherTestReceiver struct { + done chan struct{} + once sync.Once + data any + err error +} + +func (r *dispatcherTestReceiver) CompleteYield(_ uint64, data any, err error) { + r.data = data + r.err = err + r.once.Do(func() { close(r.done) }) +} + +func dispatcherTestContext(t *testing.T, source cdcapi.Source, id registry.ID, node relay.Node) context.Context { + t.Helper() + reg := cdcsystem.NewRegistry(nil) + require.NoError(t, reg.Register(id, source, cdcapi.SQLite)) + return dispatcherTestContextWithRegistry(reg, node) +} + +func dispatcherTestContextWithRegistry(reg cdcapi.Registry, node relay.Node) context.Context { + root := ctxapi.NewRootContext() + root = cdcapi.WithRegistry(root, reg) + root = relay.WithNode(root, node) + ctx, _ := ctxapi.OpenFrameContext(root) + return ctx +} + +func dispatcherTestCommand(id registry.ID) cdcapi.SubscribeCmd { + target := pid.PID{Host: "test", UniqID: "cdc"} + return cdcapi.SubscribeCmd{ + PID: target.Precomputed(), + Source: id.String(), + Topic: "cdc@test", + } +} + +func waitResult(t *testing.T, receiver *dispatcherTestReceiver) { + t.Helper() + select { + case <-receiver.done: + case <-time.After(time.Second): + t.Fatal("timed out waiting for dispatcher result") + } +} + +func TestDispatcherUsesSystemRegistryAndRelaysChanges(t *testing.T) { + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change, 1)} + id := registry.NewID("test", "source") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 2)} + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + sub, ok := receiver.data.(cdcapi.Subscription) + require.True(t, ok) + require.NotNil(t, sub.Stop) + + stream.changes <- cdcapi.Change{SourceID: id, Source: id.String(), Op: "insert"} + select { + case pkg := <-node.packages: + require.Len(t, pkg.Messages, 1) + require.Len(t, pkg.Messages[0].Payloads, 1) + got, ok := pkg.Messages[0].Payloads[0].Data().(cdcapi.Change) + require.True(t, ok) + assert.Equal(t, "insert", got.Op) + case <-time.After(time.Second): + t.Fatal("timed out waiting for relayed change") + } + + sub.Stop() + assert.Eventually(t, stream.closed.Load, time.Second, 10*time.Millisecond) +} + +func TestDispatcherRelaysTypedStreamErrorBeforeTerminal(t *testing.T) { + streamErr := errors.New("capture gap") + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change), err: streamErr} + id := registry.NewID("test", "source") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 2)} + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + close(stream.changes) + select { + case pkg := <-node.packages: + require.Len(t, pkg.Messages, 1) + payloads := pkg.Messages[0].Payloads + require.Len(t, payloads, 2) + gotErr, ok := payloads[0].Data().(error) + require.True(t, ok) + assert.ErrorIs(t, gotErr, streamErr) + assert.True(t, payload.IsTerminal(payloads[1])) + case <-time.After(time.Second): + t.Fatal("timed out waiting for typed terminal") + } +} + +func TestDispatcherStopsActiveRelayAndWaitsForCleanup(t *testing.T) { + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change)} + id := registry.NewID("test", "source") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, d.Stop(stopCtx)) + assert.True(t, stream.closed.Load()) + + postStop := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 2, postStop)) + waitResult(t, postStop) + assert.ErrorIs(t, postStop.err, ErrDispatcherNotStarted) +} + +func TestDispatcherCancelsRelayAfterNodeFailure(t *testing.T) { + relayErr := errors.New("node unavailable") + var sends atomic.Int32 + node := &dispatcherTestNode{send: func(*relay.Package) error { + sends.Add(1) + return relayErr + }} + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change, 1)} + id := registry.NewID("test", "source") + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + stream.changes <- cdcapi.Change{Source: id.String(), Op: "insert"} + assert.Eventually(t, stream.closed.Load, time.Second, 10*time.Millisecond) + assert.Equal(t, int32(1), sends.Load()) +} + +func TestDispatcherRejectsNilRegistrySource(t *testing.T) { + id := registry.NewID("test", "nil") + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContextWithRegistry(nilSourceRegistry{}, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + assert.ErrorIs(t, receiver.err, ErrNilSource) +} + +func TestDispatcherStopCancelsBlockingSubscribe(t *testing.T) { + source := &blockingSubscribeSource{started: make(chan struct{})} + id := registry.NewID("test", "blocking") + reg := cdcsystem.NewRegistry(nil) + require.NoError(t, reg.Register(id, source, cdcapi.SQLite)) + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContextWithRegistry(reg, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + select { + case <-source.started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for blocking subscription") + } + + stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, d.Stop(stopCtx)) + waitResult(t, receiver) + assert.ErrorIs(t, receiver.err, ErrDispatcherStopping) +} + +func TestDispatcherHandleAndStopAreSafeConcurrently(t *testing.T) { + root := ctxapi.NewRootContext() + ctx, _ := ctxapi.OpenFrameContext(root) + d := NewDispatcher(WithWorkers(2)) + require.NoError(t, d.Start(ctx)) + require.ErrorIs(t, d.Start(ctx), ErrDispatcherStarted) + + const commands = 32 + receivers := make([]*dispatcherTestReceiver, commands) + var wg sync.WaitGroup + for i := 0; i < commands; i++ { + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + receivers[i] = receiver + wg.Add(1) + go func(tag uint64, receiver *dispatcherTestReceiver) { + defer wg.Done() + _ = d.Handle(ctx, dispatcherTestCommand(registry.NewID("test", "source")), tag, receiver) + }(uint64(i), receiver) + } + + stopDone := make(chan error, 1) + go func() { stopDone <- d.Stop(context.Background()) }() + wg.Wait() + require.NoError(t, <-stopDone) + for _, receiver := range receivers { + waitResult(t, receiver) + } +} From 6cf614c10b0c6bb8f9e196f4ac02b021cffaee29 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:10:41 -0400 Subject: [PATCH 07/47] fix(lua): harden CDC stream options --- runtime/lua/modules/cdc/module.go | 248 +++++++++++++++++++---- runtime/lua/modules/cdc/module_test.go | 259 +++++++++++++++++++++++-- runtime/lua/modules/cdc/types.go | 74 ++++++- runtime/lua/modules/cdc/yields.go | 23 ++- 4 files changed, 551 insertions(+), 53 deletions(-) diff --git a/runtime/lua/modules/cdc/module.go b/runtime/lua/modules/cdc/module.go index 892e7a240..d2c1a8e36 100644 --- a/runtime/lua/modules/cdc/module.go +++ b/runtime/lua/modules/cdc/module.go @@ -4,10 +4,13 @@ package cdc import ( "fmt" + "math" + "strings" "sync" "sync/atomic" lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/runtime" luaapi "github.com/wippyai/runtime/api/runtime/lua" cdcapi "github.com/wippyai/runtime/api/service/cdc" @@ -15,13 +18,22 @@ import ( "github.com/wippyai/runtime/runtime/lua/engine/value" ) -const cdcStreamTypeName = "cdc.Stream" +const ( + cdcStreamTypeName = "cdc.Stream" + + // Keep Lua-side option allocations bounded independently of the source + // implementation. The source may apply a smaller limit, but this upper + // bound prevents a malformed table or direct internal option from causing + // an unbounded map/slice/channel allocation in the Lua adapter. + defaultStreamBuffer = 64 + maxStreamItems = 65536 +) var subscriptionCounter uint64 var Module = &luaapi.ModuleDef{ Name: "cdc", - Description: "Postgres CDC source streams", + Description: "Driver-neutral CDC source streams", Class: []string{luaapi.ClassStorage, luaapi.ClassNondeterministic}, Build: func() (*lua.LTable, []luaapi.YieldType) { value.RegisterTypeMethods(nil, cdcStreamTypeName, nil, streamMethods) @@ -67,16 +79,21 @@ func listSources(l *lua.LState) int { return 2 } - inspector := cdcapi.GetSourceInspector(ctx) - if inspector == nil { - l.Push(lua.LNil) - l.Push(lua.NewLuaError(l, "cdc source inspector not found"). - WithKind(lua.Internal). - WithRetryable(false)) - return 2 + var infos []cdcapi.SourceInfo + if registry := cdcapi.GetRegistry(ctx); registry != nil { + infos = registry.List() + } else { + inspector := cdcapi.GetSourceInspector(ctx) + if inspector == nil { + l.Push(lua.LNil) + l.Push(lua.NewLuaError(l, "cdc source inspector not found"). + WithKind(lua.Internal). + WithRetryable(false)) + return 2 + } + infos = inspector.List() } - infos := inspector.List() result := l.CreateTable(len(infos), 0) for i, info := range infos { result.RawSetInt(i+1, sourceInfoToTable(l, info)) @@ -105,8 +122,26 @@ func getSource(l *lua.LState) int { return 2 } - inspector := cdcapi.GetSourceInspector(ctx) - if inspector == nil { + var ( + info cdcapi.SourceInfo + ok bool + ) + if cdcRegistry := cdcapi.GetRegistry(ctx); cdcRegistry != nil { + id := registry.ParseID(name) + source, found := cdcRegistry.Get(id) + if found && source != nil { + info = source.Info() + if info.ID.NS == "" && info.ID.Name == "" { + info.ID = id + } + if info.Name == "" { + info.Name = info.ID.String() + } + ok = true + } + } else if inspector := cdcapi.GetSourceInspector(ctx); inspector != nil { + info, ok = inspector.Get(name) + } else { l.Push(lua.LNil) l.Push(lua.NewLuaError(l, "cdc source inspector not found"). WithKind(lua.Internal). @@ -114,7 +149,6 @@ func getSource(l *lua.LState) int { return 2 } - info, ok := inspector.Get(name) if !ok { l.Push(lua.LNil) l.Push(lua.LNil) @@ -151,7 +185,7 @@ func openStream(l *lua.LState) int { return 2 } - ch := engine.NewChannel(64) + ch := engine.NewChannel(streamBufferCapacity(opts.Buffer)) engine.PushChannel(l, ch) l.Pop(1) @@ -264,7 +298,33 @@ func (s *Stream) closeWithUnsubscribe(unsubscribe bool) { } func sourceInfoToTable(l *lua.LState, info cdcapi.SourceInfo) *lua.LTable { - t := l.CreateTable(0, 12) + t := l.CreateTable(0, 24) + if !isZeroRegistryID(info.ID) { + t.RawSetString("id", lua.LString(registryIDString(info.ID))) + } + if info.Kind != "" { + t.RawSetString("kind", lua.LString(info.Kind)) + } + state := string(info.State) + if state == "" { + state = string(cdcapi.SourceStateUnknown) + } + t.RawSetString("state", lua.LString(state)) + if info.Generation != "" { + t.RawSetString("generation", lua.LString(info.Generation)) + } + + capabilities := l.CreateTable(0, 6) + capabilities.RawSetString("snapshot", lua.LBool(info.Capabilities.Snapshot)) + capabilities.RawSetString("durable", lua.LBool(info.Capabilities.Durable)) + capabilities.RawSetString("replayable", lua.LBool(info.Capabilities.Replayable)) + capabilities.RawSetString("captures_external_writes", lua.LBool(info.Capabilities.CapturesExternalWrites)) + capabilities.RawSetString("before_images", lua.LBool(info.Capabilities.BeforeImages)) + capabilities.RawSetString("coalesced", lua.LBool(info.Capabilities.Coalesced)) + t.RawSetString("capabilities", capabilities) + + // Keep the legacy identity fields present with their historical defaults; + // newer callers should use id/kind/state above. t.RawSetString("name", lua.LString(info.Name)) t.RawSetString("slot", lua.LString(info.Slot)) if info.Engine != "" { @@ -312,44 +372,164 @@ func streamOptionsFromLua(l *lua.LState, idx int) (cdcapi.StreamOptions, *lua.Er WithRetryable(false) } - opts.Tables = stringArrayField(table, "tables") - opts.Ops = stringArrayField(table, "ops") + if errMsg := validateOptionKeys(table); errMsg != "" { + return opts, invalidStreamOption(l, errMsg) + } + + var errMsg string + if opts.Tables, errMsg = stringArrayField(table, "tables"); errMsg != "" { + return opts, invalidStreamOption(l, errMsg) + } + if opts.Ops, errMsg = stringArrayField(table, "ops"); errMsg != "" { + return opts, invalidStreamOption(l, errMsg) + } if v := table.RawGetString("buffer"); v != lua.LNil { if v.Type() != lua.LTNumber && v.Type() != lua.LTInteger { - return opts, lua.NewLuaError(l, "buffer must be a number"). - WithKind(lua.Invalid). - WithRetryable(false) + return opts, invalidStreamOption(l, "buffer must be a number") } - n := int(lua.LVAsNumber(v)) - if n <= 0 || n > 65536 { - return opts, lua.NewLuaError(l, "buffer must be between 1 and 65536"). - WithKind(lua.Invalid). - WithRetryable(false) + number := lua.LVAsNumber(v) + if math.IsNaN(float64(number)) || math.IsInf(float64(number), 0) || + math.Trunc(float64(number)) != float64(number) || + number < 1 || number > lua.LNumber(maxStreamItems) { + return opts, invalidStreamOption(l, "buffer must be a positive integer") } + n := int(number) opts.Buffer = n } if v := table.RawGetString("snapshot"); v != lua.LNil { + if v.Type() != lua.LTBool { + return opts, invalidStreamOption(l, "snapshot must be a boolean") + } opts.Snapshot = lua.LVAsBool(v) } + if v := table.RawGetString("after"); v != lua.LNil { + if v.Type() != lua.LTString { + return opts, invalidStreamOption(l, "after must be a string") + } + after := string(v.(lua.LString)) + if strings.TrimSpace(after) == "" { + return opts, invalidStreamOption(l, "after must not be empty") + } + opts.After = after + } return opts, nil } -func stringArrayField(table *lua.LTable, field string) []string { +func streamBufferCapacity(buffer int) int { + if buffer <= 0 { + return defaultStreamBuffer + } + if buffer > maxStreamItems { + return maxStreamItems + } + return buffer +} + +func invalidStreamOption(l *lua.LState, message string) *lua.Error { + return lua.NewLuaError(l, message). + WithKind(lua.Invalid). + WithRetryable(false) +} + +func validateOptionKeys(table *lua.LTable) string { + var errMsg string + table.ForEach(func(key, _ lua.LValue) { + if errMsg != "" { + return + } + name, ok := key.(lua.LString) + if !ok { + errMsg = "stream options contains unknown or non-string field" + return + } + switch string(name) { + case "tables", "ops", "buffer", "snapshot", "after": + default: + errMsg = "stream options contains unknown field: " + string(name) + } + }) + return errMsg +} + +func stringArrayField(table *lua.LTable, field string) ([]string, string) { v := table.RawGetString(field) if v == lua.LNil { - return nil + return nil, "" } t, ok := v.(*lua.LTable) if !ok { - return nil - } - out := make([]string, 0, t.Len()) - t.ForEach(func(_, value lua.LValue) { - if value.Type() == lua.LTString { - out = append(out, value.String()) + return nil, field + " must be an array of strings" + } + count := t.Len() + if count > maxStreamItems { + return nil, field + " must contain at most 65536 entries" + } + values := make(map[int]string, count) + max := 0 + var errMsg string + t.ForEach(func(key, value lua.LValue) { + if errMsg != "" { + return + } + var position int + switch key.Type() { + case lua.LTInteger: + index, ok := key.(lua.LInteger) + if !ok || index <= 0 || index > lua.LInteger(maxStreamItems) { + errMsg = field + " must be an array of strings" + return + } + position = int(index) + case lua.LTNumber: + number := lua.LVAsNumber(key) + if math.IsNaN(float64(number)) || math.IsInf(float64(number), 0) || + math.Trunc(float64(number)) != float64(number) || number <= 0 || + number > lua.LNumber(maxStreamItems) { + errMsg = field + " must be an array of strings" + return + } + position = int(number) + if position <= 0 { + errMsg = field + " must be an array of strings" + return + } + default: + errMsg = field + " must be an array of strings" + return + } + if value.Type() != lua.LTString || strings.TrimSpace(value.String()) == "" { + errMsg = field + " must contain non-empty strings" + return + } + if _, exists := values[position]; !exists && len(values) >= maxStreamItems { + errMsg = field + " must contain at most 65536 entries" + return + } + values[position] = string(value.(lua.LString)) + if position > max { + max = position } }) - return out + if errMsg != "" { + return nil, errMsg + } + out := make([]string, max) + for i := 1; i <= max; i++ { + value, ok := values[i] + if !ok { + return nil, field + " must be a contiguous array of strings" + } + out[i-1] = value + } + return out, "" +} + +func isZeroRegistryID(id registry.ID) bool { + return id.NS == "" && id.Name == "" +} + +func registryIDString(id registry.ID) string { + return id.String() } func markSubscribed(stream *Stream, proc *engine.Process, topic string, cancelCleanup func()) error { diff --git a/runtime/lua/modules/cdc/module_test.go b/runtime/lua/modules/cdc/module_test.go index a645f8a7d..07b0a7deb 100644 --- a/runtime/lua/modules/cdc/module_test.go +++ b/runtime/lua/modules/cdc/module_test.go @@ -4,12 +4,16 @@ package cdc import ( "context" + "errors" "testing" "github.com/stretchr/testify/require" lua "github.com/wippyai/go-lua" + "github.com/wippyai/go-lua/types/typ" + ctxapi "github.com/wippyai/runtime/api/context" "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/registry" cdcapi "github.com/wippyai/runtime/api/service/cdc" ) @@ -43,6 +47,45 @@ func newStateWithInspector(t *testing.T, inspector cdcapi.SourceInspector) *lua. return l } +type fakeSource struct { + info cdcapi.SourceInfo +} + +func (f *fakeSource) Info() cdcapi.SourceInfo { return f.info } + +func (f *fakeSource) Subscribe(context.Context, cdcapi.StreamOptions) (cdcapi.Stream, error) { + return nil, nil +} + +type fakeRegistry struct { + all []cdcapi.SourceInfo + source cdcapi.Source +} + +func (f *fakeRegistry) List() []cdcapi.SourceInfo { return f.all } + +func (f *fakeRegistry) Get(id registry.ID) (cdcapi.Source, bool) { + if f.source == nil { + return nil, false + } + info := f.source.Info() + return f.source, !isZeroRegistryID(info.ID) && registryIDString(info.ID) == registryIDString(id) +} + +func newStateWithRegistry(t *testing.T, registry cdcapi.Registry) *lua.LState { + t.Helper() + l := lua.NewState() + t.Cleanup(l.Close) + + ctx := ctxapi.NewRootContext() + ctx = cdcapi.WithRegistry(ctx, registry) + l.SetContext(ctx) + + tbl, _ := Module.Build() + l.SetGlobal(Module.Name, tbl) + return l +} + func TestModuleLoad(t *testing.T) { l := lua.NewState() defer l.Close() @@ -60,8 +103,34 @@ func TestModuleLoad(t *testing.T) { func TestListSourcesReturnsAllInfos(t *testing.T) { l := newStateWithInspector(t, &fakeInspector{ all: []cdcapi.SourceInfo{ - {Name: "id-a", Slot: "slot_a", Publication: "pub_a", Streaming: true}, - {Name: "id-b", Slot: "slot_b", Tables: []string{"public.t"}, Failover: true}, + { + ID: registry.NewID("test", "id-a"), + Kind: "db.cdc.postgres", + State: cdcapi.SourceStateRunning, + Generation: "generation-a", + Capabilities: cdcapi.Capabilities{ + Snapshot: true, + Durable: true, + Replayable: true, + CapturesExternalWrites: true, + BeforeImages: true, + }, + Name: "id-a", + Slot: "slot_a", + Publication: "pub_a", + Streaming: true, + }, + { + ID: registry.NewID("test", "id-b"), + Kind: "db.cdc.sqlite", + State: cdcapi.SourceStateFaulted, + Capabilities: cdcapi.Capabilities{BeforeImages: true, Coalesced: true}, + Name: "id-b", + Slot: "slot_b", + Tables: []string{"public.t"}, + Failover: true, + Faulted: true, + }, }, }) @@ -69,6 +138,15 @@ func TestListSourcesReturnsAllInfos(t *testing.T) { local rows, err = cdc.list_sources() assert(err == nil, "unexpected error: " .. tostring(err)) assert(#rows == 2, "expected 2 rows, got " .. tostring(#rows)) + assert(rows[1].id == "test:id-a") + assert(rows[1].kind == "db.cdc.postgres") + assert(rows[1].state == "running") + assert(rows[1].generation == "generation-a") + assert(rows[1].capabilities.snapshot == true) + assert(rows[1].capabilities.durable == true) + assert(rows[1].capabilities.replayable == true) + assert(rows[1].capabilities.captures_external_writes == true) + assert(rows[1].capabilities.before_images == true) assert(rows[1].slot == "slot_a") assert(rows[1].publication == "pub_a") assert(rows[1].tables == nil, "row with no tables should omit the tables key") @@ -78,6 +156,27 @@ func TestListSourcesReturnsAllInfos(t *testing.T) { assert(rows[2].tables[1] == "public.t") assert(#rows[2].tables == 1) assert(rows[2].failover == true) + assert(rows[2].capabilities.coalesced == true) + `)) +} + +func TestListSourcesUsesDriverNeutralRegistry(t *testing.T) { + id := registry.NewID("test", "registry-source") + l := newStateWithRegistry(t, &fakeRegistry{ + all: []cdcapi.SourceInfo{{ + ID: id, + Kind: "db.cdc.sqlite", + State: cdcapi.SourceStateRunning, + Name: id.String(), + }}, + }) + + require.NoError(t, l.DoString(` + local rows, err = cdc.list_sources() + assert(err == nil, "unexpected error: " .. tostring(err)) + assert(#rows == 1) + assert(rows[1].id == "test:registry-source") + assert(rows[1].kind == "db.cdc.sqlite") `)) } @@ -99,6 +198,25 @@ func TestSourceByName(t *testing.T) { `)) } +func TestSourceByRegistryID(t *testing.T) { + id := registry.NewID("test", "source") + source := &fakeSource{info: cdcapi.SourceInfo{ + ID: id, + Kind: "db.cdc.sqlite", + State: cdcapi.SourceStateRunning, + Name: id.String(), + }} + l := newStateWithRegistry(t, &fakeRegistry{source: source}) + + require.NoError(t, l.DoString(` + local info, err = cdc.source("test:source") + assert(err == nil, "unexpected error: " .. tostring(err)) + assert(info ~= nil) + assert(info.id == "test:source") + assert(info.kind == "db.cdc.sqlite") + `)) +} + func TestSourceRequiresName(t *testing.T) { l := newStateWithInspector(t, &fakeInspector{}) @@ -117,6 +235,8 @@ func TestStreamOpenAndRelease(t *testing.T) { tables = {"public.accounts"}, ops = {"insert", "update"}, buffer = 4, + snapshot = true, + after = "cursor-1", }) assert(err == nil, "unexpected error: " .. tostring(err)) assert(stream ~= nil) @@ -146,30 +266,106 @@ func TestStreamRejectsInvalidBuffer(t *testing.T) { `)) } +func TestStreamRejectsMalformedOptions(t *testing.T) { + cases := map[string]string{ + "tables type": `{ tables = "accounts" }`, + "tables element": `{ tables = { 1 } }`, + "ops element": `{ ops = { "insert", 2 } }`, + "fractional buffer": `{ buffer = 1.5 }`, + "zero buffer": `{ buffer = 0 }`, + "oversized buffer": `{ buffer = 65537 }`, + "snapshot type": `{ snapshot = "true" }`, + "after type": `{ after = 42 }`, + "empty after": `{ after = "" }`, + "whitespace after": `{ after = " \t\n" }`, + "unknown field": `{ unsupported = true }`, + "numeric field": `{ [1] = "unsupported" }`, + } + for name, options := range cases { + t.Run(name, func(t *testing.T) { + l := newStateWithInspector(t, &fakeInspector{}) + script := ` + local stream, err = cdc.stream("source", ` + options + `) + assert(stream == nil) + assert(err ~= nil) + ` + require.NoError(t, l.DoString(script)) + }) + } +} + +func TestStringArrayFieldRejectsOutOfRangeIndex(t *testing.T) { + l := lua.NewState() + defer l.Close() + + values := l.CreateTable(0, 1) + values.RawSetInt(maxStreamItems+1, lua.LString("out-of-range")) + options := l.CreateTable(0, 1) + options.RawSetString("tables", values) + _, errMsg := stringArrayField(options, "tables") + require.NotEmpty(t, errMsg) +} + +func TestStreamBufferCapacityIsBounded(t *testing.T) { + for _, test := range []struct { + name string + input int + wanted int + }{ + {name: "default", input: 0, wanted: defaultStreamBuffer}, + {name: "negative uses default", input: -1, wanted: defaultStreamBuffer}, + {name: "configured", input: 128, wanted: 128}, + {name: "maximum", input: maxStreamItems, wanted: maxStreamItems}, + {name: "internal overflow is capped", input: maxStreamItems + 1, wanted: maxStreamItems}, + } { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.wanted, streamBufferCapacity(test.input)) + }) + } +} + +func TestModuleTypesUseIntegerBufferAndTypedChannel(t *testing.T) { + manifest := ModuleTypes() + streamOptions, ok := manifest.LookupType("StreamOptions") + require.True(t, ok) + optionsRecord, ok := streamOptions.(*typ.Record) + require.True(t, ok) + require.Equal(t, typ.Integer, optionsRecord.GetField("buffer").Type) + require.NotEqual(t, typ.Any, cdcChannelType) +} + func TestChangeHandlerUsesCanonicalLuaKeys(t *testing.T) { l := lua.NewState() defer l.Close() got := cdcChangeHandler(context.Background(), l, pid.Zero(), "cdc@1", []payload.Payload{ payload.New(cdcapi.Change{ - Source: "test:cdc", - Op: "insert", - Schema: "public", - Table: "accounts", - Relation: "public.accounts", - LSN: "0/16B6C50", - CommitLSN: "0/16B6C98", - XID: 42, - After: map[string]any{"email": "a@w.ai"}, + SourceID: registry.NewID("test", "cdc"), + Source: "test:cdc", + Op: "insert", + Schema: "public", + Table: "accounts", + Relation: "public.accounts", + LSN: "0/16B6C50", + CommitLSN: "0/16B6C98", + Cursor: "cursor-1", + Generation: "generation-1", + Transaction: "transaction-1", + XID: 42, + After: map[string]any{"email": "a@w.ai"}, }), }) tbl, ok := got.(*lua.LTable) require.True(t, ok) require.Equal(t, "test:cdc", tbl.RawGetString("source").String()) + require.Equal(t, "test:cdc", tbl.RawGetString("source_id").String()) require.Equal(t, "insert", tbl.RawGetString("op").String()) require.Equal(t, "public.accounts", tbl.RawGetString("relation").String()) require.Equal(t, "0/16B6C98", tbl.RawGetString("commit_lsn").String()) + require.Equal(t, "cursor-1", tbl.RawGetString("cursor").String()) + require.Equal(t, "generation-1", tbl.RawGetString("generation").String()) + require.Equal(t, "transaction-1", tbl.RawGetString("transaction").String()) require.Equal(t, lua.LNil, tbl.RawGetString("commit_lsn,omitempty")) require.Equal(t, lua.LNil, tbl.RawGetString("after,omitempty")) @@ -178,6 +374,47 @@ func TestChangeHandlerUsesCanonicalLuaKeys(t *testing.T) { require.Equal(t, "a@w.ai", after.RawGetString("email").String()) } +func TestChangeHandlerConvertsTypedStreamError(t *testing.T) { + l := lua.NewState() + defer l.Close() + + got := cdcChangeHandler(context.Background(), l, pid.Zero(), "cdc@1", []payload.Payload{ + payload.NewError(errors.New("capture gap")), + }) + streamErr, ok := lua.AsError(got) + require.True(t, ok) + require.Contains(t, streamErr.Error(), "capture gap") +} + +func TestModuleTypesMatchRuntimeFields(t *testing.T) { + manifest := ModuleTypes() + + assertRecordFields := func(name string, want []string) { + t.Helper() + value, ok := manifest.LookupType(name) + require.True(t, ok, "%s type is not defined", name) + record, ok := value.(*typ.Record) + require.True(t, ok, "%s is %T, want record", name, value) + for _, field := range want { + require.NotNil(t, record.GetField(field), "%s.%s is missing", name, field) + } + } + + assertRecordFields("Capabilities", []string{ + "snapshot", "durable", "replayable", "captures_external_writes", "before_images", "coalesced", + }) + assertRecordFields("SourceInfo", []string{ + "id", "kind", "state", "generation", "capabilities", "name", "slot", "publication", + "engine", "file", "db_resource", "epoch", "error", "tables", "streaming", "failover", + "temporary", "snapshot", "faulted", + }) + assertRecordFields("StreamOptions", []string{"tables", "ops", "buffer", "snapshot", "after"}) + assertRecordFields("Change", []string{ + "source_id", "source", "op", "schema", "table", "relation", "lsn", "commit_lsn", "cursor", + "generation", "transaction", "error", "xid", "before", "after", + }) +} + func TestListSourcesFailsWithoutInspector(t *testing.T) { l := lua.NewState() defer l.Close() diff --git a/runtime/lua/modules/cdc/types.go b/runtime/lua/modules/cdc/types.go index bffd78314..adee8383f 100644 --- a/runtime/lua/modules/cdc/types.go +++ b/runtime/lua/modules/cdc/types.go @@ -5,34 +5,94 @@ package cdc import ( "github.com/wippyai/go-lua/types/io" "github.com/wippyai/go-lua/types/typ" + "github.com/wippyai/runtime/runtime/lua/engine" ) +var cdcChannelType typ.Type + +var sourceCapabilitiesType = typ.NewRecord(). + Field("snapshot", typ.Boolean). + Field("durable", typ.Boolean). + Field("replayable", typ.Boolean). + Field("captures_external_writes", typ.Boolean). + Field("before_images", typ.Boolean). + Field("coalesced", typ.Boolean). + Build() + var sourceInfoType = typ.NewRecord(). + OptField("id", typ.String). + OptField("kind", typ.String). + Field("state", typ.String). + OptField("generation", typ.String). + Field("capabilities", sourceCapabilitiesType). Field("name", typ.String). Field("slot", typ.String). OptField("publication", typ.String). + OptField("engine", typ.String). + OptField("file", typ.String). + OptField("db_resource", typ.String). + OptField("epoch", typ.String). + OptField("error", typ.String). OptField("tables", typ.NewArray(typ.String)). Field("streaming", typ.Boolean). Field("failover", typ.Boolean). Field("temporary", typ.Boolean). Field("snapshot", typ.Boolean). + Field("faulted", typ.Boolean). Build() var streamOptionsType = typ.NewRecord(). OptField("tables", typ.NewArray(typ.String)). OptField("ops", typ.NewArray(typ.String)). - OptField("buffer", typ.Number). + OptField("buffer", typ.Integer). + OptField("snapshot", typ.Boolean). + OptField("after", typ.String). + Build() + +var changeType = typ.NewRecord(). + OptField("source_id", typ.String). + Field("source", typ.String). + Field("op", typ.String). + Field("schema", typ.String). + Field("table", typ.String). + Field("relation", typ.String). + Field("lsn", typ.String). + OptField("commit_lsn", typ.String). + OptField("cursor", typ.String). + OptField("generation", typ.String). + OptField("transaction", typ.String). + OptField("error", typ.String). + OptField("xid", typ.Integer). + OptField("before", typ.NewMap(typ.String, typ.Any)). + OptField("after", typ.NewMap(typ.String, typ.Any)). Build() -var cdcStreamType = typ.NewInterface("cdc.Stream", []typ.Method{ - {Name: "channel", Type: typ.Func().Param("self", typ.Self).Returns(typ.Any).Build()}, - {Name: "receive", Type: typ.Func().Param("self", typ.Self).Returns(typ.Any).Build()}, - {Name: "close", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, - {Name: "release", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, -}) +var cdcStreamType *typ.Interface + +func init() { + cdcChannelType = typ.Any + if manifest := engine.ChannelModuleTypes(); manifest != nil { + if t, ok := manifest.LookupType("Channel"); ok { + if gen, ok := t.(*typ.Generic); ok { + cdcChannelType = typ.Instantiate(gen, changeType) + } + } + } + + cdcStreamType = typ.NewInterface("cdc.Stream", []typ.Method{ + {Name: "channel", Type: typ.Func().Param("self", typ.Self).Returns(cdcChannelType).Build()}, + {Name: "receive", Type: typ.Func().Param("self", typ.Self).Returns(cdcChannelType).Build()}, + {Name: "close", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, + {Name: "release", Type: typ.Func().Param("self", typ.Self).Returns(typ.Boolean, typ.NewOptional(typ.LuaError)).Build()}, + }) +} func ModuleTypes() *io.Manifest { m := io.NewManifest("cdc") + m.DefineType("Capabilities", sourceCapabilitiesType) + m.DefineType("SourceInfo", sourceInfoType) + m.DefineType("StreamOptions", streamOptionsType) + m.DefineType("Change", changeType) moduleType := typ.NewInterface("cdc", []typ.Method{ {Name: "list_sources", Type: typ.Func().Returns(typ.NewArray(sourceInfoType), typ.NewOptional(typ.LuaError)).Build()}, diff --git a/runtime/lua/modules/cdc/yields.go b/runtime/lua/modules/cdc/yields.go index c0b3ac2cc..d272e3c5b 100644 --- a/runtime/lua/modules/cdc/yields.go +++ b/runtime/lua/modules/cdc/yields.go @@ -122,6 +122,15 @@ func cdcChangeHandler(_ context.Context, l *lua.LState, _ pid.PID, _ string, pay if len(payloads) == 0 { return lua.LNil } + if payloads[0].Format() == payload.GoError { + streamErr, ok := payloads[0].Data().(error) + if !ok { + return lua.NewLuaError(l, fmt.Sprintf("cdc stream error payload has invalid type %T", payloads[0].Data())). + WithKind(lua.Internal). + WithRetryable(false) + } + return lua.WrapErrorWithLua(l, streamErr, "cdc stream") + } change, ok := payloads[0].Data().(cdcapi.Change) if !ok { if ptr, ptrOK := payloads[0].Data().(*cdcapi.Change); ptrOK && ptr != nil { @@ -145,7 +154,10 @@ func cdcChangeHandler(_ context.Context, l *lua.LState, _ pid.PID, _ string, pay } func changeToLua(l *lua.LState, change cdcapi.Change) (lua.LValue, error) { - tbl := l.CreateTable(0, 10) + tbl := l.CreateTable(0, 18) + if !isZeroRegistryID(change.SourceID) { + tbl.RawSetString("source_id", lua.LString(registryIDString(change.SourceID))) + } tbl.RawSetString("source", lua.LString(change.Source)) tbl.RawSetString("op", lua.LString(change.Op)) tbl.RawSetString("schema", lua.LString(change.Schema)) @@ -155,6 +167,15 @@ func changeToLua(l *lua.LState, change cdcapi.Change) (lua.LValue, error) { if change.CommitLSN != "" { tbl.RawSetString("commit_lsn", lua.LString(change.CommitLSN)) } + if change.Cursor != "" { + tbl.RawSetString("cursor", lua.LString(change.Cursor)) + } + if change.Generation != "" { + tbl.RawSetString("generation", lua.LString(change.Generation)) + } + if change.Transaction != "" { + tbl.RawSetString("transaction", lua.LString(change.Transaction)) + } if change.Error != "" { tbl.RawSetString("error", lua.LString(change.Error)) } From ee19ea5dbba3c92d9e4c69408affdfd8f4300884 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:13:11 -0400 Subject: [PATCH 08/47] fix(cdc/postgres): harden source capabilities and subscriptions --- service/cdc/postgres/decoder.go | 378 ++++++++++++++--- service/cdc/postgres/decoder_stream_test.go | 123 +++++- service/cdc/postgres/driver.go | 36 +- service/cdc/postgres/driver_test.go | 51 +++ service/cdc/postgres/integration_lua_test.go | 9 +- service/cdc/postgres/service.go | 419 +++++++++++++++---- service/cdc/postgres/stream.go | 40 +- service/cdc/postgres/stream_test.go | 52 +++ 8 files changed, 935 insertions(+), 173 deletions(-) create mode 100644 service/cdc/postgres/driver_test.go diff --git a/service/cdc/postgres/decoder.go b/service/cdc/postgres/decoder.go index ba0ee3e44..87892d1f8 100644 --- a/service/cdc/postgres/decoder.go +++ b/service/cdc/postgres/decoder.go @@ -11,6 +11,16 @@ import ( type bufferedChange struct { rc RowChange subxid uint32 + bytes int64 +} + +// decodeResult describes the progress made by one logical replication +// message. Changes are returned only at a transaction boundary. A safe result +// means the WAL position after this message can be acknowledged without +// reconstructing decoder state after a restart. +type decodeResult struct { + changes []RowChange + safe bool } type decoder struct { @@ -21,17 +31,42 @@ type decoder struct { curTopXid uint32 streaming bool inStream bool + txActive bool + limits decoderLimits + usage map[uint32]int64 } -func newDecoder() *decoder { - return &decoder{rels: newRelationCache()} +func newDecoder(limits ...decoderLimits) *decoder { + return newDecoderWithMode(false, limits...) } -func newStreamingDecoder() *decoder { - return &decoder{rels: newRelationCache(), streaming: true, buffer: map[uint32][]bufferedChange{}} +func newStreamingDecoder(limits ...decoderLimits) *decoder { + return newDecoderWithMode(true, limits...) +} + +func newDecoderWithMode(streaming bool, limits ...decoderLimits) *decoder { + configured := defaultDecoderLimits() + if len(limits) > 0 { + configured = normalizeDecoderLimits(limits[0]) + } + return &decoder{ + rels: newRelationCache(), + buffer: make(map[uint32][]bufferedChange), + streaming: streaming, + limits: configured, + usage: make(map[uint32]int64), + } } func (d *decoder) decode(walData []byte, walStart pglogrepl.LSN) ([]RowChange, error) { + result, err := d.decodeResult(walData, walStart) + if err != nil { + return nil, err + } + return result.changes, nil +} + +func (d *decoder) decodeResult(walData []byte, walStart pglogrepl.LSN) (decodeResult, error) { var ( msg pglogrepl.Message err error @@ -42,65 +77,216 @@ func (d *decoder) decode(walData []byte, walStart pglogrepl.LSN) ([]RowChange, e msg, err = pglogrepl.Parse(walData) } if err != nil { - return nil, fmt.Errorf("parse logical message: %w", err) + return decodeResult{}, fmt.Errorf("parse logical message: %w", err) } - return d.apply(msg, walStart) + return d.applyResult(msg, walStart) } +// apply is retained as the small decoder test seam. It deliberately returns +// no rows until Commit or StreamCommit; callers that need checkpoint progress +// use applyResult/decodeResult instead. func (d *decoder) apply(msg pglogrepl.Message, walStart pglogrepl.LSN) ([]RowChange, error) { + result, err := d.applyResult(msg, walStart) + if err != nil { + return nil, err + } + return result.changes, nil +} + +func (d *decoder) applyResult(msg pglogrepl.Message, walStart pglogrepl.LSN) (decodeResult, error) { switch m := msg.(type) { case *pglogrepl.RelationMessage: d.rels.put(m) + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.OriginMessage: + // Origin is transaction metadata. The row API has no origin field, but + // the message is valid and must not interrupt an otherwise valid stream. + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.TypeMessage: + // Type definitions are metadata for output plugins. Tuple decoding is + // intentionally text-preserving, so there is no row-level state to + // update here. + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.LogicalDecodingMessage: + // Logical messages are valid pgoutput records but are not row changes. + // They remain commit-gated by the decoder state and are therefore safe + // to ignore without advancing a checkpoint inside a transaction. + return decodeResult{safe: !d.inTransaction()}, nil + case *pglogrepl.BeginMessage: + // Protocol v2 may interleave an ordinary (small) transaction with + // streamed transactions whose segments have already been stopped. The + // ordinary transaction has its own buffer (key 0), so an existing + // streamed buffer is not a nested Begin. + if d.txActive || d.inStream { + return decodeResult{}, fmt.Errorf("%w: nested begin", ErrInvalidTransaction) + } d.commitLSN = m.FinalLSN d.xid = m.Xid + d.txActive = true + d.buffer[0] = nil + return decodeResult{}, nil + case *pglogrepl.CommitMessage: - d.commitLSN = 0 - d.xid = 0 + if !d.txActive { + return decodeResult{}, fmt.Errorf("%w: commit without begin", ErrInvalidTransaction) + } + return decodeResult{changes: d.flushTransaction(m.CommitLSN), safe: len(d.buffer) == 0}, nil + case *pglogrepl.InsertMessage: return d.one(OpInsert, m.RelationID, nil, m.Tuple, walStart) + case *pglogrepl.UpdateMessage: return d.one(OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) + case *pglogrepl.DeleteMessage: return d.one(OpDelete, m.RelationID, m.OldTuple, nil, walStart) + case *pglogrepl.TruncateMessage: - return d.truncate(m, walStart) + return d.truncateResult(m.RelationIDs, walStart) + case *pglogrepl.RelationMessageV2: d.rels.put(&m.RelationMessage) + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.TypeMessageV2: + return decodeResult{safe: !d.inTransaction()}, nil + + case *pglogrepl.LogicalDecodingMessageV2: + return decodeResult{safe: !d.inTransaction()}, nil + case *pglogrepl.StreamStartMessageV2: + if !d.streaming { + return decodeResult{}, fmt.Errorf("%w: stream start in protocol v1", ErrInvalidTransaction) + } + if d.inStream || d.txActive { + return decodeResult{}, fmt.Errorf("%w: nested stream start", ErrInvalidTransaction) + } d.inStream = true d.curTopXid = m.Xid if _, ok := d.buffer[m.Xid]; !ok { d.buffer[m.Xid] = nil } + return decodeResult{}, nil + case *pglogrepl.StreamStopMessageV2: + if !d.inStream { + return decodeResult{}, fmt.Errorf("%w: stream stop without start", ErrInvalidTransaction) + } d.inStream = false + return decodeResult{}, nil + case *pglogrepl.StreamCommitMessageV2: - return d.flushStream(m.Xid, m.CommitLSN), nil + if d.inStream { + return decodeResult{}, fmt.Errorf("%w: stream commit before stream stop", ErrInvalidTransaction) + } + if _, ok := d.buffer[m.Xid]; !ok { + return decodeResult{}, fmt.Errorf("%w: stream commit for unknown xid %d", ErrInvalidTransaction, m.Xid) + } + changes := d.flushStream(m.Xid, m.CommitLSN) + return decodeResult{changes: changes, safe: len(d.buffer) == 0}, nil + case *pglogrepl.StreamAbortMessageV2: + if d.inStream { + return decodeResult{}, fmt.Errorf("%w: stream abort before stream stop", ErrInvalidTransaction) + } + if _, ok := d.buffer[m.Xid]; !ok { + return decodeResult{}, fmt.Errorf("%w: stream abort for unknown xid %d", ErrInvalidTransaction, m.Xid) + } d.abortStream(m.Xid, m.SubXid) + return decodeResult{safe: len(d.buffer) == 0}, nil + case *pglogrepl.InsertMessageV2: if d.inStream { - return nil, d.bufferOne(m.Xid, OpInsert, m.RelationID, nil, m.Tuple, walStart) + return decodeResult{}, d.bufferOne(m.Xid, OpInsert, m.RelationID, nil, m.Tuple, walStart) } return d.one(OpInsert, m.RelationID, nil, m.Tuple, walStart) + case *pglogrepl.UpdateMessageV2: if d.inStream { - return nil, d.bufferOne(m.Xid, OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) + return decodeResult{}, d.bufferOne(m.Xid, OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) } return d.one(OpUpdate, m.RelationID, m.OldTuple, m.NewTuple, walStart) + case *pglogrepl.DeleteMessageV2: if d.inStream { - return nil, d.bufferOne(m.Xid, OpDelete, m.RelationID, m.OldTuple, nil, walStart) + return decodeResult{}, d.bufferOne(m.Xid, OpDelete, m.RelationID, m.OldTuple, nil, walStart) } return d.one(OpDelete, m.RelationID, m.OldTuple, nil, walStart) + case *pglogrepl.TruncateMessageV2: if d.inStream { - return nil, d.bufferTruncate(m, walStart) + return decodeResult{}, d.bufferTruncate(m, walStart) } - return d.truncate(&m.TruncateMessage, walStart) + return d.truncateResult(m.RelationIDs, walStart) + + default: + return decodeResult{}, fmt.Errorf("%w: %T", ErrUnsupportedMessage, msg) + } +} + +func (d *decoder) inTransaction() bool { + return d.txActive || d.inStream || len(d.buffer) > 0 +} + +func (d *decoder) flushTransaction(commitLSN pglogrepl.LSN) []RowChange { + if commitLSN == 0 { + commitLSN = d.commitLSN + } + buffered := d.buffer[0] + delete(d.buffer, 0) + delete(d.usage, 0) + changes := make([]RowChange, 0, len(buffered)) + for i := range buffered { + buffered[i].rc.CommitLSN = commitLSN.String() + changes = append(changes, buffered[i].rc) + } + d.commitLSN = 0 + d.xid = 0 + d.txActive = false + return changes +} + +func (d *decoder) one(op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) (decodeResult, error) { + if !d.txActive { + return decodeResult{}, fmt.Errorf("%w: row without begin", ErrInvalidTransaction) + } + bytes, err := d.reserveRow(0, relID, oldT, newT) + if err != nil { + return decodeResult{}, err + } + rc, err := d.changeFor(op, relID, oldT, newT, walStart) + if err != nil { + return decodeResult{}, err + } + rc.XID = d.xid + rc.CommitLSN = d.commitLSN.String() + d.buffer[0] = append(d.buffer[0], bufferedChange{rc: rc, subxid: d.xid, bytes: bytes}) + return decodeResult{}, nil +} + +func (d *decoder) truncateResult(relationIDs []uint32, walStart pglogrepl.LSN) (decodeResult, error) { + if !d.txActive { + return decodeResult{}, fmt.Errorf("%w: truncate without begin", ErrInvalidTransaction) } - return nil, nil + relations, bytes, err := d.truncateBudget(0, relationIDs) + if err != nil { + return decodeResult{}, err + } + for i, rel := range relations { + d.buffer[0] = append(d.buffer[0], bufferedChange{rc: RowChange{ + Op: OpTruncate, + Schema: rel.Namespace, + Table: rel.RelationName, + LSN: walStart.String(), + CommitLSN: d.commitLSN.String(), + XID: d.xid, + }, subxid: d.xid, bytes: bytes[i]}) + } + return decodeResult{}, nil } func (d *decoder) changeFor(op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) (RowChange, error) { @@ -118,32 +304,32 @@ func (d *decoder) changeFor(op Op, relID uint32, oldT, newT *pglogrepl.TupleData }, nil } -func (d *decoder) one(op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) ([]RowChange, error) { - rc, err := d.changeFor(op, relID, oldT, newT, walStart) +func (d *decoder) bufferOne(subxid uint32, op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) error { + if !d.inStream { + return fmt.Errorf("%w: streamed row without stream start", ErrInvalidTransaction) + } + bytes, err := d.reserveRow(d.curTopXid, relID, oldT, newT) if err != nil { - return nil, err + return err } - rc.XID = d.xid - rc.CommitLSN = d.commitLSN.String() - return []RowChange{rc}, nil -} - -func (d *decoder) bufferOne(subxid uint32, op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walStart pglogrepl.LSN) error { rc, err := d.changeFor(op, relID, oldT, newT, walStart) if err != nil { return err } rc.XID = d.curTopXid - d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: subxid}) + d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: subxid, bytes: bytes}) return nil } func (d *decoder) bufferTruncate(m *pglogrepl.TruncateMessageV2, walStart pglogrepl.LSN) error { - for _, relID := range m.RelationIDs { - rel, ok := d.rels.get(relID) - if !ok { - return fmt.Errorf("%w: %d", ErrUnknownRelation, relID) - } + if !d.inStream { + return fmt.Errorf("%w: streamed truncate without stream start", ErrInvalidTransaction) + } + relations, bytes, err := d.truncateBudget(d.curTopXid, m.RelationIDs) + if err != nil { + return err + } + for i, rel := range relations { rc := RowChange{ Op: OpTruncate, Schema: rel.Namespace, @@ -151,7 +337,7 @@ func (d *decoder) bufferTruncate(m *pglogrepl.TruncateMessageV2, walStart pglogr LSN: walStart.String(), XID: d.curTopXid, } - d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: m.Xid}) + d.buffer[d.curTopXid] = append(d.buffer[d.curTopXid], bufferedChange{rc: rc, subxid: m.Xid, bytes: bytes[i]}) } return nil } @@ -159,7 +345,9 @@ func (d *decoder) bufferTruncate(m *pglogrepl.TruncateMessageV2, walStart pglogr func (d *decoder) flushStream(topXid uint32, commitLSN pglogrepl.LSN) []RowChange { buffered := d.buffer[topXid] delete(d.buffer, topXid) + delete(d.usage, topXid) d.inStream = false + d.curTopXid = 0 out := make([]RowChange, 0, len(buffered)) for i := range buffered { @@ -171,40 +359,126 @@ func (d *decoder) flushStream(topXid uint32, commitLSN pglogrepl.LSN) []RowChang func (d *decoder) abortStream(topXid, subXid uint32) { d.inStream = false + d.curTopXid = 0 if topXid == subXid { delete(d.buffer, topXid) + delete(d.usage, topXid) return } + + // PostgreSQL's logical apply worker does not remove only changes whose + // XID equals subXid. It records the first streamed offset for each + // subtransaction and truncates the transaction at the aborted subxact's + // offset. This also discards nested subtransactions (and the remainder of + // that stream segment); changes after the rollback are sent in a later + // stream segment. The decoder has the same ordering in its buffer, so the + // first change for subXid is the equivalent truncation point. src := d.buffer[topXid] - n := 0 - for _, bc := range src { - if bc.subxid != subXid { - src[n] = bc - n++ + cut := -1 + for i, bc := range src { + if bc.subxid == subXid { + cut = i + break } } - if n == 0 { + if cut < 0 { + // Empty subtransactions are valid and have no buffered offset. There + // is nothing to truncate in that case. + return + } + // Clear the discarded tail before reslicing so decoded row maps and their + // byte slices are no longer retained by the backing array. + clear(src[cut:]) + src = src[:cut] + if len(src) == 0 { delete(d.buffer, topXid) + delete(d.usage, topXid) return } - d.buffer[topXid] = src[:n] + d.buffer[topXid] = src + var bytes int64 + for _, bc := range src { + bytes += bc.bytes + } + d.usage[topXid] = bytes } -func (d *decoder) truncate(m *pglogrepl.TruncateMessage, walStart pglogrepl.LSN) ([]RowChange, error) { - changes := make([]RowChange, 0, len(m.RelationIDs)) - for _, relID := range m.RelationIDs { +func (d *decoder) reserveRow(key, relID uint32, oldT, newT *pglogrepl.TupleData) (int64, error) { + rel, ok := d.rels.get(relID) + if !ok { + return 0, fmt.Errorf("%w: %d", ErrUnknownRelation, relID) + } + bytes := estimateChangeBytes(rel, oldT, newT) + if err := d.reserve(key, 1, bytes); err != nil { + return 0, err + } + return bytes, nil +} + +func (d *decoder) truncateBudget(key uint32, relationIDs []uint32) ([]*pglogrepl.RelationMessage, []int64, error) { + relations := make([]*pglogrepl.RelationMessage, len(relationIDs)) + bytes := make([]int64, len(relationIDs)) + var total int64 + for i, relID := range relationIDs { rel, ok := d.rels.get(relID) if !ok { - return nil, fmt.Errorf("%w: %d", ErrUnknownRelation, relID) + return nil, nil, fmt.Errorf("%w: %d", ErrUnknownRelation, relID) + } + relations[i] = rel + bytes[i] = estimateRelationChangeBytes(rel) + total += bytes[i] + } + if err := d.reserve(key, len(relationIDs), total); err != nil { + return nil, nil, err + } + return relations, bytes, nil +} + +func (d *decoder) reserve(key uint32, changes int, bytes int64) error { + if changes < 0 || bytes < 0 { + return fmt.Errorf("%w: invalid estimated transaction size", ErrTransactionLimit) + } + currentChanges := len(d.buffer[key]) + if changes > d.limits.maxChanges-currentChanges { + return fmt.Errorf("%w: changes=%d limit=%d", ErrTransactionLimit, currentChanges+changes, d.limits.maxChanges) + } + currentBytes := d.usage[key] + if bytes > d.limits.maxBytes-currentBytes { + return fmt.Errorf("%w: bytes=%d limit=%d", ErrTransactionLimit, currentBytes+bytes, d.limits.maxBytes) + } + d.usage[key] = currentBytes + bytes + return nil +} + +const ( + changeEstimateBase int64 = 256 + columnEstimateBase int64 = 64 + stringCopyEstimate int64 = 1 +) + +func estimateChangeBytes(rel *pglogrepl.RelationMessage, oldT, newT *pglogrepl.TupleData) int64 { + return changeEstimateBase + int64(len(rel.Namespace)+len(rel.RelationName)) + + estimateTupleBytes(rel, oldT) + estimateTupleBytes(rel, newT) +} + +func estimateRelationChangeBytes(rel *pglogrepl.RelationMessage) int64 { + return changeEstimateBase + int64(len(rel.Namespace)+len(rel.RelationName)) +} + +func estimateTupleBytes(rel *pglogrepl.RelationMessage, tuple *pglogrepl.TupleData) int64 { + if tuple == nil { + return 0 + } + bytes := columnEstimateBase + for i, col := range tuple.Columns { + bytes += columnEstimateBase + int64(len(col.Data)) + if col.DataType != pglogrepl.TupleDataTypeNull && col.DataType != pglogrepl.TupleDataTypeToast { + // tupleToMap converts values to strings, retaining a second copy. + bytes += stringCopyEstimate * int64(len(col.Data)) + } + if i < len(rel.Columns) { + bytes += int64(len(rel.Columns[i].Name)) } - changes = append(changes, RowChange{ - Op: OpTruncate, - Schema: rel.Namespace, - Table: rel.RelationName, - LSN: walStart.String(), - CommitLSN: d.commitLSN.String(), - XID: d.xid, - }) } - return changes, nil + return bytes } diff --git a/service/cdc/postgres/decoder_stream_test.go b/service/cdc/postgres/decoder_stream_test.go index d5de5f6f6..dc7127e94 100644 --- a/service/cdc/postgres/decoder_stream_test.go +++ b/service/cdc/postgres/decoder_stream_test.go @@ -45,11 +45,31 @@ func TestStreamingDecoderBuffersUntilCommit(t *testing.T) { assert.Equal(t, "a@w.ai", changes[0].After["email"]) } +func TestStreamingDecoderRequiresStopBeforeCommitOrAbort(t *testing.T) { + for _, tc := range []struct { + name string + msg pglogrepl.Message + }{ + {name: "commit", msg: &pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x99}}, + {name: "abort", msg: &pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 100}}, + } { + t.Run(tc.name, func(t *testing.T) { + d := newStreamingDecoder() + _, err := d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + + _, err = d.apply(tc.msg, 0) + require.ErrorIs(t, err, ErrInvalidTransaction) + }) + } +} + func TestStreamingDecoderTopLevelAbortDiscards(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(relV2(), 0) _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) _, _ = d.apply(insertV2(100, "1", "a@w.ai"), 0x20) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) _, err := d.apply(&pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 100}, 0) require.NoError(t, err) @@ -64,6 +84,7 @@ func TestStreamingDecoderSubtransactionAbortDropsOnlyThatSubxid(t *testing.T) { _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) _, _ = d.apply(insertV2(100, "1", "keep@w.ai"), 0x20) _, _ = d.apply(insertV2(200, "2", "drop@w.ai"), 0x30) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) _, err := d.apply(&pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 200}, 0) require.NoError(t, err) @@ -74,6 +95,28 @@ func TestStreamingDecoderSubtransactionAbortDropsOnlyThatSubxid(t *testing.T) { assert.Equal(t, "keep@w.ai", changes[0].After["email"]) } +func TestStreamingDecoderSubtransactionAbortDropsNestedDescendants(t *testing.T) { + d := newStreamingDecoder() + _, _ = d.apply(relV2(), 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(100, "1", "keep@w.ai"), 0x20) + _, _ = d.apply(insertV2(200, "2", "drop-parent@w.ai"), 0x30) + _, _ = d.apply(insertV2(300, "3", "drop-child@w.ai"), 0x40) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + + // A subtransaction abort includes all of its nested subtransactions. + // PostgreSQL represents this by truncating the streamed changes at the + // aborted subtransaction's first offset, not by sending a parent XID on + // every descendant row. + _, err := d.apply(&pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 200}, 0) + require.NoError(t, err) + + changes, err := d.apply(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x99}, 0) + require.NoError(t, err) + require.Len(t, changes, 1) + assert.Equal(t, "keep@w.ai", changes[0].After["email"]) +} + func TestStreamingDecoderInterleavedTransactions(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(relV2(), 0) @@ -97,25 +140,97 @@ func TestStreamingDecoderInterleavedTransactions(t *testing.T) { assert.Equal(t, "tx100@w.ai", c100[0].After["email"]) } -func TestStreamingDecoderNonStreamedV2EmitsImmediately(t *testing.T) { +func TestStreamingDecoderNonStreamedV2EmitsAtCommit(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(relV2(), 0) _, _ = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) changes, err := d.apply(insertV2(0, "1", "v2small@w.ai"), 0x20) require.NoError(t, err) - require.Len(t, changes, 1, "non-streamed v2 insert (inStream=false) must emit immediately, not buffer") + assert.Empty(t, changes, "non-streamed v2 rows must wait for commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) + require.Len(t, changes, 1) assert.Equal(t, uint32(7), changes[0].XID) assert.Equal(t, "v2small@w.ai", changes[0].After["email"]) } -func TestStreamingDecoderNonStreamedStillWorks(t *testing.T) { +func TestStreamingDecoderNonStreamedStillWaitsForCommit(t *testing.T) { d := newStreamingDecoder() _, _ = d.apply(accountsRel(), 0) _, _ = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "small@w.ai")}, 0x20) require.NoError(t, err) - require.Len(t, changes, 1, "small (non-streamed) transactions must still emit immediately") + assert.Empty(t, changes, "non-streamed rows must wait for commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) + require.Len(t, changes, 1) assert.Equal(t, uint32(7), changes[0].XID) } + +func TestStreamingDecoderDoesNotMarkInterleavedCommitSafe(t *testing.T) { + d := newStreamingDecoder() + _, _ = d.apply(relV2(), 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(100, "1", "tx100@w.ai"), 0x20) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 200, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(200, "2", "tx200@w.ai"), 0x30) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + + result, err := d.applyResult(&pglogrepl.StreamCommitMessageV2{Xid: 200, CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.False(t, result.safe, "an earlier streamed transaction is still open") + assert.Len(t, result.changes, 1) + + result, err = d.applyResult(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x50}, 0) + require.NoError(t, err) + assert.True(t, result.safe) + assert.Len(t, result.changes, 1) +} + +func TestStreamingDecoderAllowsOrdinaryTransactionAlongsideStream(t *testing.T) { + d := newStreamingDecoder() + _, _ = d.apply(relV2(), 0) + _, _ = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + _, _ = d.apply(insertV2(100, "1", "streamed@w.ai"), 0x20) + _, _ = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + + _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x30, Xid: 7}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(0, "2", "ordinary@w.ai"), 0x31) + require.NoError(t, err) + result, err := d.applyResult(&pglogrepl.CommitMessage{CommitLSN: 0x32}, 0) + require.NoError(t, err) + assert.False(t, result.safe, "streamed transaction is still buffered") + require.Len(t, result.changes, 1) + assert.Equal(t, "ordinary@w.ai", result.changes[0].After["email"]) + + result, err = d.applyResult(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.True(t, result.safe) + require.Len(t, result.changes, 1) + assert.Equal(t, "streamed@w.ai", result.changes[0].After["email"]) +} + +func TestStreamingDecoderAcceptsMetadataMessages(t *testing.T) { + d := newStreamingDecoder() + metadata := []pglogrepl.Message{ + &pglogrepl.TypeMessageV2{}, + &pglogrepl.LogicalDecodingMessageV2{}, + } + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.True(t, result.safe) + } + + _, err := d.applyResult(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.False(t, result.safe) + } +} diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index 964fbb474..e1ea8bb56 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -93,6 +93,7 @@ func (s *sourceAdapter) Info() config.SourceInfo { s.mu.RUnlock() source.mu.Lock() state := source.state + sourceErr := source.sourceErr source.mu.Unlock() info := config.SourceInfo{ @@ -101,15 +102,21 @@ func (s *sourceAdapter) Info() config.SourceInfo { Slot: source.slot, Publication: source.publication, Tables: append([]string(nil), source.tables...), - Streaming: state == sourceRunning, - Failover: source.failover, - Temporary: source.temporary, - Snapshot: source.snapshot, - State: postgresSourceState(state), + // Streaming is a legacy field describing the configured pgoutput + // protocol mode, not the current lifecycle state. State is exposed by + // SourceState above. + Streaming: source.streaming, + Failover: source.failover, + Temporary: source.temporary, + Snapshot: source.snapshot, + State: postgresSourceState(state), Capabilities: config.Capabilities{ - Snapshot: source.snapshot, - Durable: true, - Replayable: true, + // Snapshot is a source-start bootstrap operation. Subscribe rejects + // per-consumer snapshot requests, so it is not a common API + // capability of this adapter. + Snapshot: false, + Durable: !source.temporary, + Replayable: false, CapturesExternalWrites: true, BeforeImages: false, }, @@ -117,6 +124,9 @@ func (s *sourceAdapter) Info() config.SourceInfo { if state == sourceFailed { info.Faulted = true } + if sourceErr != nil { + info.Error = sourceErr.Error() + } return info } @@ -187,15 +197,7 @@ func (s *sourceAdapter) Dispose(ctx context.Context) error { source := s.source s.mu.RUnlock() source.MarkForSlotDrop() - stopErr := source.Stop(ctx) - source.mu.Lock() - temporary := source.temporary - source.mu.Unlock() - if temporary { - return stopErr - } - cleanupErr := source.dropSlotAndCheckpoint(ctx) - return errors.Join(stopErr, cleanupErr) + return source.Stop(ctx) } func (s *sourceAdapter) LifecycleConfig() supervisor.LifecycleConfig { diff --git a/service/cdc/postgres/driver_test.go b/service/cdc/postgres/driver_test.go new file mode 100644 index 000000000..0a04268f7 --- /dev/null +++ b/service/cdc/postgres/driver_test.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cconfig "github.com/wippyai/runtime/api/service/cdc" +) + +func TestSourceAdapterInfoReportsConservativeCapabilities(t *testing.T) { + source := NewSource(SourceOptions{ + Name: "app:events", + Slot: "events_slot", + Snapshot: true, + Streaming: true, + Temporary: true, + }) + terminalErr := errors.New("replication connection lost") + source.mu.Lock() + source.state = sourceFailed + source.sourceErr = terminalErr + source.mu.Unlock() + + adapter := &sourceAdapter{source: source} + info := adapter.Info() + + assert.Equal(t, cconfig.SourceStateFaulted, info.State) + assert.True(t, info.Faulted) + assert.Equal(t, terminalErr.Error(), info.Error) + assert.True(t, info.Snapshot, "legacy snapshot field preserves configured bootstrap mode") + assert.True(t, info.Streaming, "legacy streaming field preserves configured protocol mode") + assert.False(t, info.Capabilities.Snapshot, "per-consumer snapshots are unsupported") + assert.False(t, info.Capabilities.Replayable, "After cursors are unsupported") + assert.False(t, info.Capabilities.Durable, "temporary slots are not durable") + assert.False(t, info.Capabilities.BeforeImages) +} + +func TestSourceStopAfterDisposeCleanupIsIdempotent(t *testing.T) { + source := NewSource(SourceOptions{Slot: "events_slot"}) + source.dropDone.Store(true) + source.MarkForSlotDrop() + + // A completed cleanup is a tombstone: retries must not reopen a connection + // or attempt to drop the same slot again. + require.NoError(t, source.Stop(nil)) + require.NoError(t, source.Stop(nil)) +} diff --git a/service/cdc/postgres/integration_lua_test.go b/service/cdc/postgres/integration_lua_test.go index 3312706a3..4690729ae 100644 --- a/service/cdc/postgres/integration_lua_test.go +++ b/service/cdc/postgres/integration_lua_test.go @@ -85,11 +85,10 @@ func TestLuaSeesRealRunningSourceAndItsChanges(t *testing.T) { require.NoError(t, sup.Start(supCtx)) manager := &Manager{ - bus: bus, - log: zap.NewNop(), - sources: map[registry.ID]*Source{}, - infos: map[registry.ID]cdcapi.SourceInfo{}, - infosByKey: map[string]registry.ID{}, + bus: bus, + log: zap.NewNop(), + sources: map[registry.ID]*Source{}, + infos: map[registry.ID]cdcapi.SourceInfo{}, } entryID := registry.NewID("test", "cdc-lua-e2e") diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 09bfa7b1c..1c0fd72e7 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -55,39 +55,75 @@ type SourceOptions struct { Snapshot bool Streaming bool Failover bool + // MaxTransactionChanges bounds the number of row changes retained before + // an ordinary or streamed transaction commits. Zero uses the safe default. + MaxTransactionChanges int + // MaxTransactionBytes bounds the estimated memory retained for one + // ordinary or streamed transaction. Zero uses the safe default. + MaxTransactionBytes int64 } type Source struct { - log *zap.Logger - coll metrics.Collector - injectedCP Checkpointer - cancel context.CancelFunc - done chan struct{} - subs map[uint64]*sourceSubscription - replDSN string - adminDSN string - name string - slot string - publication string - tables []string - standbyInterval time.Duration - statusInterval time.Duration - mu sync.Mutex - subMu sync.RWMutex - nextSubID uint64 - snapshotFetchSize int - temporary bool - snapshot bool - streaming bool - failover bool - stopped atomic.Bool - dropSlot atomic.Bool + log *zap.Logger + coll metrics.Collector + injectedCP Checkpointer + cancel context.CancelFunc + done chan struct{} + subs map[uint64]*sourceSubscription + replDSN string + adminDSN string + name string + slot string + publication string + tables []string + standbyInterval time.Duration + statusInterval time.Duration + mu sync.Mutex + subMu sync.RWMutex + nextSubID uint64 + snapshotFetchSize int + temporary bool + snapshot bool + streaming bool + failover bool + maxTransactionChanges int + maxTransactionBytes int64 + permanentlyClosed bool + sourceErr error + state sourceState + dropSlot atomic.Bool + dropDone atomic.Bool + dropMu sync.Mutex } +type sourceState uint8 + +const ( + sourceNew sourceState = iota + sourceStarting + sourceRunning + sourceStopping + sourceFailed + sourceStopped +) + var snapshotFailpoint func() error func (s *Source) MarkForSlotDrop() { s.dropSlot.Store(true) + s.mu.Lock() + s.permanentlyClosed = true + s.mu.Unlock() +} + +// Close permanently retires a source. Stop alone is restartable so a +// supervisor can recover a failed generation; callers removing a source from +// the registry should use Close when the instance must not be started again. +func (s *Source) Close(ctx context.Context) error { + s.mu.Lock() + s.permanentlyClosed = true + s.mu.Unlock() + return s.Stop(ctx) } func NewSource(opts SourceOptions) *Source { @@ -107,85 +143,154 @@ func NewSource(opts SourceOptions) *Source { if fetch <= 0 { fetch = defaultSnapshotFetchSize } + limits := normalizeDecoderLimits(decoderLimits{ + maxChanges: opts.MaxTransactionChanges, + maxBytes: opts.MaxTransactionBytes, + }) return &Source{ - log: log, - injectedCP: opts.Checkpoint, - replDSN: opts.ReplDSN, - adminDSN: opts.AdminDSN, - name: opts.Name, - slot: opts.Slot, - publication: opts.Publication, - tables: opts.Tables, - subs: make(map[uint64]*sourceSubscription), - temporary: opts.Temporary, - snapshot: opts.Snapshot, - streaming: opts.Streaming, - failover: opts.Failover, - standbyInterval: standby, - statusInterval: status, - snapshotFetchSize: fetch, + log: log, + injectedCP: opts.Checkpoint, + replDSN: opts.ReplDSN, + adminDSN: opts.AdminDSN, + name: opts.Name, + slot: opts.Slot, + publication: opts.Publication, + tables: append([]string(nil), opts.Tables...), + subs: make(map[uint64]*sourceSubscription), + temporary: opts.Temporary, + snapshot: opts.Snapshot, + streaming: opts.Streaming, + failover: opts.Failover, + standbyInterval: standby, + statusInterval: status, + snapshotFetchSize: fetch, + maxTransactionChanges: limits.maxChanges, + maxTransactionBytes: limits.maxBytes, } } func (s *Source) Start(ctx context.Context) (<-chan any, error) { - if s.stopped.Load() { + if ctx == nil { + ctx = context.Background() + } + runCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + + s.mu.Lock() + if s.permanentlyClosed { + s.mu.Unlock() + cancel() return nil, ErrSourceClosed } + switch s.state { + case sourceStarting, sourceRunning: + s.mu.Unlock() + cancel() + return nil, ErrSourceRunning + case sourceStopping: + s.mu.Unlock() + cancel() + return nil, ErrSourceStopping + default: + s.state = sourceStarting + s.cancel = cancel + s.done = done + // A failed snapshot/start may have dropped and checkpoint-cleaned the + // previous slot. This start may create a new slot generation, so the + // destructive cleanup marker must apply to that generation as well. + s.dropDone.Store(false) + } + s.mu.Unlock() + + failStart := func(startErr error) { + cancel() + s.mu.Lock() + if s.done == done { + if s.state == sourceStopping { + s.state = sourceStopped + } else if s.state == sourceStarting { + s.state = sourceFailed + s.sourceErr = startErr + } + s.cancel = nil + close(done) + } + s.mu.Unlock() + } adminDB, err := sql.Open("postgres", s.adminDSN) if err != nil { - return nil, fmt.Errorf("open admin connection: %w", err) + startErr := fmt.Errorf("open admin connection: %w", err) + failStart(startErr) + return nil, startErr } adminDB.SetMaxOpenConns(2) adminDB.SetMaxIdleConns(1) - if err := adminDB.PingContext(ctx); err != nil { + if err := adminDB.PingContext(runCtx); err != nil { _ = adminDB.Close() - return nil, fmt.Errorf("ping admin connection: %w", err) + startErr := fmt.Errorf("ping admin connection: %w", err) + failStart(startErr) + return nil, startErr } cp := s.injectedCP if cp == nil { - dbcp, cpErr := NewDBCheckpointer(ctx, adminDB) + dbcp, cpErr := NewDBCheckpointer(runCtx, adminDB) if cpErr != nil { _ = adminDB.Close() + failStart(cpErr) return nil, cpErr } cp = dbcp } - publication, err := s.ensurePublication(ctx, adminDB) + publication, err := s.ensurePublication(runCtx, adminDB) if err != nil { _ = adminDB.Close() + failStart(err) return nil, err } - conn, err := pgconn.Connect(ctx, s.replDSN) + conn, err := pgconn.Connect(runCtx, s.replDSN) if err != nil { _ = adminDB.Close() - return nil, fmt.Errorf("replication connect: %w", err) + startErr := fmt.Errorf("replication connect: %w", err) + failStart(startErr) + return nil, startErr } - sysident, err := pglogrepl.IdentifySystem(ctx, conn) + sysident, err := pglogrepl.IdentifySystem(runCtx, conn) if err != nil { - _ = conn.Close(ctx) + _ = conn.Close(context.Background()) _ = adminDB.Close() - return nil, fmt.Errorf("identify system: %w", err) + startErr := fmt.Errorf("identify system: %w", err) + failStart(startErr) + return nil, startErr } - startLSN, snapshotName, err := s.prepareSlot(ctx, conn, adminDB, cp, sysident.XLogPos) + startLSN, snapshotName, err := s.prepareSlot(runCtx, conn, adminDB, cp, sysident.XLogPos) if err != nil { - _ = conn.Close(ctx) + _ = conn.Close(context.Background()) _ = adminDB.Close() + failStart(err) return nil, err } - runCtx, cancel := context.WithCancel(ctx) status := make(chan any, 8) - done := make(chan struct{}) s.mu.Lock() - s.cancel = cancel - s.done = done + if s.state != sourceStarting { + // Stop may have been requested while the synchronous connection and + // slot setup was in progress. Do not publish a source that is already + // being stopped. + s.mu.Unlock() + _ = conn.Close(context.Background()) + _ = adminDB.Close() + failStart(ErrSourceClosed) + return nil, ErrSourceClosed + } + s.state = sourceRunning + s.sourceErr = nil s.mu.Unlock() s.log.Info("cdc source started", @@ -198,18 +303,38 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { default: } - s.coll = metrics.GetCollector(ctx) + s.coll = metrics.GetCollector(runCtx) go s.run(runCtx, conn, adminDB, cp, startLSN, snapshotName, publication, s.coll, status, done) return status, nil } func (s *Source) Stop(ctx context.Context) error { - if !s.stopped.CompareAndSwap(false, true) { - return nil + if ctx == nil { + ctx = context.Background() } - defer s.closeSubscriptions() s.mu.Lock() + if s.state == sourceStopped { + drop := s.dropSlot.Load() && !s.temporary + s.mu.Unlock() + if drop { + return s.dropSlotAndCheckpoint(ctx) + } + return nil + } + if s.state == sourceNew || s.state == sourceFailed { + s.state = sourceStopped + s.cancel = nil + s.mu.Unlock() + s.closeSubscriptions() + if s.dropSlot.Load() && !s.temporary { + return s.dropSlotAndCheckpoint(ctx) + } + return nil + } + if s.state == sourceStarting || s.state == sourceRunning { + s.state = sourceStopping + } cancel := s.cancel done := s.done s.mu.Unlock() @@ -217,9 +342,14 @@ func (s *Source) Stop(ctx context.Context) error { if cancel != nil { cancel() } + s.closeSubscriptions() if done != nil { select { case <-done: + s.mu.Lock() + s.state = sourceStopped + s.cancel = nil + s.mu.Unlock() case <-ctx.Done(): return ctx.Err() } @@ -243,7 +373,19 @@ func (s *Source) run( status chan any, done chan struct{}, ) { - defer close(done) + defer func() { + s.mu.Lock() + if s.done == done { + if s.state == sourceStopping { + s.state = sourceStopped + } else if s.state == sourceRunning || s.state == sourceStarting { + s.state = sourceFailed + } + s.cancel = nil + } + s.mu.Unlock() + close(done) + }() defer close(status) defer s.closeSubscriptions() defer func() { _ = adminDB.Close() }() @@ -270,28 +412,59 @@ func (s *Source) run( } if err := pglogrepl.StartReplication(ctx, conn, s.slot, startLSN, pglogrepl.StartReplicationOptions{PluginArgs: pluginArgs}); err != nil { + if snapshotName != "" { + s.abortFreshSnapshot(conn) + } s.fail(ctx, status, err) return } - dec := newDecoder() + limits := decoderLimits{ + maxChanges: s.maxTransactionChanges, + maxBytes: s.maxTransactionBytes, + } + dec := newDecoder(limits) if s.streaming { - dec = newStreamingDecoder() + dec = newStreamingDecoder(limits) } var opLabels map[Op]metrics.Labels if mc != nil { opLabels = map[Op]metrics.Labels{ - OpInsert: {"slot": s.slot, "op": string(OpInsert)}, - OpUpdate: {"slot": s.slot, "op": string(OpUpdate)}, - OpDelete: {"slot": s.slot, "op": string(OpDelete)}, - OpTruncate: {"slot": s.slot, "op": string(OpTruncate)}, - OpSnapshot: {"slot": s.slot, "op": string(OpSnapshot)}, + OpInsert: {"source": s.name, "op": string(OpInsert)}, + OpUpdate: {"source": s.name, "op": string(OpUpdate)}, + OpDelete: {"source": s.name, "op": string(OpDelete)}, + OpTruncate: {"source": s.name, "op": string(OpTruncate)}, + OpSnapshot: {"source": s.name, "op": string(OpSnapshot)}, } } - clientPos := startLSN - lastSaved := pglogrepl.LSN(0) + // safePos is the furthest position that can be replayed without losing + // decoder state. It advances only after a complete transaction boundary; + // the server WAL end in a keepalive is never a safe checkpoint. + safePos := startLSN + lastSaved := startLSN + defer func() { + if safePos <= lastSaved { + return + } + flushCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := cp.Save(flushCtx, s.slot, safePos); err != nil { + s.log.Warn("failed to persist final cdc checkpoint", + zap.String("slot", s.slot), zap.String("lsn", safePos.String()), zap.Error(err)) + } + }() + saveSafe := func() error { + if safePos <= lastSaved { + return nil + } + if err := cp.Save(ctx, s.slot, safePos); err != nil { + return err + } + lastSaved = safePos + return nil + } now := time.Now() nextStandby := now.Add(s.standbyInterval) nextStatus := now.Add(s.statusInterval) @@ -303,15 +476,16 @@ func (s *Source) run( now = time.Now() if !now.Before(nextStandby) { - if clientPos > lastSaved { - if err := cp.Save(ctx, s.slot, clientPos); err != nil { - s.fail(ctx, status, err) - return - } - lastSaved = clientPos + if err := saveSafe(); err != nil { + s.fail(ctx, status, err) + return } if err := pglogrepl.SendStandbyStatusUpdate(ctx, conn, - pglogrepl.StandbyStatusUpdate{WALWritePosition: clientPos}); err != nil { + pglogrepl.StandbyStatusUpdate{ + WALWritePosition: safePos, + WALFlushPosition: safePos, + WALApplyPosition: safePos, + }); err != nil { s.fail(ctx, status, err) return } @@ -340,6 +514,10 @@ func (s *Source) run( if !ok { continue } + if len(cd.Data) == 0 { + s.fail(ctx, status, fmt.Errorf("%w: empty CopyData payload", ErrUnsupportedMessage)) + return + } switch cd.Data[0] { case pglogrepl.PrimaryKeepaliveMessageByteID: @@ -348,12 +526,17 @@ func (s *Source) run( s.fail(ctx, status, kaErr) return } - if ka.ServerWALEnd > clientPos { - clientPos = ka.ServerWALEnd - } if ka.ReplyRequested { + if err := saveSafe(); err != nil { + s.fail(ctx, status, err) + return + } if err := pglogrepl.SendStandbyStatusUpdate(ctx, conn, - pglogrepl.StandbyStatusUpdate{WALWritePosition: clientPos}); err != nil { + pglogrepl.StandbyStatusUpdate{ + WALWritePosition: safePos, + WALFlushPosition: safePos, + WALApplyPosition: safePos, + }); err != nil { s.fail(ctx, status, err) return } @@ -364,20 +547,25 @@ func (s *Source) run( s.fail(ctx, status, xErr) return } - changes, dErr := dec.decode(xld.WALData, xld.WALStart) + result, dErr := dec.decodeResult(xld.WALData, xld.WALStart) if dErr != nil { s.fail(ctx, status, dErr) return } - for i := range changes { - s.emitChange(ctx, changes[i]) + for i := range result.changes { + s.emitChange(ctx, result.changes[i]) if mc != nil { - mc.CounterInc(changesCounter, opLabels[changes[i].Op]) + mc.CounterInc(changesCounter, opLabels[result.changes[i].Op]) } } - if end := xld.WALStart + pglogrepl.LSN(len(xld.WALData)); end > clientPos { - clientPos = end + if result.safe { + if end := xld.WALStart + pglogrepl.LSN(len(xld.WALData)); end > safePos { + safePos = end + } } + default: + s.fail(ctx, status, fmt.Errorf("%w: copy data kind %q", ErrUnsupportedMessage, cd.Data[0])) + return } } } @@ -407,15 +595,22 @@ func (s *Source) reportLag(ctx context.Context, adminDB *sql.DB, mc metrics.Coll return } if mc != nil { - mc.GaugeSet(retainedWALGauge, float64(retained), metrics.Labels{"slot": s.slot}) + mc.GaugeSet(retainedWALGauge, float64(retained), metrics.Labels{"source": s.name}) } } func (s *Source) fail(_ context.Context, status chan any, err error) { + if err == nil { + err = ErrSourceClosed + } + s.mu.Lock() + s.sourceErr = err + s.mu.Unlock() s.log.Error("cdc stream error", zap.String("slot", s.slot), zap.Error(err)) if s.coll != nil { - s.coll.CounterInc(errorsCounter, metrics.Labels{"slot": s.slot}) + s.coll.CounterInc(errorsCounter, metrics.Labels{"source": s.name}) } + s.closeSubscriptionsWithError(err) select { case status <- err: default: @@ -445,6 +640,18 @@ func (s *Source) prepareSlot( if err != nil { return 0, "", err } + if exists && !resumed { + // A persistent slot is the server-side durable cursor. Never fall + // back to the current system WAL position when local checkpoint + // state is missing; doing so can skip retained logical changes. + confirmed, valid, err := slotConfirmedFlush(ctx, adminDB, s.slot) + if err != nil { + return 0, "", err + } + if valid { + start = confirmed + } + } } snapshotName := "" @@ -646,6 +853,25 @@ func slotExists(ctx context.Context, adminDB *sql.DB, slot string) (bool, error) return n > 0, nil } +func slotConfirmedFlush(ctx context.Context, adminDB *sql.DB, slot string) (pglogrepl.LSN, bool, error) { + var raw sql.NullString + err := adminDB.QueryRowContext(ctx, + `SELECT confirmed_flush_lsn::text + FROM pg_replication_slots + WHERE slot_name = $1`, slot).Scan(&raw) + if err != nil { + return 0, false, fmt.Errorf("read slot confirmed flush position: %w", err) + } + if !raw.Valid || raw.String == "" { + return 0, false, nil + } + lsn, err := pglogrepl.ParseLSN(raw.String) + if err != nil { + return 0, false, fmt.Errorf("parse slot confirmed flush position %q: %w", raw.String, err) + } + return lsn, true, nil +} + func (s *Source) ensurePublication(ctx context.Context, adminDB *sql.DB) (string, error) { if s.publication != "" { return s.publication, nil @@ -688,6 +914,12 @@ func (s *Source) abortFreshSnapshot(conn *pgconn.PgConn) { } func (s *Source) dropSlotAndCheckpoint(ctx context.Context) error { + s.dropMu.Lock() + defer s.dropMu.Unlock() + if s.dropDone.Load() { + return nil + } + adminDB, err := sql.Open("postgres", s.adminDSN) if err != nil { return fmt.Errorf("open admin connection for slot drop: %w", err) @@ -704,11 +936,13 @@ func (s *Source) dropSlotAndCheckpoint(ctx context.Context) error { if err := s.injectedCP.Delete(ctx, s.slot); err != nil { return fmt.Errorf("delete checkpoint: %w", err) } + s.dropDone.Store(true) return nil } if _, err := adminDB.ExecContext(ctx, `DELETE FROM wippy_cdc_offsets WHERE slot = $1`, s.slot); err != nil { return fmt.Errorf("delete checkpoint: %w", err) } + s.dropDone.Store(true) return nil } @@ -722,6 +956,11 @@ func dropReplicationSlot(ctx context.Context, adminDB *sql.DB, slot string) erro lastErr = err var pqErr *pq.Error + if errors.As(err, &pqErr) && string(pqErr.Code) == "42704" { + // Delete is intentionally idempotent. A source can have already + // dropped its slot during Stop before the manager retries Dispose. + return nil + } if !errors.As(err, &pqErr) || string(pqErr.Code) != slotActiveSQLState { return err } diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 4f5a36a60..065609306 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -4,6 +4,7 @@ package postgres import ( "context" + "errors" "strings" "sync" "sync/atomic" @@ -16,6 +17,11 @@ const ( maxStreamBuffer = 65536 ) +// errSubscriberOverflow is terminal for one subscription only. A consumer +// that cannot keep up must not back-pressure the replication receive loop or +// unrelated subscribers. +var errSubscriberOverflow = errors.New("postgres cdc subscriber backlog overflow") + type sourceSubscription struct { source *Source in chan config.Change @@ -26,9 +32,11 @@ type sourceSubscription struct { id uint64 once sync.Once closed atomic.Bool + errMu sync.RWMutex + err error } -func (s *Source) Subscribe(opts config.StreamOptions) config.ChangeStream { +func (s *Source) Subscribe(opts config.StreamOptions) config.Stream { buffer := opts.Buffer if buffer <= 0 { buffer = defaultStreamBuffer @@ -77,6 +85,10 @@ func (s *Source) removeSubscription(id uint64) { } func (s *Source) closeSubscriptions() { + s.closeSubscriptionsWithError(nil) +} + +func (s *Source) closeSubscriptionsWithError(err error) { s.subMu.Lock() subs := make([]*sourceSubscription, 0, len(s.subs)) for id, sub := range s.subs { @@ -86,7 +98,7 @@ func (s *Source) closeSubscriptions() { s.subMu.Unlock() for _, sub := range subs { - sub.Close() + sub.closeWithError(err) } } @@ -95,8 +107,23 @@ func (s *sourceSubscription) Changes() <-chan config.Change { } func (s *sourceSubscription) Close() { + s.closeWithError(nil) +} + +func (s *sourceSubscription) Err() error { + s.errMu.RLock() + defer s.errMu.RUnlock() + return s.err +} + +func (s *sourceSubscription) closeWithError(err error) { s.once.Do(func() { s.closed.Store(true) + if err != nil { + s.errMu.Lock() + s.err = err + s.errMu.Unlock() + } if s.source != nil { s.source.removeSubscription(s.id) } @@ -125,14 +152,17 @@ func (s *sourceSubscription) run() { } } -func (s *sourceSubscription) send(ctx context.Context, change config.Change) { +func (s *sourceSubscription) send(_ context.Context, change config.Change) { if s.closed.Load() { return } select { case s.in <- change: - case <-s.done: - case <-ctx.Done(): + default: + // Never wait for a slow consumer from the replication goroutine. The + // subscription gets a terminal error and is removed; other consumers + // continue receiving the transaction. + s.closeWithError(errSubscriberOverflow) } } diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 2342d833f..ff876388d 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -4,6 +4,7 @@ package postgres import ( "context" + "errors" "testing" "time" @@ -80,3 +81,54 @@ func TestSourceSubscriptionCloseReleasesChannel(t *testing.T) { t.Fatal("timed out waiting for closed cdc stream") } } + +func TestSourceSubscriptionRetainsTerminalError(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + stream := src.Subscribe(cdcapi.StreamOptions{}) + err := errors.New("replication failed") + src.closeSubscriptionsWithError(err) + + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok) + case <-time.After(time.Second): + t.Fatal("stream did not close") + } + assert.ErrorIs(t, stream.(interface{ Err() error }).Err(), err) +} + +func TestSourceSubscriptionOverflowIsBoundedAndLocal(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + laggard := src.Subscribe(cdcapi.StreamOptions{Buffer: 1}) + reader := src.Subscribe(cdcapi.StreamOptions{Buffer: maxStreamBuffer}) + + done := make(chan struct{}) + go func() { + for i := 0; i < 1000; i++ { + src.publishChange(context.Background(), cdcapi.Change{ + Op: "insert", + Table: "accounts", + Relation: "public.accounts", + }) + } + close(done) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("replication fan-out blocked on a slow subscriber") + } + + assert.Eventually(t, func() bool { + return errors.Is(laggard.(interface{ Err() error }).Err(), errSubscriberOverflow) + }, time.Second, time.Millisecond, "laggard must terminate with an overflow error") + select { + case _, ok := <-reader.Changes(): + assert.True(t, ok, "an unrelated subscriber must remain active") + case <-time.After(time.Second): + t.Fatal("unrelated subscriber did not receive a change") + } + laggard.Close() + reader.Close() +} From ca2732bf64e6011aeda58a964479fd275b355bf4 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:15:13 -0400 Subject: [PATCH 09/47] fix(cdc): enforce source lifecycle and stream contracts --- api/service/cdc/context.go | 20 ++--- service/cdc/manager.go | 75 ++++++++++++++--- service/cdc/manager_test.go | 161 ++++++++++++++++++++++++++++++++++++ service/cdc/slot.go | 31 ++++++- service/cdc/stream.go | 5 +- system/cdc/registry.go | 18 +++- system/cdc/registry_test.go | 17 ++++ 7 files changed, 296 insertions(+), 31 deletions(-) diff --git a/api/service/cdc/context.go b/api/service/cdc/context.go index 234f138da..f76797e8e 100644 --- a/api/service/cdc/context.go +++ b/api/service/cdc/context.go @@ -33,22 +33,22 @@ type Capabilities struct { Coalesced bool `json:"coalesced,omitempty"` } -// Stream is the driver-neutral event stream. Err is intentionally optional so -// existing stream implementations remain source-compatible; new sources may -// implement ErrStream to expose a terminal cause without synthesizing an -// error-valued row. +// Stream is the driver-neutral event stream. Err reports the terminal cause +// after Changes is closed; a normal caller-initiated Close has a nil error. +// Keeping the terminal state on the stream avoids synthesizing an +// error-valued change row and gives every registry-backed source the same +// failure contract. type Stream interface { Changes() <-chan Change Close() -} - -// ErrStream exposes a terminal stream error. A consumer should type-assert -// this interface after Changes is closed. -type ErrStream interface { - Stream Err() error } +// ErrStream is retained as a deprecated compatibility alias. Stream now +// requires Err directly; ChangeStream below remains the legacy stream shape +// used by pre-registry SourceStreamer implementations. +type ErrStream = Stream + // Source is the common source contract implemented by every CDC driver. // Subscribe receives a context so a source can reject subscriptions while it // is not ready and can bind snapshot work to the caller's lifetime. diff --git a/service/cdc/manager.go b/service/cdc/manager.go index 0db607ecc..c8b092b76 100644 --- a/service/cdc/manager.go +++ b/service/cdc/manager.go @@ -9,6 +9,8 @@ import ( "context" "errors" "fmt" + "reflect" + "sort" "sync" "github.com/wippyai/runtime/api/event" @@ -27,6 +29,7 @@ var ( ErrUnsupportedKind = errors.New("cdc manager: unsupported source kind") ErrSourceExists = errors.New("cdc manager: source already exists") ErrSourceNotFound = errors.New("cdc manager: source not found") + ErrSourceKindChange = errors.New("cdc manager: source kind cannot change") ) // Dependencies are the shared collaborators available to concrete drivers. @@ -160,7 +163,7 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { if err != nil { return err } - if source == nil { + if isNilSource(source) { return ErrDriverRequired } slot := newSourceSlot(id, entry.Kind, source, m.log.With(zap.String("id", id.String()))) @@ -182,33 +185,44 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) } id := canonicalID(entry.ID) - _, exists := m.registry.Get(id) + existing, exists := m.registry.Get(id) if !exists { return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) } - + existingKind, knownKind := sourceKind(existing) + if !knownKind { + return errors.New("cdc manager: registered source has no kind") + } + if existingKind != entry.Kind { + return fmt.Errorf("%w: %s -> %s", ErrSourceKindChange, existingKind, entry.Kind) + } // Build the replacement before changing visibility. A malformed entry or // failed dependency acquisition leaves the old source untouched. replacement, err := driver.Create(ctx, entry, m.deps) if err != nil { return err } - if replacement == nil { + if isNilSource(replacement) { return ErrDriverRequired } - slot, ok := m.registry.Get(id) - if !ok { - _ = replacement.Stop(ctx) - return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) - } - managedSlot, ok := slot.(*sourceSlot) + managedSlot, ok := existing.(*sourceSlot) if !ok { _ = replacement.Stop(ctx) return errors.New("cdc manager: source is not managed by a stable slot") } + oldLifecycle := normalizeLifecycleConfig(managedSlot.LifecycleConfig()) if err := managedSlot.Replace(ctx, replacement); err != nil { return err } + newLifecycle := normalizeLifecycleConfig(managedSlot.LifecycleConfig()) + if !reflect.DeepEqual(oldLifecycle, newLifecycle) { + // ServiceUpdate is emitted by the supervisor for status changes and is + // not a reconfiguration primitive. Re-register the same stable slot in + // order, so its controller rebuilds security/dependency/autostart state + // without exposing a second source identity. + m.unregisterSupervisor(ctx, id) + m.registerSupervisorWithConfig(ctx, id, managedSlot, newLifecycle) + } m.log.Info("updated cdc source", zap.String("id", id.String()), zap.String("kind", entry.Kind)) return nil } @@ -256,7 +270,10 @@ func (m *Manager) registerSupervisor(ctx context.Context, id registry.ID, source }); ok { cfg = configured.LifecycleConfig() } - cfg.InitDefaults() + m.registerSupervisorWithConfig(ctx, id, source, normalizeLifecycleConfig(cfg)) +} + +func (m *Manager) registerSupervisorWithConfig(ctx context.Context, id registry.ID, source ManagedSource, cfg supervisor.LifecycleConfig) { m.bus.Send(ctx, event.Event{ System: supervisor.System, Kind: supervisor.ServiceRegister, @@ -277,16 +294,52 @@ func (m *Manager) unregisterSupervisor(ctx context.Context, id registry.ID) { } func stopSource(ctx context.Context, source api.Source) error { + if isNilSource(source) { + return nil + } if managed, ok := source.(supervisor.Service); ok { return managed.Stop(ctx) } return nil } +func isNilSource(source api.Source) bool { + if source == nil { + return true + } + v := reflect.ValueOf(source) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + default: + return false + } +} + func canonicalID(id registry.ID) registry.ID { return registry.ParseID(id.String()) } +func sourceKind(source api.Source) (registry.Kind, bool) { + if isNilSource(source) { + return "", false + } + if slot, ok := source.(*sourceSlot); ok { + return slot.kind, slot.kind != "" + } + info := source.Info() + return info.Kind, info.Kind != "" +} + +func normalizeLifecycleConfig(cfg supervisor.LifecycleConfig) supervisor.LifecycleConfig { + cfg.InitDefaults() + dependencies := cfg.RequiredServices() + sort.Strings(dependencies) + cfg.Requires = dependencies + cfg.DependsOn = nil + return cfg +} + var ( _ registry.EntryListener = (*Manager)(nil) _ api.Registry = (*Manager)(nil) diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go index b8a3c98a4..1eabfa429 100644 --- a/service/cdc/manager_test.go +++ b/service/cdc/manager_test.go @@ -25,6 +25,7 @@ type testStream struct { func (s *testStream) Changes() <-chan api.Change { return s.changes } func (s *testStream) Close() { close(s.changes) } +func (s *testStream) Err() error { return nil } type managedTestSource struct { info api.SourceInfo @@ -36,6 +37,7 @@ type managedTestSource struct { stopCount atomic.Int32 active atomic.Int32 maxActive atomic.Int32 + lifecycle *supervisor.LifecycleConfig } type disposableTestSource struct { @@ -85,6 +87,9 @@ func (s *managedTestSource) Stop(context.Context) error { } func (s *managedTestSource) LifecycleConfig() supervisor.LifecycleConfig { + if s.lifecycle != nil { + return *s.lifecycle + } return supervisor.LifecycleConfig{AutoStart: true} } @@ -262,6 +267,34 @@ func TestManagerUpdateBuildFailureLeavesOldSource(t *testing.T) { require.EqualValues(t, 0, old.stopCount.Load()) } +func TestManagerUpdateRejectsEntryKindChange(t *testing.T) { + oldKind := registry.Kind("db.cdc.old") + newKind := registry.Kind("db.cdc.new") + var replacementCreates atomic.Int32 + old := &managedTestSource{info: api.SourceInfo{Engine: "old"}} + oldDriver := testDriver{ + kind: oldKind, + create: func(registry.Entry) (ManagedSource, error) { return old, nil }, + } + newDriver := testDriver{ + kind: newKind, + create: func(registry.Entry) (ManagedSource, error) { + replacementCreates.Add(1) + return &managedTestSource{info: api.SourceInfo{Engine: "new"}}, nil + }, + } + m, _ := newManagerTest(t, oldDriver) + m.drivers[newKind] = newDriver + id := registry.NewID("app", "events") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: id, Kind: oldKind})) + + err := m.Update(context.Background(), registry.Entry{ID: id, Kind: newKind}) + require.ErrorIs(t, err, ErrSourceKindChange) + require.EqualValues(t, 0, replacementCreates.Load(), "a kind-changing update must not construct another driver") + slot := mustSlot(t, m, id) + require.Same(t, old, slot.currentSource()) +} + func TestManagerUpdateAtomicallyReplacesAndStopsOld(t *testing.T) { var next int var created []*managedTestSource @@ -340,6 +373,57 @@ func TestManagerUpdateKeepsStableSupervisorRegistration(t *testing.T) { require.Same(t, current, registered.Service) } +func TestManagerUpdateReRegistersSupervisorWhenLifecycleChanges(t *testing.T) { + oldLifecycle := supervisor.LifecycleConfig{ + AutoStart: true, + Requires: []string{"service-a"}, + } + newLifecycle := supervisor.LifecycleConfig{ + AutoStart: false, + DependsOn: []string{"service-b"}, + StartTimeout: 20 * time.Second, + } + old := &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + lifecycle: &oldLifecycle, + } + candidate := &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + lifecycle: &newLifecycle, + } + next := 0 + driver := testDriver{ + kind: "db.cdc.test", + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: driver.kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + + events := bus.snapshot() + require.Len(t, events, 3) + require.Equal(t, supervisor.ServiceRegister, events[0].Kind) + require.Equal(t, supervisor.ServiceRemove, events[1].Kind) + require.Equal(t, supervisor.ServiceRegister, events[2].Kind) + require.Equal(t, id.String(), events[1].Path) + require.Equal(t, id.String(), events[2].Path) + registered := events[2].Data.(*supervisor.Entry) + require.Same(t, mustSlot(t, m, id), registered.Service) + require.False(t, registered.Config.AutoStart) + require.Equal(t, []string{"service-b"}, registered.Config.Requires) + require.Empty(t, registered.Config.DependsOn) +} + func TestManagerUpdateFailedStartRetainsRunningGeneration(t *testing.T) { var next int old := &managedTestSource{info: api.SourceInfo{Name: "old"}} @@ -402,6 +486,7 @@ func TestManagerUpdateSameExclusiveKeyStopsAndRestores(t *testing.T) { require.EqualValues(t, 1, old.stopCount.Load()) require.EqualValues(t, 2, old.startCount.Load(), "old generation must be restored after its initial start") require.EqualValues(t, 1, old.maxActive.Load(), "exclusive generations must never overlap") + require.Equal(t, "2", slot.Info().Generation, "restoring old ownership creates a new stream generation") } func TestManagerUpdateSameExclusiveKeyStopFailureFaultsSlot(t *testing.T) { @@ -458,6 +543,82 @@ func TestSourceSlotStampsCanonicalIdentityAndGeneration(t *testing.T) { require.NoError(t, slot.Stop(context.Background())) } +func TestSourceSlotInfoNormalizesLegacyAliases(t *testing.T) { + id := registry.NewID("app", "events") + source := &managedTestSource{info: api.SourceInfo{ + Name: "driver-alias", + Engine: "driver-engine", + Epoch: "driver-epoch", + Streaming: true, + Faulted: true, + DBResource: "resource:db", + }} + slot := newSourceSlot(id, "db.cdc.test", source) + + info := slot.Info() + require.Equal(t, id, info.ID) + require.Equal(t, id.String(), info.Name) + require.Equal(t, "1", info.Generation) + require.Equal(t, "1", info.Epoch) + require.Equal(t, api.SourceStateUnknown, info.State) + require.False(t, info.Streaming) + require.False(t, info.Faulted) + require.Equal(t, "driver-engine", info.Engine) + require.Equal(t, "resource:db", info.DBResource) + + slot.mu.Lock() + slot.state = slotRunning + slot.mu.Unlock() + info = slot.Info() + require.Equal(t, api.SourceStateRunning, info.State) + require.True(t, info.Streaming) + require.False(t, info.Faulted) + + slot.mu.Lock() + slot.state = slotFaulted + slot.mu.Unlock() + info = slot.Info() + require.Equal(t, api.SourceStateFaulted, info.State) + require.False(t, info.Streaming) + require.True(t, info.Faulted) +} + +func TestSourceSlotInfoHandlesNilSource(t *testing.T) { + id := registry.NewID("app", "events") + info := newSourceSlot(id, "db.cdc.test", nil).Info() + require.Equal(t, id, info.ID) + require.Equal(t, id.String(), info.Name) + require.Equal(t, "1", info.Generation) + require.Equal(t, "1", info.Epoch) + require.Equal(t, api.SourceStateUnknown, info.State) +} + +func TestSourceSlotRestartAdvancesGeneration(t *testing.T) { + id := registry.NewID("app", "events") + source := &managedTestSource{stream: &testStream{changes: make(chan api.Change, 1)}} + slot := newSourceSlot(id, "db.cdc.test", source) + + _, err := slot.Start(context.Background()) + require.NoError(t, err) + require.Equal(t, "1", slot.Info().Generation) + require.NoError(t, slot.Stop(context.Background())) + + _, err = slot.Start(context.Background()) + require.NoError(t, err) + require.Equal(t, "2", slot.Info().Generation) + stream, err := slot.Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + source.stream.changes <- api.Change{Op: "insert"} + select { + case change := <-stream.Changes(): + require.Equal(t, "2", change.Generation) + case <-time.After(time.Second): + t.Fatal("timed out waiting for restarted stream") + } + stream.Close() + require.NoError(t, slot.Stop(context.Background())) +} + func mustSlot(t *testing.T, m *Manager, id registry.ID) *sourceSlot { t.Helper() source, ok := m.Get(id) diff --git a/service/cdc/slot.go b/service/cdc/slot.go index d15aaeb9d..a00ea2aa4 100644 --- a/service/cdc/slot.go +++ b/service/cdc/slot.go @@ -73,25 +73,35 @@ func (s *sourceSlot) Info() api.SourceInfo { generation := s.generation state := s.state s.mu.RUnlock() - if current == nil { + if isNilSource(current) { return api.SourceInfo{ ID: s.id, Kind: s.kind, + Name: s.id.String(), Generation: generationString(generation), State: sourceState(state), + Streaming: state == slotRunning, + Faulted: state == slotFaulted, + Epoch: generationString(generation), } } info := current.Info() info.ID = s.id info.Kind = s.kind + info.Name = s.id.String() info.Generation = generationString(generation) info.State = sourceState(state) + info.Streaming = state == slotRunning + info.Faulted = state == slotFaulted + if info.Generation != "" { + info.Epoch = info.Generation + } return info } func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api.Stream, error) { s.mu.RLock() - if s.state != slotRunning || s.current == nil || s.disposing { + if s.state != slotRunning || isNilSource(s.current) || s.disposing { s.mu.RUnlock() return nil, api.ErrSourceNotReady } @@ -140,11 +150,12 @@ func (s *sourceSlot) Start(ctx context.Context) (<-chan any, error) { s.mu.Unlock() return nil, ErrSourceBusy } - if s.current == nil { + if isNilSource(s.current) { s.mu.Unlock() return nil, ErrSourceClosed } current := s.current + restart := s.state == slotStopped || s.state == slotFaulted status := make(chan any, 8) s.status = status s.statusDone = false @@ -167,6 +178,9 @@ func (s *sourceSlot) Start(ctx context.Context) (<-chan any, error) { s.mu.Lock() s.state = slotRunning + if restart { + s.generation++ + } generation := s.generation s.mu.Unlock() s.watchStatus(current, generation, underlying) @@ -238,7 +252,9 @@ func (s *sourceSlot) Dispose(ctx context.Context) error { cancel() } var err error - if disposable, ok := current.(Disposable); ok { + if isNilSource(current) { + err = ErrSourceClosed + } else if disposable, ok := current.(Disposable); ok { err = disposable.Dispose(ctx) } else { err = stopSource(ctx, current) @@ -328,6 +344,7 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error if restartErr == nil { s.mu.Lock() s.state = slotRunning + s.generation++ s.replacing = false generation := s.generation s.mu.Unlock() @@ -386,6 +403,9 @@ func (s *sourceSlot) LifecycleConfig() supervisor.LifecycleConfig { s.mu.RLock() current := s.current s.mu.RUnlock() + if isNilSource(current) { + return supervisor.LifecycleConfig{} + } if configured, ok := current.(interface { LifecycleConfig() supervisor.LifecycleConfig }); ok { @@ -423,6 +443,9 @@ func (s *sourceSlot) watchStatus(source ManagedSource, generation uint64, update } func exclusiveResourceKey(source ManagedSource) string { + if isNilSource(source) { + return "" + } if keyed, ok := source.(ExclusiveResource); ok { return keyed.ExclusiveResourceKey() } diff --git a/service/cdc/stream.go b/service/cdc/stream.go index febf488d0..4b2adda51 100644 --- a/service/cdc/stream.go +++ b/service/cdc/stream.go @@ -57,10 +57,7 @@ func (s *stampedStream) Close() { } func (s *stampedStream) Err() error { - if withError, ok := s.upstream.(interface{ Err() error }); ok { - return withError.Err() - } - return nil + return s.upstream.Err() } func (s *stampedStream) run() { diff --git a/system/cdc/registry.go b/system/cdc/registry.go index 232766966..8b7405cc8 100644 --- a/system/cdc/registry.go +++ b/system/cdc/registry.go @@ -7,6 +7,7 @@ package cdc import ( "errors" + "reflect" "sort" "sync" @@ -48,7 +49,7 @@ func NewRegistry(log *zap.Logger) *Registry { // must use Replace for an update so an accidental duplicate cannot orphan a // running source. func (r *Registry) Register(id registry.ID, source api.Source, kind registry.Kind) error { - if source == nil { + if nilSource(source) { return errors.New("cdc source is nil") } id = canonicalID(id) @@ -67,7 +68,7 @@ func (r *Registry) Register(id registry.ID, source api.Source, kind registry.Kin // visible source. The old source is not stopped here: lifecycle ownership stays // with service/cdc, which can stop it after the visibility swap. func (r *Registry) Replace(id registry.ID, source api.Source, kind registry.Kind) (api.Source, bool, error) { - if source == nil { + if nilSource(source) { return nil, false, errors.New("cdc source is nil") } id = canonicalID(id) @@ -149,4 +150,17 @@ func canonicalID(id registry.ID) registry.ID { return registry.ParseID(id.String()) } +func nilSource(source api.Source) bool { + if source == nil { + return true + } + v := reflect.ValueOf(source) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return v.IsNil() + default: + return false + } +} + var _ api.Registry = (*Registry)(nil) diff --git a/system/cdc/registry_test.go b/system/cdc/registry_test.go index babe22617..bf04130c3 100644 --- a/system/cdc/registry_test.go +++ b/system/cdc/registry_test.go @@ -39,6 +39,23 @@ func TestRegistryCanonicalIDAndDuplicateProtection(t *testing.T) { require.ErrorIs(t, r.Register(id, newTestSource("other"), "db.cdc.test"), ErrSourceExists) } +func TestRegistryRejectsTypedNilSource(t *testing.T) { + r := NewRegistry(nil) + id := registry.NewID("app", "events") + + var nilSource *testSource + require.Error(t, r.Register(id, nilSource, "db.cdc.test")) + + actual := newTestSource("actual") + require.NoError(t, r.Register(id, actual, "db.cdc.test")) + _, replaced, err := r.Replace(id, nilSource, "db.cdc.test") + require.Error(t, err) + require.False(t, replaced) + got, ok := r.Get(id) + require.True(t, ok) + require.Same(t, actual, got) +} + func TestRegistryReplaceIsAtomicAndReturnsOld(t *testing.T) { r := NewRegistry(nil) id := registry.NewID("app", "events") From 7cdd57c7ca3868c54d12456252897f6d14b3a13e Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:27:36 -0400 Subject: [PATCH 10/47] fix(cdc/postgres): clean fresh slots and reconcile publications --- service/cdc/postgres/driver.go | 39 ++--- service/cdc/postgres/driver_test.go | 8 + service/cdc/postgres/errors.go | 6 + service/cdc/postgres/identifiers.go | 47 ++++++ service/cdc/postgres/identifiers_test.go | 41 +++++ .../postgres/integration_lifecycle_test.go | 127 +++++++++++++++ service/cdc/postgres/limits.go | 37 +++++ service/cdc/postgres/service.go | 154 ++++++++++++++---- 8 files changed, 405 insertions(+), 54 deletions(-) create mode 100644 service/cdc/postgres/identifiers.go create mode 100644 service/cdc/postgres/identifiers_test.go create mode 100644 service/cdc/postgres/integration_lifecycle_test.go create mode 100644 service/cdc/postgres/limits.go diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index e1ea8bb56..a9973c2d0 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -4,7 +4,6 @@ package postgres import ( "context" - "errors" "fmt" "net" "strconv" @@ -39,6 +38,23 @@ func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice. if err := cfg.Validate(); err != nil { return nil, NewInvalidConfigError(err) } + if err := validatePostgresIdentifier(cfg.SlotName, "slot_name"); err != nil { + return nil, NewInvalidConfigError(err) + } + if cfg.Publication != "" { + if err := validatePostgresIdentifier(cfg.Publication, "publication"); err != nil { + return nil, NewInvalidConfigError(err) + } + } else { + for _, table := range cfg.Tables { + if _, err := quoteQualifiedIdent(table); err != nil { + return nil, NewInvalidConfigError(err) + } + } + if _, err := quotePostgresIdentifier(cfg.SlotName+"_pub", "publication"); err != nil { + return nil, NewInvalidConfigError(err) + } + } standby, _ := cfg.StandbyDuration() status, _ := cfg.StatusDuration() replDSN, adminDSN, err := buildDSNs(cfg) @@ -69,7 +85,6 @@ func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice. } return &sourceAdapter{ source: NewSource(opts), - opts: opts, lifecycle: cfg.Lifecycle, exclusiveKey: postgresExclusiveKey(cfg), }, nil @@ -82,7 +97,6 @@ func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice. type sourceAdapter struct { mu sync.RWMutex source *Source - opts SourceOptions lifecycle supervisor.LifecycleConfig exclusiveKey string } @@ -161,24 +175,7 @@ func (s *sourceAdapter) Subscribe(ctx context.Context, opts config.StreamOptions func (s *sourceAdapter) Start(ctx context.Context) (<-chan any, error) { s.mu.RLock() source := s.source - opts := s.opts s.mu.RUnlock() - status, err := source.Start(ctx) - if !errors.Is(err, ErrSourceClosed) { - return status, err - } - - // Source deliberately makes a stopped generation terminal so a stale - // replication connection can never be reused. The stable manager slot can - // still restart the logical generation by constructing a fresh source with - // the same immutable configuration and checkpoint identity. - fresh := NewSource(opts) - s.mu.Lock() - if s.source == source { - s.source = fresh - } - source = s.source - s.mu.Unlock() return source.Start(ctx) } @@ -217,7 +214,7 @@ func (s *sourceAdapter) ExclusiveResourceKey() string { func postgresExclusiveKey(cfg *config.Config) string { host := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(cfg.Host), ".")) endpoint := net.JoinHostPort(host, strconv.Itoa(cfg.Port)) - return "postgres/" + endpoint + "/" + cfg.Database + "/slot/" + cfg.SlotName + return "postgres/" + endpoint + "/slot/" + cfg.SlotName } func postgresSourceState(state sourceState) config.SourceState { diff --git a/service/cdc/postgres/driver_test.go b/service/cdc/postgres/driver_test.go index 0a04268f7..e4ece526e 100644 --- a/service/cdc/postgres/driver_test.go +++ b/service/cdc/postgres/driver_test.go @@ -49,3 +49,11 @@ func TestSourceStopAfterDisposeCleanupIsIdempotent(t *testing.T) { require.NoError(t, source.Stop(nil)) require.NoError(t, source.Stop(nil)) } + +func TestPostgresExclusiveKeyIsClusterWide(t *testing.T) { + first := &cconfig.Config{Host: "db.internal", Port: 5432, Database: "one", SlotName: "events"} + second := &cconfig.Config{Host: "db.internal", Port: 5432, Database: "two", SlotName: "events"} + + assert.Equal(t, postgresExclusiveKey(first), postgresExclusiveKey(second)) + assert.NotContains(t, postgresExclusiveKey(first), "password") +} diff --git a/service/cdc/postgres/errors.go b/service/cdc/postgres/errors.go index 43174e65c..2baccc89b 100644 --- a/service/cdc/postgres/errors.go +++ b/service/cdc/postgres/errors.go @@ -12,8 +12,14 @@ import ( var ( ErrUnknownRelation = errors.New("cdc: unknown relation id") + ErrInvalidTransaction = errors.New("cdc: invalid transaction message sequence") + ErrTransactionLimit = errors.New("cdc: transaction buffer limit exceeded") + ErrUnsupportedMessage = errors.New("cdc: unsupported logical replication message") ErrSourceClosed = errors.New("cdc: source is closed") + ErrSourceRunning = errors.New("cdc: source is already running") + ErrSourceStopping = errors.New("cdc: source is stopping") ErrNoPublication = errors.New("cdc: no publication and no tables configured") + ErrInvalidIdentifier = errors.New("cdc: invalid PostgreSQL identifier") ErrTranscoderRequired = apierror.New(apierror.Invalid, "transcoder is required").WithRetryable(apierror.False) ErrEventBusRequired = apierror.New(apierror.Invalid, "event bus is required").WithRetryable(apierror.False) ErrNoSourceStreamer = apierror.New(apierror.Internal, "cdc source streamer not available").WithRetryable(apierror.False) diff --git a/service/cdc/postgres/identifiers.go b/service/cdc/postgres/identifiers.go new file mode 100644 index 000000000..6cdb66d7c --- /dev/null +++ b/service/cdc/postgres/identifiers.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "github.com/lib/pq" +) + +// PostgreSQL stores ordinary identifiers in NameData, whose default +// NAMEDATALEN leaves 63 bytes for the identifier. Replication slot and +// publication names are identifiers in the replication command grammar. +const postgresIdentifierMaxBytes = 63 + +func quotePostgresIdentifier(value, field string) (string, error) { + if value == "" || value != strings.TrimSpace(value) || + len(value) > postgresIdentifierMaxBytes || !utf8.ValidString(value) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + for _, r := range value { + if unicode.IsControl(r) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + } + return pq.QuoteIdentifier(value), nil +} + +func validatePostgresIdentifier(value, field string) error { + _, err := quotePostgresIdentifier(value, field) + return err +} + +func quotePostgresLiteral(value, field string) (string, error) { + if value == "" || !utf8.ValidString(value) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + for _, r := range value { + if r == 0 || unicode.IsControl(r) { + return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) + } + } + return pq.QuoteLiteral(value), nil +} diff --git a/service/cdc/postgres/identifiers_test.go b/service/cdc/postgres/identifiers_test.go new file mode 100644 index 000000000..5aa8ac00a --- /dev/null +++ b/service/cdc/postgres/identifiers_test.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestQuotePostgresIdentifierRejectsUnsafeNames(t *testing.T) { + for _, name := range []string{"", " slot", "slot ", "slot\nname", string([]byte{0xff})} { + _, err := quotePostgresIdentifier(name, "slot_name") + assert.ErrorIs(t, err, ErrInvalidIdentifier, "name %q", name) + } + _, err := quotePostgresIdentifier(strings.Repeat("x", postgresIdentifierMaxBytes+1), "slot_name") + assert.ErrorIs(t, err, ErrInvalidIdentifier) +} + +func TestQuotePostgresIdentifierUsesServerIdentifierQuoting(t *testing.T) { + quoted, err := quotePostgresIdentifier(`slot"name`, "slot_name") + require.NoError(t, err) + assert.Equal(t, `"slot""name"`, quoted) + + literal, err := quotePostgresLiteral(`publication'name`, "publication") + require.NoError(t, err) + assert.Equal(t, `'publication''name'`, literal) + + qualified, err := quoteQualifiedIdent("public.accounts") + require.NoError(t, err) + assert.Equal(t, `"public"."accounts"`, qualified) +} + +func TestQuoteQualifiedIdentRejectsMalformedTable(t *testing.T) { + for _, table := range []string{"public.accounts.extra", ".accounts", "public."} { + _, err := quoteQualifiedIdent(table) + assert.ErrorIs(t, err, ErrInvalidIdentifier, "table %q", table) + } +} diff --git a/service/cdc/postgres/integration_lifecycle_test.go b/service/cdc/postgres/integration_lifecycle_test.go new file mode 100644 index 000000000..affb7bbfb --- /dev/null +++ b/service/cdc/postgres/integration_lifecycle_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package postgres + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/lib/pq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + freshFailureSlot = "wippy_cdc_fresh_failure" + autoPublicationSlot = "wippy_cdc_auto_pub" +) + +func TestFreshSlotIsDroppedWhenReplicationStartFails(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + dropNamedSlot(t, repl, freshFailureSlot) + defer dropNamedSlot(t, repl, freshFailureSlot) + + src := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: freshFailureSlot, + Publication: "wippy_cdc_missing_publication", + StandbyInterval: time.Millisecond, StatusInterval: time.Hour, + }) + status, err := src.Start(context.Background()) + require.NoError(t, err, "startup returns before the replication command is issued") + + select { + case <-statusClosed(status): + case <-time.After(10 * time.Second): + t.Fatal("source did not terminate after invalid replication publication") + } + assert.Eventually(t, func() bool { + var count int + if err := db.QueryRow(`SELECT count(*) FROM pg_replication_slots WHERE slot_name=$1`, freshFailureSlot).Scan(&count); err != nil { + return false + } + return count == 0 + }, 5*time.Second, 100*time.Millisecond, "fresh slot must be cleaned after StartReplication failure") +} + +func TestMissingSlotDeletesStaleCheckpointBeforeRecreate(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + dropNamedSlot(t, repl, freshFailureSlot) + defer dropNamedSlot(t, repl, freshFailureSlot) + + _, err = db.Exec(`INSERT INTO wippy_cdc_offsets (slot, lsn) VALUES ($1, $2)`, freshFailureSlot, "F/FFFFFFF") + require.NoError(t, err) + + src := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: freshFailureSlot, + Publication: "wippy_cdc_pub", StandbyInterval: 200 * time.Millisecond, + StatusInterval: time.Hour, + }) + status, err := src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + var count int + require.NoError(t, db.QueryRow(`SELECT count(*) FROM wippy_cdc_offsets WHERE slot=$1`, freshFailureSlot).Scan(&count)) + assert.Zero(t, count, "an offset from a removed slot must not survive recreation") + _ = status +} + +func TestAutoPublicationReconcilesTableMembership(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + const extraTable = "wippy_cdc_auto_extra" + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS ` + pq.QuoteIdentifier(extraTable) + ` (id bigint PRIMARY KEY)`) + require.NoError(t, err) + pubName := autoPublicationSlot + "_pub" + _, err = db.Exec(`DROP PUBLICATION IF EXISTS ` + pq.QuoteIdentifier(pubName)) + require.NoError(t, err) + dropNamedSlot(t, repl, autoPublicationSlot) + defer func() { + dropNamedSlot(t, repl, autoPublicationSlot) + _, _ = db.Exec(`DROP PUBLICATION IF EXISTS ` + pq.QuoteIdentifier(pubName)) + _, _ = db.Exec(`DROP TABLE IF EXISTS ` + pq.QuoteIdentifier(extraTable)) + }() + + first := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: autoPublicationSlot, + Tables: []string{"public.accounts"}, StandbyInterval: 200 * time.Millisecond, + StatusInterval: time.Hour, + }) + _, err = first.Start(context.Background()) + require.NoError(t, err) + require.NoError(t, first.Stop(context.Background())) + + second := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: autoPublicationSlot, + Tables: []string{extraTable}, StandbyInterval: 200 * time.Millisecond, + StatusInterval: time.Hour, + }) + _, err = second.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = second.Stop(context.Background()) }() + + rows, err := db.Query(`SELECT schemaname || '.' || tablename FROM pg_publication_tables WHERE pubname=$1 ORDER BY 1`, pubName) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + var got []string + for rows.Next() { + var table string + require.NoError(t, rows.Scan(&table)) + got = append(got, table) + } + require.NoError(t, rows.Err()) + assert.Equal(t, []string{"public." + extraTable}, got) +} diff --git a/service/cdc/postgres/limits.go b/service/cdc/postgres/limits.go new file mode 100644 index 000000000..b8bf98cab --- /dev/null +++ b/service/cdc/postgres/limits.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MPL-2.0 + +package postgres + +import ( + config "github.com/wippyai/runtime/api/service/cdc" +) + +const ( + // These aliases keep the decoder package independent from entry parsing + // while making the finite defaults owned by the public CDC configuration. + defaultMaxTransactionChanges = config.DefaultPostgresMaxTransactionChanges + defaultMaxTransactionBytes = config.DefaultPostgresMaxTransactionBytes +) + +type decoderLimits struct { + maxChanges int + maxBytes int64 +} + +func defaultDecoderLimits() decoderLimits { + return decoderLimits{ + maxChanges: defaultMaxTransactionChanges, + maxBytes: defaultMaxTransactionBytes, + } +} + +func normalizeDecoderLimits(limits decoderLimits) decoderLimits { + defaults := defaultDecoderLimits() + if limits.maxChanges <= 0 { + limits.maxChanges = defaults.maxChanges + } + if limits.maxBytes <= 0 { + limits.maxBytes = defaults.maxBytes + } + return limits +} diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 1c0fd72e7..a57173c47 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -268,10 +268,13 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { return nil, startErr } - startLSN, snapshotName, err := s.prepareSlot(runCtx, conn, adminDB, cp, sysident.XLogPos) + startLSN, snapshotName, slotCreated, err := s.prepareSlot(runCtx, conn, adminDB, cp, sysident.XLogPos) if err != nil { _ = conn.Close(context.Background()) _ = adminDB.Close() + if slotCreated { + s.cleanupFreshSlot() + } failStart(err) return nil, err } @@ -286,6 +289,9 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { s.mu.Unlock() _ = conn.Close(context.Background()) _ = adminDB.Close() + if slotCreated { + s.cleanupFreshSlot() + } failStart(ErrSourceClosed) return nil, ErrSourceClosed } @@ -304,7 +310,7 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { } s.coll = metrics.GetCollector(runCtx) - go s.run(runCtx, conn, adminDB, cp, startLSN, snapshotName, publication, s.coll, status, done) + go s.run(runCtx, conn, adminDB, cp, startLSN, snapshotName, slotCreated, publication, s.coll, status, done) return status, nil } @@ -315,7 +321,7 @@ func (s *Source) Stop(ctx context.Context) error { s.mu.Lock() if s.state == sourceStopped { - drop := s.dropSlot.Load() && !s.temporary + drop := s.dropSlot.Load() s.mu.Unlock() if drop { return s.dropSlotAndCheckpoint(ctx) @@ -327,7 +333,7 @@ func (s *Source) Stop(ctx context.Context) error { s.cancel = nil s.mu.Unlock() s.closeSubscriptions() - if s.dropSlot.Load() && !s.temporary { + if s.dropSlot.Load() { return s.dropSlotAndCheckpoint(ctx) } return nil @@ -355,7 +361,7 @@ func (s *Source) Stop(ctx context.Context) error { } } - if s.dropSlot.Load() && !s.temporary { + if s.dropSlot.Load() { return s.dropSlotAndCheckpoint(ctx) } return nil @@ -368,6 +374,7 @@ func (s *Source) run( cp Checkpointer, startLSN pglogrepl.LSN, snapshotName string, + slotCreated bool, publication string, mc metrics.Collector, status chan any, @@ -393,7 +400,7 @@ func (s *Source) run( if snapshotName != "" { if err := s.snapshotExisting(ctx, adminDB, publication, snapshotName); err != nil { - s.abortFreshSnapshot(conn) + s.abortFreshSlot(conn, slotCreated) s.fail(ctx, status, err) return } @@ -403,18 +410,28 @@ func (s *Source) run( if s.streaming { protoVersion = config.StreamingProtocolVersion } + publicationLiteral, err := quotePostgresLiteral(publication, "publication") + if err != nil { + s.abortFreshSlot(conn, slotCreated) + s.fail(ctx, status, err) + return + } pluginArgs := []string{ fmt.Sprintf("proto_version '%d'", protoVersion), - fmt.Sprintf("publication_names '%s'", publication), + fmt.Sprintf("publication_names %s", publicationLiteral), } if s.streaming { pluginArgs = append(pluginArgs, "streaming 'on'") } - if err := pglogrepl.StartReplication(ctx, conn, s.slot, startLSN, + slotIdentifier, err := quotePostgresIdentifier(s.slot, "slot_name") + if err != nil { + s.abortFreshSlot(conn, slotCreated) + s.fail(ctx, status, err) + return + } + if err := pglogrepl.StartReplication(ctx, conn, slotIdentifier, startLSN, pglogrepl.StartReplicationOptions{PluginArgs: pluginArgs}); err != nil { - if snapshotName != "" { - s.abortFreshSnapshot(conn) - } + s.abortFreshSlot(conn, slotCreated) s.fail(ctx, status, err) return } @@ -623,11 +640,11 @@ func (s *Source) prepareSlot( adminDB *sql.DB, cp Checkpointer, fallback pglogrepl.LSN, -) (pglogrepl.LSN, string, error) { +) (pglogrepl.LSN, string, bool, error) { var start pglogrepl.LSN resumed := false if cpLSN, ok, err := cp.Load(ctx, s.slot); err != nil { - return 0, "", err + return 0, "", false, err } else if ok { start = cpLSN resumed = true @@ -638,7 +655,17 @@ func (s *Source) prepareSlot( var err error exists, err = slotExists(ctx, adminDB, s.slot) if err != nil { - return 0, "", err + return 0, "", false, err + } + if !exists && resumed { + // A local offset is meaningful only for the server-side slot + // incarnation that produced it. If that slot disappeared, do not + // reuse the old LSN for a newly-created slot. + if err := cp.Delete(ctx, s.slot); err != nil { + return 0, "", false, fmt.Errorf("delete stale cdc checkpoint: %w", err) + } + start = 0 + resumed = false } if exists && !resumed { // A persistent slot is the server-side durable cursor. Never fall @@ -646,28 +673,42 @@ func (s *Source) prepareSlot( // state is missing; doing so can skip retained logical changes. confirmed, valid, err := slotConfirmedFlush(ctx, adminDB, s.slot) if err != nil { - return 0, "", err + return 0, "", false, err } if valid { start = confirmed } } + } else if resumed { + // Temporary slots are destroyed with their replication connection, so + // any persisted offset belongs to an older slot incarnation. + if err := cp.Delete(ctx, s.slot); err != nil { + return 0, "", false, fmt.Errorf("delete stale cdc checkpoint: %w", err) + } + start = 0 + resumed = false } snapshotName := "" + slotCreated := false if !exists { + slotIdentifier, err := quotePostgresIdentifier(s.slot, "slot_name") + if err != nil { + return 0, "", false, err + } opts := pglogrepl.CreateReplicationSlotOptions{Temporary: s.temporary} wantSnapshot := s.snapshot && !resumed if wantSnapshot { opts.SnapshotAction = "EXPORT_SNAPSHOT" } - res, err := pglogrepl.CreateReplicationSlot(ctx, conn, s.slot, config.OutputPlugin, opts) + res, err := pglogrepl.CreateReplicationSlot(ctx, conn, slotIdentifier, config.OutputPlugin, opts) if err != nil { - return 0, "", fmt.Errorf("create replication slot: %w", err) + return 0, "", false, fmt.Errorf("create replication slot: %w", err) } + slotCreated = true cpoint, err := pglogrepl.ParseLSN(res.ConsistentPoint) if err != nil { - return 0, "", fmt.Errorf("parse consistent point %q: %w", res.ConsistentPoint, err) + return 0, "", slotCreated, fmt.Errorf("parse consistent point %q: %w", res.ConsistentPoint, err) } if cpoint > start { start = cpoint @@ -679,18 +720,22 @@ func (s *Source) prepareSlot( if s.failover && !s.temporary { if err := s.setSlotFailover(ctx, conn); err != nil { - return 0, "", err + return 0, "", slotCreated, err } } if start == 0 { start = fallback } - return start, snapshotName, nil + return start, snapshotName, slotCreated, nil } func (s *Source) setSlotFailover(ctx context.Context, conn *pgconn.PgConn) error { - cmd := fmt.Sprintf("ALTER_REPLICATION_SLOT %s ( FAILOVER )", s.slot) + slotIdentifier, err := quotePostgresIdentifier(s.slot, "slot_name") + if err != nil { + return err + } + cmd := fmt.Sprintf("ALTER_REPLICATION_SLOT %s ( FAILOVER )", slotIdentifier) if err := conn.Exec(ctx, cmd).Close(); err != nil { return fmt.Errorf("set slot failover: %w", err) } @@ -874,12 +919,35 @@ func slotConfirmedFlush(ctx context.Context, adminDB *sql.DB, slot string) (pglo func (s *Source) ensurePublication(ctx context.Context, adminDB *sql.DB) (string, error) { if s.publication != "" { + if err := validatePostgresIdentifier(s.publication, "publication"); err != nil { + return "", err + } return s.publication, nil } if len(s.tables) == 0 { return "", ErrNoPublication } name := s.slot + "_pub" + quotedName, err := quotePostgresIdentifier(name, "publication") + if err != nil { + return "", err + } + quotedTables := make([]string, 0, len(s.tables)) + seenTables := make(map[string]struct{}, len(s.tables)) + for _, table := range s.tables { + quotedTable, err := quoteQualifiedIdent(table) + if err != nil { + return "", err + } + if _, exists := seenTables[quotedTable]; exists { + continue + } + seenTables[quotedTable] = struct{}{} + quotedTables = append(quotedTables, quotedTable) + } + if len(quotedTables) == 0 { + return "", ErrNoPublication + } var n int if err := adminDB.QueryRowContext(ctx, @@ -887,28 +955,40 @@ func (s *Source) ensurePublication(ctx context.Context, adminDB *sql.DB) (string return "", fmt.Errorf("check publication: %w", err) } if n == 0 { - quoted := make([]string, len(s.tables)) - for i, t := range s.tables { - quoted[i] = quoteQualifiedIdent(t) - } stmt := fmt.Sprintf("CREATE PUBLICATION %s FOR TABLE %s", - pq.QuoteIdentifier(name), strings.Join(quoted, ", ")) + quotedName, strings.Join(quotedTables, ", ")) if _, err := adminDB.ExecContext(ctx, stmt); err != nil { return "", fmt.Errorf("create publication: %w", err) } + } else { + // The generated name is owned by this source configuration. Reconcile + // its membership exactly on every start so an update cannot silently + // continue publishing an old table set. User-supplied publications take + // the early return above and are never altered or dropped. + stmt := fmt.Sprintf("ALTER PUBLICATION %s SET TABLE %s", + quotedName, strings.Join(quotedTables, ", ")) + if _, err := adminDB.ExecContext(ctx, stmt); err != nil { + return "", fmt.Errorf("reconcile publication: %w", err) + } } return name, nil } -func (s *Source) abortFreshSnapshot(conn *pgconn.PgConn) { - _ = conn.Close(context.Background()) - if s.temporary { +func (s *Source) abortFreshSlot(conn *pgconn.PgConn, created bool) { + if conn != nil { + _ = conn.Close(context.Background()) + } + if !created { return } + s.cleanupFreshSlot() +} + +func (s *Source) cleanupFreshSlot() { cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := s.dropSlotAndCheckpoint(cleanupCtx); err != nil { - s.log.Warn("cdc cleanup after snapshot failure failed", + s.log.Warn("cdc cleanup after fresh slot failure failed", zap.String("slot", s.slot), zap.Error(err)) } } @@ -973,10 +1053,18 @@ func dropReplicationSlot(ctx context.Context, adminDB *sql.DB, slot string) erro return lastErr } -func quoteQualifiedIdent(name string) string { +func quoteQualifiedIdent(name string) (string, error) { parts := strings.Split(name, ".") + if len(parts) < 1 || len(parts) > 2 { + return "", fmt.Errorf("%w: table", ErrInvalidIdentifier) + } + quoted := make([]string, len(parts)) for i, p := range parts { - parts[i] = pq.QuoteIdentifier(p) + quotedPart, err := quotePostgresIdentifier(p, "table") + if err != nil { + return "", err + } + quoted[i] = quotedPart } - return strings.Join(parts, ".") + return strings.Join(quoted, "."), nil } From 0897344c4bb7abd1541fb5ac1abcfb3eaa492f76 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:28:00 -0400 Subject: [PATCH 11/47] fix(cdc): guard replacement resources and exclusive leases --- service/cdc/manager.go | 161 +++++++++++++++++++++----- service/cdc/manager_test.go | 221 +++++++++++++++++++++++++++++++++-- service/cdc/slot.go | 224 ++++++++++++++++++++++++++++++++---- 3 files changed, 546 insertions(+), 60 deletions(-) diff --git a/service/cdc/manager.go b/service/cdc/manager.go index c8b092b76..c80630001 100644 --- a/service/cdc/manager.go +++ b/service/cdc/manager.go @@ -23,13 +23,15 @@ import ( ) var ( - ErrRegistryRequired = errors.New("cdc manager: registry is required") - ErrEventBusRequired = errors.New("cdc manager: event bus is required") - ErrDriverRequired = errors.New("cdc manager: driver is required") - ErrUnsupportedKind = errors.New("cdc manager: unsupported source kind") - ErrSourceExists = errors.New("cdc manager: source already exists") - ErrSourceNotFound = errors.New("cdc manager: source not found") - ErrSourceKindChange = errors.New("cdc manager: source kind cannot change") + ErrRegistryRequired = errors.New("cdc manager: registry is required") + ErrEventBusRequired = errors.New("cdc manager: event bus is required") + ErrDriverRequired = errors.New("cdc manager: driver is required") + ErrUnsupportedKind = errors.New("cdc manager: unsupported source kind") + ErrSourceExists = errors.New("cdc manager: source already exists") + ErrSourceNotFound = errors.New("cdc manager: source not found") + ErrSourceKindChange = errors.New("cdc manager: source kind cannot change") + ErrSourceKindMismatch = errors.New("cdc manager: source kind does not match entry") + ErrExclusiveOwned = errors.New("cdc manager: exclusive resource is already owned") ) // Dependencies are the shared collaborators available to concrete drivers. @@ -107,9 +109,16 @@ type Manager struct { deps Dependencies log *zap.Logger drivers map[registry.Kind]Driver + leases map[string]resourceLease + leaseSeq uint64 mu sync.Mutex } +type resourceLease struct { + id registry.ID + token uint64 +} + func NewManager( reg Registry, dtt payload.Transcoder, @@ -137,6 +146,7 @@ func NewManager( }, log: log, drivers: make(map[registry.Kind]Driver), + leases: make(map[string]resourceLease), } for _, opt := range opts { if opt != nil { @@ -166,8 +176,18 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { if isNilSource(source) { return ErrDriverRequired } + key := exclusiveResourceKey(source) + leaseToken, err := m.reserveLeaseLocked(key, id) + if err != nil { + _ = stopSource(ctx, source) + return err + } slot := newSourceSlot(id, entry.Kind, source, m.log.With(zap.String("id", id.String()))) + slot.setRetiredCleanupHook(func(retiredKey string, retiredToken uint64) { + go m.releaseLease(id, retiredKey, retiredToken) + }) if err := m.registry.Register(id, slot, entry.Kind); err != nil { + m.releaseLeaseLocked(id, key, leaseToken) _ = source.Stop(ctx) return err } @@ -180,22 +200,24 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { m.mu.Lock() defer m.mu.Unlock() + id := canonicalID(entry.ID) + existing, exists := m.registry.Get(id) + if exists { + existingKind, knownKind := sourceKind(existing) + if !knownKind { + return errors.New("cdc manager: registered source has no kind") + } + if existingKind != entry.Kind { + return fmt.Errorf("%w: %s -> %s", ErrSourceKindChange, existingKind, entry.Kind) + } + } driver, ok := m.drivers[entry.Kind] if !ok { return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) } - id := canonicalID(entry.ID) - existing, exists := m.registry.Get(id) if !exists { return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) } - existingKind, knownKind := sourceKind(existing) - if !knownKind { - return errors.New("cdc manager: registered source has no kind") - } - if existingKind != entry.Kind { - return fmt.Errorf("%w: %s -> %s", ErrSourceKindChange, existingKind, entry.Kind) - } // Build the replacement before changing visibility. A malformed entry or // failed dependency acquisition leaves the old source untouched. replacement, err := driver.Create(ctx, entry, m.deps) @@ -210,23 +232,58 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { _ = replacement.Stop(ctx) return errors.New("cdc manager: source is not managed by a stable slot") } + oldKey := exclusiveResourceKey(managedSlot.currentSource()) + newKey := exclusiveResourceKey(replacement) + oldToken := m.leaseTokenLocked(oldKey, id) + reservedNew := oldKey != newKey + newToken := oldToken + if reservedNew { + if managedSlot.hasRetiredKey(newKey) { + _ = stopSource(ctx, replacement) + return ErrSourceBusy + } + newToken, err = m.reserveLeaseLocked(newKey, id) + if err != nil { + _ = stopSource(ctx, replacement) + return err + } + } oldLifecycle := normalizeLifecycleConfig(managedSlot.LifecycleConfig()) - if err := managedSlot.Replace(ctx, replacement); err != nil { - return err + if replaceErr := managedSlot.Replace(ctx, replacement, oldToken); replaceErr != nil { + // A failed candidate start leaves the old generation current and the + // speculative lease can be released. A retired-resource cleanup error + // leaves the candidate current but faulted; retain its lease until it is + // healthy or deleted. + committed := managedSlot.currentSource() == replacement + if reservedNew && !committed { + m.releaseLeaseLocked(id, newKey, newToken) + } + if committed { + m.reconfigureSupervisorIfChanged(ctx, id, managedSlot, oldLifecycle) + } + return replaceErr } - newLifecycle := normalizeLifecycleConfig(managedSlot.LifecycleConfig()) - if !reflect.DeepEqual(oldLifecycle, newLifecycle) { - // ServiceUpdate is emitted by the supervisor for status changes and is - // not a reconfiguration primitive. Re-register the same stable slot in - // order, so its controller rebuilds security/dependency/autostart state - // without exposing a second source identity. - m.unregisterSupervisor(ctx, id) - m.registerSupervisorWithConfig(ctx, id, managedSlot, newLifecycle) + if oldKey != newKey && !managedSlot.hasRetiredKey(oldKey) { + m.releaseLeaseLocked(id, oldKey, oldToken) } + m.reconfigureSupervisorIfChanged(ctx, id, managedSlot, oldLifecycle) m.log.Info("updated cdc source", zap.String("id", id.String()), zap.String("kind", entry.Kind)) return nil } +func (m *Manager) reconfigureSupervisorIfChanged(ctx context.Context, id registry.ID, source *sourceSlot, old supervisor.LifecycleConfig) { + newLifecycle := normalizeLifecycleConfig(source.LifecycleConfig()) + if reflect.DeepEqual(old, newLifecycle) { + return + } + // ServiceUpdate is emitted by the supervisor for status changes and is + // not a reconfiguration primitive. Re-register the same stable slot in + // order, so its controller rebuilds security/dependency/autostart state + // without exposing a second source identity. + m.unregisterSupervisor(ctx, id) + m.registerSupervisorWithConfig(ctx, id, source, newLifecycle) +} + func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { m.mu.Lock() defer m.mu.Unlock() @@ -236,6 +293,12 @@ func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { if !ok { return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) } + if entry.Kind != "" { + kind, known := sourceKind(source) + if !known || kind != entry.Kind { + return fmt.Errorf("%w: %s", ErrSourceKindMismatch, entry.Kind) + } + } var err error if disposable, ok := source.(Disposable); ok { err = disposable.Dispose(ctx) @@ -250,6 +313,11 @@ func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { if _, ok := m.registry.Unregister(id); !ok { return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) } + if slot, ok := source.(*sourceSlot); ok { + for _, key := range slot.resourceKeys() { + m.releaseLeaseLocked(id, key, m.leaseTokenLocked(key, id)) + } + } m.unregisterSupervisor(ctx, id) m.log.Info("removed cdc source", zap.String("id", id.String())) return nil @@ -293,6 +361,47 @@ func (m *Manager) unregisterSupervisor(ctx context.Context, id registry.ID) { }) } +func (m *Manager) reserveLeaseLocked(key string, id registry.ID) (uint64, error) { + if key == "" { + return 0, nil + } + if owner, ok := m.leases[key]; ok { + if owner.id != id { + return 0, fmt.Errorf("%w: %s (owner %s)", ErrExclusiveOwned, key, owner.id) + } + return owner.token, nil + } + m.leaseSeq++ + owner := resourceLease{id: id, token: m.leaseSeq} + m.leases[key] = owner + return owner.token, nil +} + +func (m *Manager) releaseLease(id registry.ID, key string, token uint64) { + m.mu.Lock() + m.releaseLeaseLocked(id, key, token) + m.mu.Unlock() +} + +func (m *Manager) releaseLeaseLocked(id registry.ID, key string, token uint64) { + if key == "" { + return + } + if owner, ok := m.leases[key]; ok && owner.id == id && owner.token == token { + delete(m.leases, key) + } +} + +func (m *Manager) leaseTokenLocked(key string, id registry.ID) uint64 { + if key == "" { + return 0 + } + if owner, ok := m.leases[key]; ok && owner.id == id { + return owner.token + } + return 0 +} + func stopSource(ctx context.Context, source api.Source) error { if isNilSource(source) { return nil diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go index 1eabfa429..97dad50f5 100644 --- a/service/cdc/manager_test.go +++ b/service/cdc/manager_test.go @@ -323,7 +323,7 @@ func TestManagerUpdateAtomicallyReplacesAndStopsOld(t *testing.T) { require.EqualValues(t, 1, created[0].stopCount.Load()) } -func TestManagerUpdateDoesNotFailAfterNewGenerationCommits(t *testing.T) { +func TestManagerUpdateDoesNotPublishAfterOldStopFails(t *testing.T) { old := &managedTestSource{info: api.SourceInfo{Name: "old"}, stopErr: errors.New("old cleanup failed")} newSource := &managedTestSource{info: api.SourceInfo{Name: "new"}} next := 0 @@ -341,9 +341,10 @@ func TestManagerUpdateDoesNotFailAfterNewGenerationCommits(t *testing.T) { id := registry.NewID("app", "events") entry := registry.Entry{ID: id, Kind: driver.kind} require.NoError(t, m.Add(context.Background(), entry)) - require.NoError(t, m.Update(context.Background(), entry)) - require.Same(t, newSource, mustSlot(t, m, id).currentSource()) + require.EqualError(t, m.Update(context.Background(), entry), "old cleanup failed") + require.Same(t, old, mustSlot(t, m, id).currentSource()) require.EqualValues(t, 1, old.stopCount.Load()) + require.EqualValues(t, 1, newSource.stopCount.Load()) } func TestManagerUpdateKeepsStableSupervisorRegistration(t *testing.T) { @@ -424,6 +425,214 @@ func TestManagerUpdateReRegistersSupervisorWhenLifecycleChanges(t *testing.T) { require.Empty(t, registered.Config.DependsOn) } +func TestManagerExclusiveResourceLeaseAcrossIDs(t *testing.T) { + kind := registry.Kind("db.cdc.test") + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + return &managedTestSource{info: api.SourceInfo{Name: entry.ID.String()}, exclusive: "cluster/slot"}, nil + }, + } + m, _ := newManagerTest(t, driver) + first := registry.NewID("app", "first") + second := registry.NewID("app", "second") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: first, Kind: kind})) + require.ErrorIs(t, m.Add(context.Background(), registry.Entry{ID: second, Kind: kind}), ErrExclusiveOwned) + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: first, Kind: kind})) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: second, Kind: kind})) +} + +func TestManagerUpdateRejectsExclusiveResourceOwnedByAnotherID(t *testing.T) { + kind := registry.Kind("db.cdc.test") + first := registry.NewID("app", "first") + second := registry.NewID("app", "second") + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-old"} + other := &managedTestSource{info: api.SourceInfo{Name: "other"}, exclusive: "slot-new"} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + next := map[string]int{} + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + next[entry.ID.String()]++ + if entry.ID == first { + if next[entry.ID.String()] == 1 { + return old, nil + } + return candidate, nil + } + return other, nil + }, + } + m, _ := newManagerTest(t, driver) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: first, Kind: kind})) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: second, Kind: kind})) + + require.ErrorIs(t, m.Update(context.Background(), registry.Entry{ID: first, Kind: kind}), ErrExclusiveOwned) + require.Same(t, old, mustSlot(t, m, first).currentSource()) + require.EqualValues(t, 1, candidate.stopCount.Load(), "a lease conflict must stop the uncommitted candidate") +} + +func TestManagerDeleteChecksEntryKind(t *testing.T) { + kind := registry.Kind("db.cdc.test") + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { return &managedTestSource{}, nil }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: id, Kind: kind})) + require.ErrorIs(t, m.Delete(context.Background(), registry.Entry{ID: id, Kind: "db.cdc.other"}), ErrSourceKindMismatch) + _, ok := m.Get(id) + require.True(t, ok) +} + +func TestManagerUpdateDisposesDifferentExclusiveResource(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + require.EqualValues(t, 1, old.disposeCount.Load()) + require.EqualValues(t, 2, old.stopCount.Load(), "replacement stops before destructive disposal and Dispose remains idempotent") + require.EqualValues(t, 1, candidate.startCount.Load()) + require.Same(t, candidate, mustSlot(t, m, id).currentSource()) + require.NotContains(t, m.leases, "slot-old") +} + +func TestManagerUpdateSameExclusiveResourceNeverDisposesOld(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-shared", + }} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-shared"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.NoError(t, m.Update(context.Background(), entry)) + require.EqualValues(t, 0, old.disposeCount.Load()) + require.EqualValues(t, 0, old.stopCount.Load(), "an idle same-key source is retained without destructive cleanup") +} + +func TestManagerUpdateRetriesFailedRetiredDisposalBeforeRestart(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }, disposeErr: errors.New("retired cleanup failed")} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.EqualError(t, m.Update(context.Background(), entry), "retired cleanup failed") + slot := mustSlot(t, m, id) + require.Same(t, candidate, slot.currentSource()) + require.Equal(t, slotFaulted, slot.state) + require.EqualValues(t, 1, candidate.stopCount.Load()) + require.Contains(t, m.leases, "slot-old") + + require.NoError(t, slot.Stop(context.Background())) + require.EqualValues(t, 2, old.disposeCount.Load()) + require.EqualValues(t, 2, candidate.stopCount.Load()) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + require.EqualValues(t, 2, candidate.startCount.Load()) + require.Eventually(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.leases["slot-old"] + return !ok + }, time.Second, time.Millisecond) + require.NoError(t, slot.Stop(context.Background())) +} + +func TestManagerDeleteRetriesRetiredDisposalBeforeUnregister(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }, disposeErr: errors.New("retired cleanup failed")} + candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + require.EqualError(t, m.Update(context.Background(), entry), "retired cleanup failed") + require.NoError(t, m.Delete(context.Background(), entry)) + _, ok := m.Get(id) + require.False(t, ok, "delete must unregister only after current and retired cleanup") + require.EqualValues(t, 2, old.disposeCount.Load()) +} + +func TestManagerRetiredLeaseTokenCannotReleaseReplacement(t *testing.T) { + m, _ := newManagerTest(t) + id := registry.NewID("app", "events") + m.mu.Lock() + m.leaseSeq = 1 + m.leases["slot"] = resourceLease{id: id, token: 1} + m.releaseLeaseLocked(id, "slot", 1) + newToken, err := m.reserveLeaseLocked("slot", id) + require.NoError(t, err) + require.NotEqual(t, uint64(1), newToken) + m.releaseLeaseLocked(id, "slot", 1) + owner, ok := m.leases["slot"] + m.mu.Unlock() + require.True(t, ok) + require.Equal(t, newToken, owner.token) +} + func TestManagerUpdateFailedStartRetainsRunningGeneration(t *testing.T) { var next int old := &managedTestSource{info: api.SourceInfo{Name: "old"}} @@ -628,12 +837,6 @@ func mustSlot(t *testing.T, m *Manager, id registry.ID) *sourceSlot { return slot } -func (s *sourceSlot) currentSource() ManagedSource { - s.mu.RLock() - defer s.mu.RUnlock() - return s.current -} - func TestManagerRejectsUnsupportedAndMissingSources(t *testing.T) { m, _ := newManagerTest(t) id := registry.NewID("app", "events") diff --git a/service/cdc/slot.go b/service/cdc/slot.go index a00ea2aa4..f85612af0 100644 --- a/service/cdc/slot.go +++ b/service/cdc/slot.go @@ -40,16 +40,24 @@ type sourceSlot struct { opMu sync.Mutex mu sync.RWMutex - current ManagedSource - log *zap.Logger - generation uint64 - state slotState - runCtx context.Context - runCancel context.CancelFunc - status chan any - statusDone bool - replacing bool - disposing bool + current ManagedSource + log *zap.Logger + generation uint64 + state slotState + runCtx context.Context + runCancel context.CancelFunc + status chan any + statusDone bool + replacing bool + disposing bool + retired []retiredSource + retiredHook func(string, uint64) +} + +type retiredSource struct { + source ManagedSource + key string + token uint64 } func newSourceSlot(id registry.ID, kind registry.Kind, source ManagedSource, logs ...*zap.Logger) *sourceSlot { @@ -166,12 +174,25 @@ func (s *sourceSlot) Start(ctx context.Context) (<-chan any, error) { s.replacing = false s.mu.Unlock() + if err := s.retryRetired(ctx); err != nil { + _ = stopSource(ctx, current) + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.mu.Unlock() + return nil, err + } + underlying, err := startSource(ctx, runCtx, current) if err != nil { _ = stopSource(ctx, current) s.mu.Lock() s.state = slotFaulted s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil s.mu.Unlock() return nil, err } @@ -196,14 +217,16 @@ func (s *sourceSlot) Stop(ctx context.Context) error { s.mu.Lock() if s.disposing { - if s.state == slotStopped || s.state == slotFaulted { + if (s.state == slotStopped || s.state == slotFaulted) && len(s.retired) == 0 { s.mu.Unlock() return nil } - s.mu.Unlock() - return ErrSourceBusy + if s.state != slotStopped && s.state != slotFaulted { + s.mu.Unlock() + return ErrSourceBusy + } } - if s.state == slotStopped { + if s.state == slotStopped && len(s.retired) == 0 { s.mu.Unlock() return nil } @@ -216,6 +239,9 @@ func (s *sourceSlot) Stop(ctx context.Context) error { cancel() } err := stopSource(ctx, current) + if err == nil { + err = s.retryRetired(ctx) + } s.mu.Lock() if err != nil { @@ -259,6 +285,9 @@ func (s *sourceSlot) Dispose(ctx context.Context) error { } else { err = stopSource(ctx, current) } + if err == nil { + err = s.retryRetired(ctx) + } s.mu.Lock() if err != nil { @@ -277,8 +306,8 @@ func (s *sourceSlot) Dispose(ctx context.Context) error { // Replace starts a candidate before changing visibility whenever the slot is // running or the candidate is configured for auto-start. Failure leaves the // old generation current and untouched. -func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error { - if candidate == nil { +func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retiredTokens ...uint64) error { + if isNilSource(candidate) { return ErrDriverRequired } if ctx == nil { @@ -293,14 +322,23 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error runCtx := s.runCtx runCancel := s.runCancel disposing := s.disposing + hasRetired := len(s.retired) > 0 s.mu.RUnlock() if disposing { return ErrSourceBusy } + if hasRetired { + return ErrSourceBusy + } + retiredToken := uint64(0) + if len(retiredTokens) > 0 { + retiredToken = retiredTokens[0] + } startCandidate := state == slotRunning || lifecycleAutoStart(candidate) - sameExclusive := state == slotRunning && exclusiveResourceKey(old) != "" && - exclusiveResourceKey(old) == exclusiveResourceKey(candidate) + oldKey := exclusiveResourceKey(old) + candidateKey := exclusiveResourceKey(candidate) + sameExclusive := oldKey != "" && oldKey == candidateKey var underlying <-chan any var err error oldStopped := false @@ -316,7 +354,7 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error s.replacing = true s.mu.Unlock() } - if sameExclusive { + if sameExclusive && state == slotRunning { // A source such as PostgreSQL may not start a second generation while // the old generation owns the same slot. Stop the old generation first; // if the candidate fails, restore the old generation before returning. @@ -337,7 +375,11 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error } underlying, err = startSource(ctx, runCtx, candidate) if err != nil { - _ = stopSource(ctx, candidate) + if sameExclusive { + _ = stopSource(ctx, candidate) + } else { + _ = cleanupSource(ctx, candidate) + } if sameExclusive { var restartErr error underlying, restartErr = startSource(ctx, runCtx, old) @@ -364,11 +406,31 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error return err } } + if !sameExclusive && !oldStopped && !isNilSource(old) { + // Candidate startup is deliberately speculative. The stable slot and + // registry continue to expose the old generation until its non- + // destructive Stop succeeds, so a failed handoff cannot publish an + // unowned or half-stopped replacement. + if err := stopSource(ctx, old); err != nil { + if sameExclusive { + _ = stopSource(ctx, candidate) + } else { + _ = cleanupSource(ctx, candidate) + } + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.replacing = false + s.mu.Unlock() + return err + } + oldStopped = true + } s.mu.Lock() if s.state == slotStopping { s.mu.Unlock() - _ = stopSource(ctx, candidate) + _ = cleanupSource(ctx, candidate) return ErrSourceBusy } s.current = candidate @@ -390,15 +452,120 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource) error s.mu.Unlock() } - if old != nil && !oldStopped { - if err := stopSource(ctx, old); err != nil { - s.log.Warn("old cdc source failed to stop after replacement", - zap.String("id", s.id.String()), zap.Error(err)) + if old != nil && oldStopped && !sameExclusive { + if disposable, ok := old.(Disposable); ok { + if err := disposable.Dispose(ctx); err != nil { + s.recordRetired(old, oldKey, retiredToken) + if runCancel != nil { + runCancel() + } + _ = stopSource(ctx, candidate) + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.replacing = false + s.mu.Unlock() + return err + } } } return nil } +func cleanupSource(ctx context.Context, source api.Source) error { + if isNilSource(source) { + return nil + } + if disposable, ok := source.(Disposable); ok { + return disposable.Dispose(ctx) + } + return stopSource(ctx, source) +} + +func (s *sourceSlot) recordRetired(source ManagedSource, key string, token uint64) { + if isNilSource(source) { + return + } + s.mu.Lock() + s.retired = append(s.retired, retiredSource{source: source, key: key, token: token}) + s.mu.Unlock() +} + +func (s *sourceSlot) retryRetired(ctx context.Context) error { + for { + s.mu.RLock() + if len(s.retired) == 0 { + s.mu.RUnlock() + return nil + } + retired := s.retired[0] + s.mu.RUnlock() + + if err := cleanupSource(ctx, retired.source); err != nil { + return err + } + + s.mu.Lock() + if len(s.retired) > 0 && s.retired[0].source == retired.source { + s.retired = s.retired[1:] + } + hook := s.retiredHook + s.mu.Unlock() + if hook != nil && retired.key != "" { + hook(retired.key, retired.token) + } + } +} + +func (s *sourceSlot) setRetiredCleanupHook(hook func(string, uint64)) { + s.mu.Lock() + s.retiredHook = hook + s.mu.Unlock() +} + +func (s *sourceSlot) hasRetiredKey(key string) bool { + if key == "" { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + for _, retired := range s.retired { + if retired.key == key { + return true + } + } + return false +} + +func (s *sourceSlot) resourceKeys() []string { + s.mu.RLock() + current := s.current + retired := append([]retiredSource(nil), s.retired...) + s.mu.RUnlock() + keys := make([]string, 0, len(retired)+1) + if key := exclusiveResourceKey(current); key != "" { + keys = append(keys, key) + } + for _, item := range retired { + if item.key == "" { + continue + } + seen := false + for _, key := range keys { + if key == item.key { + seen = true + break + } + } + if !seen { + keys = append(keys, item.key) + } + } + return keys +} + func (s *sourceSlot) LifecycleConfig() supervisor.LifecycleConfig { s.mu.RLock() current := s.current @@ -459,6 +626,13 @@ func (s *sourceSlot) currentGeneration() uint64 { return generation } +func (s *sourceSlot) currentSource() ManagedSource { + s.mu.RLock() + source := s.current + s.mu.RUnlock() + return source +} + func (s *sourceSlot) closeStatusLocked() { if s.status != nil && !s.statusDone { close(s.status) From 7189fc6ffce037033d568ce9770c3ade2f22a5fd Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:28:53 -0400 Subject: [PATCH 12/47] fix(cdc/postgres): enforce replication slot names --- service/cdc/postgres/driver.go | 2 +- service/cdc/postgres/identifiers.go | 17 +++++++++++++++++ service/cdc/postgres/identifiers_test.go | 22 +++++++++++++++------- service/cdc/postgres/service.go | 6 +++--- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index a9973c2d0..c4b853196 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -38,7 +38,7 @@ func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice. if err := cfg.Validate(); err != nil { return nil, NewInvalidConfigError(err) } - if err := validatePostgresIdentifier(cfg.SlotName, "slot_name"); err != nil { + if _, err := quoteReplicationSlotName(cfg.SlotName); err != nil { return nil, NewInvalidConfigError(err) } if cfg.Publication != "" { diff --git a/service/cdc/postgres/identifiers.go b/service/cdc/postgres/identifiers.go index 6cdb66d7c..c5ee2ee9b 100644 --- a/service/cdc/postgres/identifiers.go +++ b/service/cdc/postgres/identifiers.go @@ -16,6 +16,23 @@ import ( // publication names are identifiers in the replication command grammar. const postgresIdentifierMaxBytes = 63 +// PostgreSQL replication slot names use a narrower grammar than ordinary +// identifiers. The server validates them as lowercase ASCII names composed +// only of letters, digits, and underscores; quoting does not broaden that +// rule. Keep this check separate from publication/table identifier quoting. +func quoteReplicationSlotName(value string) (string, error) { + if value == "" || len(value) > postgresIdentifierMaxBytes { + return "", fmt.Errorf("%w: slot_name", ErrInvalidIdentifier) + } + for i := 0; i < len(value); i++ { + c := value[i] + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '_' { + return "", fmt.Errorf("%w: slot_name", ErrInvalidIdentifier) + } + } + return pq.QuoteIdentifier(value), nil +} + func quotePostgresIdentifier(value, field string) (string, error) { if value == "" || value != strings.TrimSpace(value) || len(value) > postgresIdentifierMaxBytes || !utf8.ValidString(value) { diff --git a/service/cdc/postgres/identifiers_test.go b/service/cdc/postgres/identifiers_test.go index 5aa8ac00a..02f251a94 100644 --- a/service/cdc/postgres/identifiers_test.go +++ b/service/cdc/postgres/identifiers_test.go @@ -10,19 +10,27 @@ import ( "github.com/stretchr/testify/require" ) -func TestQuotePostgresIdentifierRejectsUnsafeNames(t *testing.T) { - for _, name := range []string{"", " slot", "slot ", "slot\nname", string([]byte{0xff})} { - _, err := quotePostgresIdentifier(name, "slot_name") +func TestQuoteReplicationSlotNameUsesServerGrammar(t *testing.T) { + for _, name := range []string{"events_2026_08", "slot0", strings.Repeat("x", postgresIdentifierMaxBytes)} { + quoted, err := quoteReplicationSlotName(name) + require.NoError(t, err, "name %q", name) + assert.Equal(t, `"`+name+`"`, quoted) + } + + for _, name := range []string{ + "", "Events", "events-name", `events"name`, "events name", + "événements", "slot\nname", string([]byte{0xff}), + strings.Repeat("x", postgresIdentifierMaxBytes+1), + } { + _, err := quoteReplicationSlotName(name) assert.ErrorIs(t, err, ErrInvalidIdentifier, "name %q", name) } - _, err := quotePostgresIdentifier(strings.Repeat("x", postgresIdentifierMaxBytes+1), "slot_name") - assert.ErrorIs(t, err, ErrInvalidIdentifier) } func TestQuotePostgresIdentifierUsesServerIdentifierQuoting(t *testing.T) { - quoted, err := quotePostgresIdentifier(`slot"name`, "slot_name") + quoted, err := quotePostgresIdentifier(`publication"name`, "publication") require.NoError(t, err) - assert.Equal(t, `"slot""name"`, quoted) + assert.Equal(t, `"publication""name"`, quoted) literal, err := quotePostgresLiteral(`publication'name`, "publication") require.NoError(t, err) diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index a57173c47..7d4b91eab 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -423,7 +423,7 @@ func (s *Source) run( if s.streaming { pluginArgs = append(pluginArgs, "streaming 'on'") } - slotIdentifier, err := quotePostgresIdentifier(s.slot, "slot_name") + slotIdentifier, err := quoteReplicationSlotName(s.slot) if err != nil { s.abortFreshSlot(conn, slotCreated) s.fail(ctx, status, err) @@ -692,7 +692,7 @@ func (s *Source) prepareSlot( snapshotName := "" slotCreated := false if !exists { - slotIdentifier, err := quotePostgresIdentifier(s.slot, "slot_name") + slotIdentifier, err := quoteReplicationSlotName(s.slot) if err != nil { return 0, "", false, err } @@ -731,7 +731,7 @@ func (s *Source) prepareSlot( } func (s *Source) setSlotFailover(ctx context.Context, conn *pgconn.PgConn) error { - slotIdentifier, err := quotePostgresIdentifier(s.slot, "slot_name") + slotIdentifier, err := quoteReplicationSlotName(s.slot) if err != nil { return err } From 2b60a96806af020ed0fd359a9dc155d1c3e266d5 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:31:42 -0400 Subject: [PATCH 13/47] fix(cdc/postgres): finalize bounded transactional delivery --- api/service/cdc/config.go | 61 ++++++--- api/service/cdc/config_test.go | 22 +++ api/service/cdc/errors.go | 29 ++-- service/cdc/postgres/bench_test.go | 12 ++ service/cdc/postgres/checkpoint.go | 8 +- service/cdc/postgres/checkpoint_test.go | 11 ++ service/cdc/postgres/config_decode_test.go | 26 ++-- service/cdc/postgres/decoder_test.go | 115 +++++++++++++++- service/cdc/postgres/driver.go | 16 +-- service/cdc/postgres/identifiers.go | 21 +++ service/cdc/postgres/manager.go | 133 +++++++++---------- service/cdc/postgres/manager_test.go | 34 ++--- service/cdc/postgres/service_metrics_test.go | 4 +- service/cdc/postgres/service_test.go | 24 ++++ 14 files changed, 362 insertions(+), 154 deletions(-) diff --git a/api/service/cdc/config.go b/api/service/cdc/config.go index 201d00721..6efa24490 100644 --- a/api/service/cdc/config.go +++ b/api/service/cdc/config.go @@ -19,26 +19,33 @@ const ( ProtocolVersion = 1 StreamingProtocolVersion = 2 + + // These defaults bound decoder memory when the corresponding entry fields + // are omitted. Zero in Config means "use this default", never unlimited. + DefaultPostgresMaxTransactionChanges = 1_000_000 + DefaultPostgresMaxTransactionBytes = 256 << 20 ) type Config struct { - Options map[string]string `json:"options"` - Database string `json:"database"` - Password string `json:"password"` - Host string `json:"host"` - Username string `json:"username"` - SlotName string `json:"slot_name"` - Publication string `json:"publication,omitempty"` - StandbyInterval string `json:"standby_interval,omitempty"` - StatusInterval string `json:"status_interval,omitempty"` - Tables []string `json:"tables,omitempty"` - Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` - Port int `json:"port"` - SnapshotFetchSize int `json:"snapshot_fetch_size,omitempty"` - Temporary bool `json:"temporary,omitempty"` - Snapshot bool `json:"snapshot,omitempty"` - Streaming bool `json:"streaming,omitempty"` - Failover bool `json:"failover,omitempty"` + Options map[string]string `json:"options"` + Database string `json:"database"` + Password string `json:"password"` + Host string `json:"host"` + Username string `json:"username"` + SlotName string `json:"slot_name"` + Publication string `json:"publication,omitempty"` + StandbyInterval string `json:"standby_interval,omitempty"` + StatusInterval string `json:"status_interval,omitempty"` + Tables []string `json:"tables,omitempty"` + Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` + Port int `json:"port"` + SnapshotFetchSize int `json:"snapshot_fetch_size,omitempty"` + MaxTransactionChanges int `json:"max_transaction_changes,omitempty"` + MaxTransactionBytes int64 `json:"max_transaction_bytes,omitempty"` + Temporary bool `json:"temporary,omitempty"` + Snapshot bool `json:"snapshot,omitempty"` + Streaming bool `json:"streaming,omitempty"` + Failover bool `json:"failover,omitempty"` } func (c *Config) InitDefaults() { @@ -76,6 +83,12 @@ func (c *Config) Validate() error { if c.SnapshotFetchSize < 0 { return ErrInvalidSnapshotFetchSize } + if c.MaxTransactionChanges < 0 { + return ErrInvalidMaxTransactionChanges + } + if c.MaxTransactionBytes < 0 { + return ErrInvalidMaxTransactionBytes + } if _, err := c.StandbyDuration(); err != nil { return err } @@ -85,6 +98,20 @@ func (c *Config) Validate() error { return nil } +func (c *Config) EffectiveMaxTransactionChanges() int { + if c.MaxTransactionChanges > 0 { + return c.MaxTransactionChanges + } + return DefaultPostgresMaxTransactionChanges +} + +func (c *Config) EffectiveMaxTransactionBytes() int64 { + if c.MaxTransactionBytes > 0 { + return c.MaxTransactionBytes + } + return DefaultPostgresMaxTransactionBytes +} + func (c *Config) StandbyDuration() (time.Duration, error) { return parseInterval(c.StandbyInterval) } diff --git a/api/service/cdc/config_test.go b/api/service/cdc/config_test.go index b0454bfef..455122230 100644 --- a/api/service/cdc/config_test.go +++ b/api/service/cdc/config_test.go @@ -122,6 +122,28 @@ func TestConfigSnapshotFetchSizeRejectsNegative(t *testing.T) { require.ErrorIs(t, c.Validate(), ErrInvalidSnapshotFetchSize) } +func TestConfigTransactionLimitsUseFiniteDefaults(t *testing.T) { + c := validConfig() + assert.Equal(t, DefaultPostgresMaxTransactionChanges, c.EffectiveMaxTransactionChanges()) + assert.Equal(t, int64(DefaultPostgresMaxTransactionBytes), c.EffectiveMaxTransactionBytes()) + + c.MaxTransactionChanges = 123 + c.MaxTransactionBytes = 456 + assert.Equal(t, 123, c.EffectiveMaxTransactionChanges()) + assert.Equal(t, int64(456), c.EffectiveMaxTransactionBytes()) + require.NoError(t, c.Validate()) +} + +func TestConfigTransactionLimitsRejectNegative(t *testing.T) { + c := validConfig() + c.MaxTransactionChanges = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxTransactionChanges) + + c = validConfig() + c.MaxTransactionBytes = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxTransactionBytes) +} + func TestConfigFailoverRequiresPersistentSlot(t *testing.T) { c := validConfig() c.Failover = true diff --git a/api/service/cdc/errors.go b/api/service/cdc/errors.go index 87c20fd75..725b82d58 100644 --- a/api/service/cdc/errors.go +++ b/api/service/cdc/errors.go @@ -5,17 +5,20 @@ package cdc import apierror "github.com/wippyai/runtime/api/error" var ( - ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) - ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) - ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) - ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) - ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) - ErrSlotNameRequired = apierror.New(apierror.Invalid, "slot_name is required").WithRetryable(apierror.False) - ErrPublicationRequired = apierror.New(apierror.Invalid, "publication or tables is required").WithRetryable(apierror.False) - ErrInvalidInterval = apierror.New(apierror.Invalid, "interval must be a non-negative duration (e.g. 10s)").WithRetryable(apierror.False) - ErrFailoverTemporary = apierror.New(apierror.Invalid, "failover cannot be set on a temporary slot").WithRetryable(apierror.False) - ErrInvalidSnapshotFetchSize = apierror.New(apierror.Invalid, "snapshot_fetch_size must be non-negative").WithRetryable(apierror.False) - ErrDBResourceRequired = apierror.New(apierror.Invalid, "db_resource is required").WithRetryable(apierror.False) - ErrSourceNotFound = apierror.New(apierror.NotFound, "cdc source not found").WithRetryable(apierror.False) - ErrUnsupported = apierror.New(apierror.Invalid, "cdc operation is not supported by this source").WithRetryable(apierror.False) + ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) + ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) + ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) + ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) + ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) + ErrSlotNameRequired = apierror.New(apierror.Invalid, "slot_name is required").WithRetryable(apierror.False) + ErrPublicationRequired = apierror.New(apierror.Invalid, "publication or tables is required").WithRetryable(apierror.False) + ErrInvalidInterval = apierror.New(apierror.Invalid, "interval must be a non-negative duration (e.g. 10s)").WithRetryable(apierror.False) + ErrFailoverTemporary = apierror.New(apierror.Invalid, "failover cannot be set on a temporary slot").WithRetryable(apierror.False) + ErrInvalidSnapshotFetchSize = apierror.New(apierror.Invalid, "snapshot_fetch_size must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxTransactionChanges = apierror.New(apierror.Invalid, "max_transaction_changes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxTransactionBytes = apierror.New(apierror.Invalid, "max_transaction_bytes must be non-negative").WithRetryable(apierror.False) + ErrDBResourceRequired = apierror.New(apierror.Invalid, "db_resource is required").WithRetryable(apierror.False) + ErrSourceNotFound = apierror.New(apierror.NotFound, "cdc source not found").WithRetryable(apierror.False) + ErrUnsupported = apierror.New(apierror.Invalid, "cdc operation is not supported by this source").WithRetryable(apierror.False) + ErrSourceNotReady = apierror.New(apierror.Unavailable, "cdc source is not ready").WithRetryable(apierror.True) ) diff --git a/service/cdc/postgres/bench_test.go b/service/cdc/postgres/bench_test.go index e2acfe792..7375cbf97 100644 --- a/service/cdc/postgres/bench_test.go +++ b/service/cdc/postgres/bench_test.go @@ -24,6 +24,12 @@ func BenchmarkDecoderInsert(b *testing.B) { if _, err := d.apply(msg, 0x20); err != nil { b.Fatal(err) } + if _, err := d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0); err != nil { + b.Fatal(err) + } + if _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0); err != nil { + b.Fatal(err) + } } } @@ -40,6 +46,12 @@ func BenchmarkDecoderUpdate(b *testing.B) { if _, err := d.apply(msg, 0x30); err != nil { b.Fatal(err) } + if _, err := d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x40}, 0); err != nil { + b.Fatal(err) + } + if _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0); err != nil { + b.Fatal(err) + } } } diff --git a/service/cdc/postgres/checkpoint.go b/service/cdc/postgres/checkpoint.go index aee2225e7..37e64f545 100644 --- a/service/cdc/postgres/checkpoint.go +++ b/service/cdc/postgres/checkpoint.go @@ -37,7 +37,9 @@ func (m *MemoryCheckpointer) Load(_ context.Context, slot string) (pglogrepl.LSN func (m *MemoryCheckpointer) Save(_ context.Context, slot string, lsn pglogrepl.LSN) error { m.mu.Lock() defer m.mu.Unlock() - m.pos[slot] = lsn + if current, ok := m.pos[slot]; !ok || lsn > current { + m.pos[slot] = lsn + } return nil } @@ -84,7 +86,9 @@ func (c *DBCheckpointer) Load(ctx context.Context, slot string) (pglogrepl.LSN, func (c *DBCheckpointer) Save(ctx context.Context, slot string, lsn pglogrepl.LSN) error { _, err := c.db.ExecContext(ctx, `INSERT INTO wippy_cdc_offsets (slot, lsn, updated_at) VALUES ($1, $2, now()) - ON CONFLICT (slot) DO UPDATE SET lsn = EXCLUDED.lsn, updated_at = now()`, + ON CONFLICT (slot) DO UPDATE + SET lsn = EXCLUDED.lsn, updated_at = now() + WHERE wippy_cdc_offsets.lsn::pg_lsn <= EXCLUDED.lsn::pg_lsn`, slot, lsn.String()) if err != nil { return fmt.Errorf("save offset: %w", err) diff --git a/service/cdc/postgres/checkpoint_test.go b/service/cdc/postgres/checkpoint_test.go index 50ea937aa..990f6d479 100644 --- a/service/cdc/postgres/checkpoint_test.go +++ b/service/cdc/postgres/checkpoint_test.go @@ -39,6 +39,17 @@ func TestMemoryCheckpointerRoundtrip(t *testing.T) { assert.False(t, ok) } +func TestMemoryCheckpointerIsMonotonic(t *testing.T) { + cp := NewMemoryCheckpointer() + ctx := context.Background() + require.NoError(t, cp.Save(ctx, "slot", pglogrepl.LSN(0x200))) + require.NoError(t, cp.Save(ctx, "slot", pglogrepl.LSN(0x100))) + lsn, ok, err := cp.Load(ctx, "slot") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, pglogrepl.LSN(0x200), lsn) +} + func TestMemoryCheckpointerDelete(t *testing.T) { cp := NewMemoryCheckpointer() ctx := context.Background() diff --git a/service/cdc/postgres/config_decode_test.go b/service/cdc/postgres/config_decode_test.go index 7d7921782..6d17353bd 100644 --- a/service/cdc/postgres/config_decode_test.go +++ b/service/cdc/postgres/config_decode_test.go @@ -27,17 +27,19 @@ func decodeConfig(t *testing.T, raw map[string]any) *config.Config { func TestConfigWireFormatMapsAndBuildsDSN(t *testing.T) { cfg := decodeConfig(t, map[string]any{ - "host": "db.internal", - "port": 5432, - "username": "cdc_repl", - "password": "secret", - "database": "appdb", - "slot_name": "wippy_slot", - "publication": "wippy_pub", - "snapshot": true, - "standby_interval": "5s", - "status_interval": "1m", - "tables": []any{"public.accounts", "public.orders"}, + "host": "db.internal", + "port": 5432, + "username": "cdc_repl", + "password": "secret", + "database": "appdb", + "slot_name": "wippy_slot", + "publication": "wippy_pub", + "snapshot": true, + "max_transaction_changes": 1234, + "max_transaction_bytes": 65536, + "standby_interval": "5s", + "status_interval": "1m", + "tables": []any{"public.accounts", "public.orders"}, }) require.NoError(t, cfg.Validate()) @@ -48,6 +50,8 @@ func TestConfigWireFormatMapsAndBuildsDSN(t *testing.T) { assert.Equal(t, "wippy_slot", cfg.SlotName) assert.Equal(t, "wippy_pub", cfg.Publication) assert.True(t, cfg.Snapshot) + assert.Equal(t, 1234, cfg.MaxTransactionChanges) + assert.Equal(t, int64(65536), cfg.MaxTransactionBytes) assert.Equal(t, "5s", cfg.StandbyInterval) assert.Equal(t, []string{"public.accounts", "public.orders"}, cfg.Tables) diff --git a/service/cdc/postgres/decoder_test.go b/service/cdc/postgres/decoder_test.go index d642f5b96..0f3cd9e2f 100644 --- a/service/cdc/postgres/decoder_test.go +++ b/service/cdc/postgres/decoder_test.go @@ -44,6 +44,9 @@ func TestDecoderInsert(t *testing.T) { changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "a@w.ai")}, 0x20) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -52,7 +55,7 @@ func TestDecoderInsert(t *testing.T) { assert.Equal(t, "accounts", c.Table) assert.Equal(t, uint32(7), c.XID) assert.Equal(t, "0/20", c.LSN) - assert.Equal(t, "0/10", c.CommitLSN) + assert.Equal(t, "0/30", c.CommitLSN) assert.Equal(t, map[string]any{"id": "1", "email": "a@w.ai"}, c.After) assert.Nil(t, c.Before) } @@ -67,6 +70,9 @@ func TestDecoderUpdate(t *testing.T) { NewTuple: textTuple("1", "new@w.ai"), }, 0x30) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x31}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -81,6 +87,9 @@ func TestDecoderDelete(t *testing.T) { changes, err := d.apply(&pglogrepl.DeleteMessage{RelationID: 42, OldTuple: textTuple("1", "a@w.ai")}, 0x40) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x41}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -95,6 +104,9 @@ func TestDecoderTruncate(t *testing.T) { changes, err := d.apply(&pglogrepl.TruncateMessage{RelationNum: 1, RelationIDs: []uint32{42}}, 0x50) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x51}, 0) + require.NoError(t, err) require.Len(t, changes, 1) c := changes[0] @@ -114,9 +126,14 @@ func TestDecoderTruncateMultipleRelations(t *testing.T) { Columns: []*pglogrepl.RelationMessageColumn{{Name: "id"}}, }, 0) require.NoError(t, err) + _, err = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) changes, err := d.apply(&pglogrepl.TruncateMessage{RelationNum: 2, RelationIDs: []uint32{42, 43}}, 0x60) require.NoError(t, err) + assert.Empty(t, changes, "rows are not visible before commit") + changes, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x61}, 0) + require.NoError(t, err) require.Len(t, changes, 2) assert.Equal(t, "accounts", changes[0].Table) assert.Equal(t, "orders", changes[1].Table) @@ -124,6 +141,8 @@ func TestDecoderTruncateMultipleRelations(t *testing.T) { func TestDecoderTruncateUnknownRelation(t *testing.T) { d := newDecoder() + _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) changes, err := d.apply(&pglogrepl.TruncateMessage{RelationNum: 1, RelationIDs: []uint32{999}}, 0x50) require.ErrorIs(t, err, ErrUnknownRelation) assert.Nil(t, changes) @@ -131,6 +150,8 @@ func TestDecoderTruncateUnknownRelation(t *testing.T) { func TestDecoderUnknownRelation(t *testing.T) { d := newDecoder() + _, err := d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 99, Tuple: textTuple("1")}, 0x10) require.ErrorIs(t, err, ErrUnknownRelation) assert.Nil(t, changes) @@ -151,10 +172,96 @@ func TestDecoderCommitClearsTransactionState(t *testing.T) { require.NoError(t, err) changes, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("9", "x@w.ai")}, 0x99) + require.ErrorIs(t, err, ErrInvalidTransaction) + assert.Nil(t, changes) +} + +func TestDecoderSafeProgressOnlyAtTransactionBoundary(t *testing.T) { + d := newDecoder() + result, err := d.applyResult(accountsRel(), 0) require.NoError(t, err) - require.Len(t, changes, 1) - assert.Equal(t, uint32(0), changes[0].XID, "xid must not leak from a committed transaction") - assert.Equal(t, "0/0", changes[0].CommitLSN) + assert.True(t, result.safe) + + result, err = d.applyResult(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) + assert.False(t, result.safe) + result, err = d.applyResult(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "a@w.ai")}, 0x20) + require.NoError(t, err) + assert.False(t, result.safe) + result, err = d.applyResult(&pglogrepl.CommitMessage{CommitLSN: 0x30}, 0) + require.NoError(t, err) + assert.True(t, result.safe) + assert.Len(t, result.changes, 1) +} + +func TestDecoderAcceptsPgoutputMetadataMessages(t *testing.T) { + d := newDecoder() + metadata := []pglogrepl.Message{ + &pglogrepl.OriginMessage{}, + &pglogrepl.TypeMessage{}, + &pglogrepl.LogicalDecodingMessage{}, + } + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.True(t, result.safe) + } + + _, err := d.applyResult(&pglogrepl.BeginMessage{FinalLSN: 0x10, Xid: 7}, 0) + require.NoError(t, err) + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.False(t, result.safe, "metadata inside an open transaction cannot advance the checkpoint") + } + result, err := d.applyResult(&pglogrepl.CommitMessage{CommitLSN: 0x20}, 0) + require.NoError(t, err) + assert.True(t, result.safe) +} + +func TestStreamingDecoderAcceptsPgoutputMetadataMessages(t *testing.T) { + d := newStreamingDecoder() + metadata := []pglogrepl.Message{ + &pglogrepl.TypeMessageV2{}, + &pglogrepl.LogicalDecodingMessageV2{}, + } + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.True(t, result.safe) + } + + _, err := d.applyResult(&pglogrepl.StreamStartMessageV2{Xid: 7, FirstSegment: 1}, 0) + require.NoError(t, err) + for _, msg := range metadata { + result, err := d.applyResult(msg, 0) + require.NoError(t, err) + assert.False(t, result.safe, "metadata inside a stream cannot advance the checkpoint") + } + _, err = d.applyResult(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + _, err = d.applyResult(&pglogrepl.StreamAbortMessageV2{Xid: 7, SubXid: 7}, 0) + require.NoError(t, err) +} + +func TestDecoderEnforcesTransactionChangeLimit(t *testing.T) { + d := newDecoder(decoderLimits{maxChanges: 1, maxBytes: defaultMaxTransactionBytes}) + seedRelAndBegin(t, d) + + _, err := d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("1", "a@w.ai")}, 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("2", "b@w.ai")}, 0x21) + require.ErrorIs(t, err, ErrTransactionLimit) +} + +func TestDecoderEnforcesTransactionByteLimitForStreamedSegments(t *testing.T) { + d := newStreamingDecoder(decoderLimits{maxChanges: 100, maxBytes: 1}) + _, err := d.apply(relV2(), 0) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "a@w.ai"), 0x20) + require.ErrorIs(t, err, ErrTransactionLimit) } func TestTupleToMapNullAndToast(t *testing.T) { diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index c4b853196..2ad4242ce 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -38,23 +38,9 @@ func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice. if err := cfg.Validate(); err != nil { return nil, NewInvalidConfigError(err) } - if _, err := quoteReplicationSlotName(cfg.SlotName); err != nil { + if err := validateConfigIdentifiers(cfg); err != nil { return nil, NewInvalidConfigError(err) } - if cfg.Publication != "" { - if err := validatePostgresIdentifier(cfg.Publication, "publication"); err != nil { - return nil, NewInvalidConfigError(err) - } - } else { - for _, table := range cfg.Tables { - if _, err := quoteQualifiedIdent(table); err != nil { - return nil, NewInvalidConfigError(err) - } - } - if _, err := quotePostgresIdentifier(cfg.SlotName+"_pub", "publication"); err != nil { - return nil, NewInvalidConfigError(err) - } - } standby, _ := cfg.StandbyDuration() status, _ := cfg.StatusDuration() replDSN, adminDSN, err := buildDSNs(cfg) diff --git a/service/cdc/postgres/identifiers.go b/service/cdc/postgres/identifiers.go index c5ee2ee9b..bb7e33823 100644 --- a/service/cdc/postgres/identifiers.go +++ b/service/cdc/postgres/identifiers.go @@ -9,6 +9,7 @@ import ( "unicode/utf8" "github.com/lib/pq" + config "github.com/wippyai/runtime/api/service/cdc" ) // PostgreSQL stores ordinary identifiers in NameData, whose default @@ -51,6 +52,26 @@ func validatePostgresIdentifier(value, field string) error { return err } +// validateConfigIdentifiers applies the same SQL-grammar checks to both the +// driver-backed source and the retained legacy manager path. Keeping this +// policy in one helper prevents either construction path from creating an +// auto-publication before rejecting its slot name. +func validateConfigIdentifiers(cfg *config.Config) error { + if _, err := quoteReplicationSlotName(cfg.SlotName); err != nil { + return err + } + if cfg.Publication != "" { + return validatePostgresIdentifier(cfg.Publication, "publication") + } + for _, table := range cfg.Tables { + if _, err := quoteQualifiedIdent(table); err != nil { + return err + } + } + _, err := quotePostgresIdentifier(cfg.SlotName+"_pub", "publication") + return err +} + func quotePostgresLiteral(value, field string) (string, error) { if value == "" || !utf8.ValidString(value) { return "", fmt.Errorf("%w: %s", ErrInvalidIdentifier, field) diff --git a/service/cdc/postgres/manager.go b/service/cdc/postgres/manager.go index 7ef913337..30228a0cc 100644 --- a/service/cdc/postgres/manager.go +++ b/service/cdc/postgres/manager.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "net/url" + "sort" "strconv" "sync" @@ -20,14 +21,16 @@ import ( "go.uber.org/zap" ) +// Manager is the legacy PostgreSQL-specific registry and lifecycle wrapper. +// Deprecated: use service/cdc.Manager with NewDriver so source identity and +// lifecycle are owned by the driver-neutral CDC manager. type Manager struct { - dtt payload.Transcoder - bus event.Bus - log *zap.Logger - sources map[registry.ID]*Source - infos map[registry.ID]config.SourceInfo - infosByKey map[string]registry.ID - mu sync.Mutex + dtt payload.Transcoder + bus event.Bus + log *zap.Logger + sources map[registry.ID]*Source + infos map[registry.ID]config.SourceInfo + mu sync.Mutex } func NewManager(dtt payload.Transcoder, bus event.Bus, log *zap.Logger) (*Manager, error) { @@ -41,12 +44,11 @@ func NewManager(dtt payload.Transcoder, bus event.Bus, log *zap.Logger) (*Manage log = zap.NewNop() } return &Manager{ - dtt: dtt, - bus: bus, - log: log, - sources: make(map[registry.ID]*Source), - infos: make(map[registry.ID]config.SourceInfo), - infosByKey: make(map[string]registry.ID), + dtt: dtt, + bus: bus, + log: log, + sources: make(map[registry.ID]*Source), + infos: make(map[registry.ID]config.SourceInfo), }, nil } @@ -68,7 +70,9 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { if err := cfg.Validate(); err != nil { return NewInvalidConfigError(err) } - + if err := validateConfigIdentifiers(cfg); err != nil { + return NewInvalidConfigError(err) + } standby, _ := cfg.StandbyDuration() status, _ := cfg.StatusDuration() replDSN, adminDSN, err := buildDSNs(cfg) @@ -76,20 +80,22 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { return err } src := NewSource(SourceOptions{ - ReplDSN: replDSN, - AdminDSN: adminDSN, - Name: entry.ID.String(), - Slot: cfg.SlotName, - Publication: cfg.Publication, - Tables: cfg.Tables, - Temporary: cfg.Temporary, - Snapshot: cfg.Snapshot, - Streaming: cfg.Streaming, - Failover: cfg.Failover, - StandbyInterval: standby, - StatusInterval: status, - SnapshotFetchSize: cfg.SnapshotFetchSize, - Log: m.log.With(zap.String("id", entry.ID.String())), + ReplDSN: replDSN, + AdminDSN: adminDSN, + Name: entry.ID.String(), + Slot: cfg.SlotName, + Publication: cfg.Publication, + Tables: cfg.Tables, + Temporary: cfg.Temporary, + Snapshot: cfg.Snapshot, + Streaming: cfg.Streaming, + Failover: cfg.Failover, + StandbyInterval: standby, + StatusInterval: status, + SnapshotFetchSize: cfg.SnapshotFetchSize, + MaxTransactionChanges: cfg.MaxTransactionChanges, + MaxTransactionBytes: cfg.MaxTransactionBytes, + Log: m.log.With(zap.String("id", entry.ID.String())), }) m.sources[entry.ID] = src @@ -116,7 +122,9 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { if err := cfg.Validate(); err != nil { return NewInvalidConfigError(err) } - + if err := validateConfigIdentifiers(cfg); err != nil { + return NewInvalidConfigError(err) + } replDSN, adminDSN, err := buildDSNs(cfg) if err != nil { return err @@ -132,20 +140,22 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { standby, _ := cfg.StandbyDuration() status, _ := cfg.StatusDuration() src := NewSource(SourceOptions{ - ReplDSN: replDSN, - AdminDSN: adminDSN, - Name: entry.ID.String(), - Slot: cfg.SlotName, - Publication: cfg.Publication, - Tables: cfg.Tables, - Temporary: cfg.Temporary, - Snapshot: cfg.Snapshot, - Streaming: cfg.Streaming, - Failover: cfg.Failover, - StandbyInterval: standby, - StatusInterval: status, - SnapshotFetchSize: cfg.SnapshotFetchSize, - Log: m.log.With(zap.String("id", entry.ID.String())), + ReplDSN: replDSN, + AdminDSN: adminDSN, + Name: entry.ID.String(), + Slot: cfg.SlotName, + Publication: cfg.Publication, + Tables: cfg.Tables, + Temporary: cfg.Temporary, + Snapshot: cfg.Snapshot, + Streaming: cfg.Streaming, + Failover: cfg.Failover, + StandbyInterval: standby, + StatusInterval: status, + SnapshotFetchSize: cfg.SnapshotFetchSize, + MaxTransactionChanges: cfg.MaxTransactionChanges, + MaxTransactionBytes: cfg.MaxTransactionBytes, + Log: m.log.With(zap.String("id", entry.ID.String())), }) m.sources[entry.ID] = src m.storeInfo(entry, cfg) @@ -181,16 +191,10 @@ func (m *Manager) storeInfo(entry registry.Entry, cfg *config.Config) { Snapshot: cfg.Snapshot, } m.infos[entry.ID] = info - m.infosByKey[info.Slot] = entry.ID } func (m *Manager) removeInfo(id registry.ID) { - if info, ok := m.infos[id]; ok { - if current, present := m.infosByKey[info.Slot]; present && current == id { - delete(m.infosByKey, info.Slot) - } - delete(m.infos, id) - } + delete(m.infos, id) } func (m *Manager) List() []config.SourceInfo { @@ -201,6 +205,7 @@ func (m *Manager) List() []config.SourceInfo { for _, info := range m.infos { out = append(out, info) } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) return out } @@ -208,15 +213,9 @@ func (m *Manager) Get(name string) (config.SourceInfo, bool) { m.mu.Lock() defer m.mu.Unlock() - if id, ok := m.infosByKey[name]; ok { - if info, present := m.infos[id]; present { - return info, true - } - } - for _, info := range m.infos { - if info.Name == name { - return info, true - } + id := registry.ParseID(name) + if info, ok := m.infos[id]; ok { + return info, true } return config.SourceInfo{}, false } @@ -232,17 +231,13 @@ func (m *Manager) Stream(_ context.Context, name string, opts config.StreamOptio } func (m *Manager) lookupSourceLocked(name string) (*Source, config.SourceInfo, bool) { - if id, ok := m.infosByKey[name]; ok { - if src := m.sources[id]; src != nil { - return src, m.infos[id], true - } + id := registry.ParseID(name) + info, ok := m.infos[id] + if !ok { + return nil, config.SourceInfo{}, false } - for id, info := range m.infos { - if info.Name == name { - if src := m.sources[id]; src != nil { - return src, info, true - } - } + if src := m.sources[id]; src != nil { + return src, info, true } return nil, config.SourceInfo{}, false } diff --git a/service/cdc/postgres/manager_test.go b/service/cdc/postgres/manager_test.go index 991733e88..d8d494a51 100644 --- a/service/cdc/postgres/manager_test.go +++ b/service/cdc/postgres/manager_test.go @@ -190,9 +190,8 @@ func TestBuildDSNs(t *testing.T) { func newInspectorManager() *Manager { return &Manager{ - sources: map[registry.ID]*Source{}, - infos: map[registry.ID]config.SourceInfo{}, - infosByKey: map[string]registry.ID{}, + sources: map[registry.ID]*Source{}, + infos: map[registry.ID]config.SourceInfo{}, } } @@ -215,12 +214,12 @@ func TestManagerStoreAndListInfos(t *testing.T) { sort.Strings(slots) assert.Equal(t, []string{"slot_a", "slot_b"}, slots) - a, ok := m.Get("slot_a") + a, ok := m.Get("test:id-a") require.True(t, ok) assert.Equal(t, "pub_a", a.Publication) assert.True(t, a.Streaming) - b, ok := m.Get("slot_b") + b, ok := m.Get("test:id-b") require.True(t, ok) assert.Equal(t, []string{"public.t"}, b.Tables) @@ -232,20 +231,14 @@ func TestManagerStoreAndListInfos(t *testing.T) { assert.Equal(t, infos[0].Slot, byID.Slot) } -func TestManagerStreamBySlotAndID(t *testing.T) { +func TestManagerStreamByID(t *testing.T) { m := newInspectorManager() id := registry.NewID("test", "id-stream") src := NewSource(SourceOptions{Name: id.String(), Slot: "slot_stream"}) m.sources[id] = src m.storeInfo(registry.Entry{ID: id, Kind: config.Postgres}, &config.Config{SlotName: "slot_stream", Tables: []string{"public.accounts"}}) - stream, info, err := m.Stream(context.Background(), "slot_stream", config.StreamOptions{Buffer: 2}) - require.NoError(t, err) - require.NotNil(t, stream) - assert.Equal(t, "slot_stream", info.Slot) - stream.Close() - - stream, info, err = m.Stream(context.Background(), id.String(), config.StreamOptions{}) + stream, info, err := m.Stream(context.Background(), id.String(), config.StreamOptions{Buffer: 2}) require.NoError(t, err) require.NotNil(t, stream) assert.Equal(t, id.String(), info.Name) @@ -260,11 +253,11 @@ func TestManagerRemoveInfo(t *testing.T) { m.removeInfo(idX) assert.Empty(t, m.List()) - _, ok := m.Get("slot_x") + _, ok := m.Get(idX.String()) assert.False(t, ok) } -func TestManagerCollidingSlotsDoNotLeakIndex(t *testing.T) { +func TestManagerCollidingSlotsRemainDistinctByID(t *testing.T) { m := newInspectorManager() id1 := registry.NewID("test", "id-1") id2 := registry.NewID("test", "id-2") @@ -273,14 +266,13 @@ func TestManagerCollidingSlotsDoNotLeakIndex(t *testing.T) { require.Len(t, m.List(), 2) - m.removeInfo(id1) - got, ok := m.Get("shared") + got, ok := m.Get(id1.String()) require.True(t, ok) - assert.Equal(t, id2.String(), got.Name) + assert.Equal(t, id1.String(), got.Name) - m.removeInfo(id2) - _, ok = m.Get("shared") - assert.False(t, ok) + got, ok = m.Get(id2.String()) + require.True(t, ok) + assert.Equal(t, id2.String(), got.Name) } func TestBuildDSNsCarriesOptions(t *testing.T) { diff --git a/service/cdc/postgres/service_metrics_test.go b/service/cdc/postgres/service_metrics_test.go index 31a1cff31..af1ab0384 100644 --- a/service/cdc/postgres/service_metrics_test.go +++ b/service/cdc/postgres/service_metrics_test.go @@ -15,12 +15,12 @@ import ( func TestSource_FailEmitsErrorCounter(t *testing.T) { rec := telemetrytest.NewRecorder() - s := &Source{log: zap.NewNop(), slot: "test_slot", coll: rec} + s := &Source{log: zap.NewNop(), name: "test:source", slot: "test_slot", coll: rec} status := make(chan any, 1) s.fail(context.Background(), status, errors.New("boom")) - assert.Equal(t, 1.0, rec.CounterValue(errorsCounter, metrics.Labels{"slot": "test_slot"})) + assert.Equal(t, 1.0, rec.CounterValue(errorsCounter, metrics.Labels{"source": "test:source"})) } func TestSource_FailNilCollector(t *testing.T) { diff --git a/service/cdc/postgres/service_test.go b/service/cdc/postgres/service_test.go index 8fe312044..de4ac9e10 100644 --- a/service/cdc/postgres/service_test.go +++ b/service/cdc/postgres/service_test.go @@ -37,3 +37,27 @@ func TestStopBeforeStartIsSafe(t *testing.T) { require.NoError(t, s.Stop(ctx)) require.NoError(t, s.Stop(ctx)) } + +func TestFailedSourceCanBeStoppedAndRetried(t *testing.T) { + s := NewSource(SourceOptions{}) + s.mu.Lock() + s.state = sourceFailed + s.mu.Unlock() + + require.NoError(t, s.Stop(context.Background())) + + // A canceled start fails during setup, but it must not be rejected as a + // permanently closed source. Supervisors use this path after a fault. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := s.Start(ctx) + require.Error(t, err) + assert.NotErrorIs(t, err, ErrSourceClosed) +} + +func TestClosePermanentlyRetiresSource(t *testing.T) { + s := NewSource(SourceOptions{}) + require.NoError(t, s.Close(context.Background())) + _, err := s.Start(context.Background()) + require.ErrorIs(t, err, ErrSourceClosed) +} From 99b1ddc9c9fa633e2aedad583863e57db7134ad3 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:37:37 -0400 Subject: [PATCH 14/47] fix(cdc/sqlite): close snapshot streams on worker exit --- service/cdc/sqlite/source.go | 1034 ++++++++++++++++------------- service/cdc/sqlite/source_test.go | 563 ++++++++++++++++ 2 files changed, 1136 insertions(+), 461 deletions(-) create mode 100644 service/cdc/sqlite/source_test.go diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index 4fb1d3b2f..91de056cf 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -6,14 +6,11 @@ package sqlite import ( "context" - "database/sql" "errors" "fmt" - "os" "strconv" "strings" "sync" - "sync/atomic" "time" "go.uber.org/zap" @@ -22,75 +19,61 @@ import ( "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/resource" config "github.com/wippyai/runtime/api/service/cdc" + sqlapi "github.com/wippyai/runtime/api/service/sql" sqlconfig "github.com/wippyai/runtime/api/service/sql" + "github.com/wippyai/runtime/api/supervisor" sqlservice "github.com/wippyai/runtime/service/sql" ) const ( - changesCounter = "wippy_cdc_changes_total" - walGauge = "wippy_cdc_wal_size_bytes" - defaultStatusInterval = 30 * time.Second - commitQueueSize = 256 - maxTxnRows = 200_000 - maxTxnBytes = 128 << 20 - auxBusyTimeoutMillisec = 5000 - claimAttempts = 40 - claimRetryDelay = 50 * time.Millisecond - cleanupTimeout = 5 * time.Second + defaultStatusInterval = 30 * time.Second + cleanupTimeout = 5 * time.Second + changesCounter = "wippy_cdc_changes_total" ) -type capturedChange struct { - schema string - table string - old []any - new []any - op int - rowid int64 - ncols int -} - +// Source is the SQLite CDC adapter. The SQL resource owns the SQLite +// connection, driver hooks, and observer lifetime. This adapter only borrows +// the resource long enough to subscribe to its committed-mutation capability; +// it never opens another connection and never installs hooks on a raw one. type Source struct { - res resource.Registry - poolRes resource.Resource[any] - readDB *sql.DB - writerDB *sql.DB - runCtx context.Context - cancel context.CancelFunc + res resource.Registry + log *zap.Logger + id registry.ID + name string + dbResID registry.ID + tables []string + statusTick time.Duration + lifecycle configLifecycle + snapshot bool + + subs *subscribers + + mu sync.RWMutex + state config.SourceState + generation string + sourceErr error + observer sqlapi.MutationStream + observerSource sqlapi.CommittedMutationSource + snapshotSubs map[*subscription]sqlapi.MutationStream + snapshotWG sync.WaitGroup + startDone chan struct{} runDone chan struct{} - commits chan []capturedChange - faultCh chan struct{} - subs *subscribers - cols map[string][]columnInfo - tables map[string]struct{} - log *zap.Logger - faultMsg atomic.Pointer[string] - name string - file string - epoch string - dbResID registry.ID - pending []capturedChange - statusInterval time.Duration - token uint64 - pendingBytes int - maxRows int - maxBytes int - seq atomic.Uint64 - schemaVer atomic.Int64 - colMu sync.RWMutex - mu sync.Mutex - pendMu sync.Mutex - faultOnce sync.Once - stopped atomic.Bool - faulted atomic.Bool - defaultSnap bool + startCancel context.CancelFunc + runCancel context.CancelFunc + status chan any + statusClosed bool + stopping bool } -func buildSource(opts sourceOptions) (sourceHandle, error) { +// configLifecycle is an alias kept local to this package so the Source does +// not expose configuration implementation details through its API. +type configLifecycle = supervisor.LifecycleConfig + +func buildSource(opts sourceOptions) (managedSource, error) { log := opts.log if log == nil { log = zap.NewNop() } - interval := defaultStatusInterval if opts.statusInterval != "" { d, err := time.ParseDuration(opts.statusInterval) @@ -101,525 +84,654 @@ func buildSource(opts sourceOptions) (sourceHandle, error) { interval = d } } - - return &Source{ - log: log, - res: opts.res, - subs: newSubscribers(), - name: opts.name, - statusInterval: interval, - dbResID: opts.dbResource, - tables: filterSet(opts.tables), - cols: make(map[string][]columnInfo), - faultCh: make(chan struct{}), - maxRows: maxTxnRows, - maxBytes: maxTxnBytes, - defaultSnap: opts.snapshot, - }, nil -} - -func (s *Source) Subscribe(opts config.StreamOptions) config.ChangeStream { - wantSnapshot := opts.Snapshot || s.defaultSnap - sub := s.subs.subscribe(s.name, opts, wantSnapshot) - if faulted, reason := s.Faulted(); faulted { - sub.fail(reason) - return sub - } - if !wantSnapshot { - return sub - } - - s.mu.Lock() - ctx := s.runCtx - ready := s.readDB != nil && !s.stopped.Load() - s.mu.Unlock() - - if ready && ctx != nil { - go s.bootstrapSubscription(ctx, sub) - } else { - sub.finishSnapshot() - } - - return sub -} - -func (s *Source) closeSubscriptions() { - s.subs.closeAll() -} - -func (s *Source) Epoch() string { - s.mu.Lock() - defer s.mu.Unlock() - - return s.epoch -} - -func (s *Source) Faulted() (bool, string) { - if !s.faulted.Load() { - return false, "" + name := opts.name + if name == "" && (opts.id.NS != "" || opts.id.Name != "") { + name = opts.id.String() } - - msg := "" - if p := s.faultMsg.Load(); p != nil { - msg = *p + if name == "" { + name = "sqlite" } - return true, msg -} - -func (s *Source) fault(reason string) { - s.faultOnce.Do(func() { - s.faulted.Store(true) - r := reason - s.faultMsg.Store(&r) - s.resetPending() - close(s.faultCh) - }) -} - -func (s *Source) resetPending() { - s.pendMu.Lock() - s.pending = nil - s.pendingBytes = 0 - s.pendMu.Unlock() + return &Source{ + res: opts.res, + log: log, + id: opts.id, + name: name, + dbResID: opts.dbResource, + tables: append([]string(nil), opts.tables...), + statusTick: interval, + lifecycle: opts.lifecycle, + snapshot: opts.snapshot, + subs: newSubscribers(), + snapshotSubs: make(map[*subscription]sqlapi.MutationStream), + state: config.SourceStateUnknown, + }, nil } -func (s *Source) PreUpdate(op int, schema, table string, rowid int64, ncols int, old, new []any, scanErr error) { - if s.faulted.Load() { - return - } - if !schemaAllowed(schema) || !s.tableAllowed(table) { - return +// Info reports the guarantees of this generation. SQLite observer positions +// are process-local and are not a durable LSN or resumable checkpoint. The +// SQL-owned snapshot stream provides an atomic snapshot/live handoff; writes +// made through a different unobserved database generation are not captured. +func (s *Source) Info() config.SourceInfo { + s.mu.RLock() + state := s.state + generation := s.generation + err := s.sourceErr + s.mu.RUnlock() + + info := config.SourceInfo{ + ID: s.id, + Kind: config.SQLite, + State: state, + Generation: generation, + Name: s.name, + Engine: "sqlite", + DBResource: s.dbResID.String(), + Tables: append([]string(nil), s.tables...), + Epoch: generation, + Capabilities: config.Capabilities{ + Snapshot: true, + Durable: false, + Replayable: false, + CapturesExternalWrites: false, + BeforeImages: true, + Coalesced: true, + }, + Snapshot: true, + Streaming: state == config.SourceStateRunning, + Faulted: state == config.SourceStateFaulted, } - if scanErr != nil { - s.fault("read preupdate row: " + scanErr.Error()) - return - } - - s.pendMu.Lock() - if len(s.pending) >= s.maxRows || s.pendingBytes >= s.maxBytes { - s.pendMu.Unlock() - s.fault(ErrChangeBacklogOverflow.Error()) - return + if err != nil { + info.Error = err.Error() } - s.pending = append(s.pending, capturedChange{op: op, schema: schema, table: table, rowid: rowid, ncols: ncols, old: old, new: new}) - s.pendingBytes += approxRowSize(old) + approxRowSize(new) - s.pendMu.Unlock() + return info } -func (s *Source) Commit() { - if s.faulted.Load() { - s.resetPending() - return - } +// LifecycleConfig lets the generic CDC manager register this source with the +// platform supervisor. The source itself does not emit lifecycle events. +func (s *Source) LifecycleConfig() supervisor.LifecycleConfig { return s.lifecycle } - s.pendMu.Lock() - batch := s.pending - s.pending = nil - s.pendingBytes = 0 - s.pendMu.Unlock() - if len(batch) == 0 { - return +// Start subscribes to the SQL resource's observer and starts the forwarding +// loop. The resource borrow is released immediately after Subscribe succeeds; +// the SQL generation remains the owner of the observer and closes it when the +// database generation is replaced or stopped. +func (s *Source) Start(ctx context.Context) (<-chan any, error) { + if ctx == nil { + ctx = context.Background() } - select { - case s.commits <- batch: - default: - s.fault(ErrChangeBacklogOverflow.Error()) + s.mu.Lock() + if s.state == config.SourceStateRunning { + status := s.status + s.mu.Unlock() + return status, nil } -} - -func (s *Source) Rollback() { - s.resetPending() -} - -func schemaAllowed(schema string) bool { - return schema == "" || strings.EqualFold(schema, "main") -} - -func (s *Source) tableAllowed(table string) bool { - if len(s.tables) == 0 { - return true + if s.state == config.SourceStateStarting { + s.mu.Unlock() + return nil, fmt.Errorf("%w: start already in progress", config.ErrSourceNotReady) } - _, ok := s.tables[strings.ToLower(table)] - - return ok -} - -func (s *Source) Start(ctx context.Context) (<-chan any, error) { - if s.stopped.Load() { + if s.stopping { + s.mu.Unlock() return nil, ErrSourceClosed } - dbRes, res, err := s.acquirePool(ctx) - if err != nil { - return nil, err - } - writerDB := dbRes.DB - - conn, err := writerDB.Conn(ctx) - if err != nil { - res.Release() - return nil, fmt.Errorf("acquire writer conn: %w", err) - } + startCtx, startCancel := context.WithCancel(ctx) + startDone := make(chan struct{}) + status := make(chan any, 8) + s.state = config.SourceStateStarting + s.stopping = false + s.startCancel = startCancel + s.startDone = startDone + s.status = status + s.statusClosed = false + s.mu.Unlock() + defer close(startDone) - file, token, err := s.installWithRetry(ctx, conn) + observer, err := s.acquireObserver(startCtx) if err != nil { - _ = conn.Close() - res.Release() + startCancel() + s.mu.Lock() + s.sourceErr = err + if s.stopping { + s.state = config.SourceStateStopped + } else { + s.state = config.SourceStateFaulted + } + s.startCancel = nil + s.closeStatusLocked() + s.mu.Unlock() return nil, err } - readDB, err := openReadConn(file) + stream, err := observer.Subscribe(startCtx, sqlapi.MutationOptions{ + Tables: append([]string(nil), s.tables...), + }) + // Releasing the ordinary resource borrow is part of acquireObserver; the + // observer remains owned by the SQL resource generation. if err != nil { - _ = conn.Close() - s.detachHooks(ctx, writerDB, file, token) - res.Release() - return nil, err + startCancel() + s.mu.Lock() + s.sourceErr = err + if s.stopping { + s.state = config.SourceStateStopped + } else { + s.state = config.SourceStateFaulted + } + s.startCancel = nil + s.closeStatusLocked() + s.mu.Unlock() + return nil, fmt.Errorf("subscribe sqlite mutation observer: %w", err) } - _ = conn.Close() - runCtx, cancel := context.WithCancel(ctx) - status := make(chan any, 8) + runCtx, runCancel := context.WithCancel(startCtx) runDone := make(chan struct{}) - commits := make(chan []capturedChange, commitQueueSize) s.mu.Lock() - if s.stopped.Load() { + if s.stopping { s.mu.Unlock() - cancel() - _ = readDB.Close() - s.detachHooks(ctx, writerDB, file, token) - res.Release() + runCancel() + _ = stream.Close() + startCancel() return nil, ErrSourceClosed } - epoch := strconv.FormatInt(time.Now().UnixNano(), 10) - s.poolRes = res - s.writerDB = writerDB - s.readDB = readDB - s.file = file - s.token = token - s.epoch = epoch - s.cancel = cancel - s.runCtx = runCtx + s.observer = stream + s.observerSource = observer + s.sourceErr = nil + s.runCancel = runCancel s.runDone = runDone - s.commits = commits + s.startCancel = nil + s.state = config.SourceStateRunning s.mu.Unlock() select { - case status <- "sqlite cdc started": + case status <- "sqlite cdc source started": default: } - - go s.run(runCtx, status, runDone, metrics.GetCollector(ctx)) - - s.log.Info("sqlite cdc source started", zap.String("file", file), zap.String("epoch", epoch)) - + go s.run(runCtx, stream, runDone) return status, nil } -func (s *Source) installWithRetry(ctx context.Context, conn *sql.Conn) (string, uint64, error) { - var file string - var token uint64 - for attempt := 0; attempt < claimAttempts; attempt++ { - err := conn.Raw(func(dc any) error { - f, t, e := installHooksOnRaw(dc, s) - file, token = f, t - - return e - }) - if err == nil { - return file, token, nil - } - if !errors.Is(err, errCaptureOwned) { - return "", 0, err - } - - select { - case <-ctx.Done(): - return "", 0, ctx.Err() - case <-time.After(claimRetryDelay): - } +func (s *Source) acquireObserver(ctx context.Context) (sqlapi.CommittedMutationSource, error) { + if s.res == nil { + return nil, ErrResourceRegRequired } - - return "", 0, errCaptureOwned -} - -func (s *Source) acquirePool(ctx context.Context) (sqlservice.DBResource, resource.Resource[any], error) { - res, err := s.res.Acquire(ctx, s.dbResID, resource.ModeNormal) + borrow, err := s.res.Acquire(ctx, s.dbResID, resource.ModeNormal) if err != nil { - return sqlservice.DBResource{}, nil, fmt.Errorf("acquire db resource: %w", err) + return nil, fmt.Errorf("acquire db resource: %w", err) } + defer borrow.Release() - dbAny, err := res.Get() + value, err := borrow.Get() if err != nil { - res.Release() - return sqlservice.DBResource{}, nil, fmt.Errorf("get db resource: %w", err) + return nil, fmt.Errorf("get db resource: %w", err) } - - dbRes, ok := dbAny.(sqlservice.DBResource) + db, ok := value.(sqlservice.DBResource) if !ok { - res.Release() - return sqlservice.DBResource{}, nil, fmt.Errorf("resource %s is not a database", s.name) + return nil, fmt.Errorf("resource %s is not a database", s.name) } - if dbRes.Type != sqlconfig.SQLite { - res.Release() - return sqlservice.DBResource{}, nil, fmt.Errorf("resource %s is not a sqlite database (kind %s)", s.name, dbRes.Type) + if db.Type != sqlconfig.SQLite { + return nil, fmt.Errorf("resource %s is not a sqlite database (kind %s)", s.name, db.Type) } - - return dbRes, res, nil + if db.Observer == nil { + return nil, fmt.Errorf("resource %s does not expose committed mutation observation", s.name) + } + return db.Observer, nil } -func (s *Source) Stop(ctx context.Context) error { - if !s.stopped.CompareAndSwap(false, true) { - return nil - } - defer s.closeSubscriptions() +func (s *Source) run(ctx context.Context, stream sqlapi.MutationStream, done chan struct{}) { + defer close(done) + defer func() { + s.mu.RLock() + running := s.state == config.SourceStateRunning && !s.stopping + s.mu.RUnlock() + if running { + s.fail(ErrSourceClosed) + } + }() - s.mu.Lock() - cancel := s.cancel - runDone := s.runDone - writerDB := s.writerDB - file := s.file - token := s.token - s.mu.Unlock() + collector := metrics.GetCollector(ctx) + ticker := time.NewTicker(s.statusTick) + defer ticker.Stop() - if cancel != nil { - cancel() - } - if runDone != nil { + for { select { - case <-runDone: case <-ctx.Done(): - <-runDone + s.mu.RLock() + stopping := s.stopping || s.state == config.SourceStateStopped + s.mu.RUnlock() + if !stopping { + s.fail(ctx.Err()) + } + return + case <-ticker.C: + // The stream is deliberately passive. A status tick keeps the + // lifecycle channel alive without probing a second SQL connection. + case batch, ok := <-stream.Changes(): + if !ok { + err := stream.Err() + if err == nil { + err = ErrSourceClosed + } + s.fail(err) + return + } + if err := s.processBatch(batch, collector); err != nil { + s.fail(err) + return + } } } +} - if writerDB != nil { - s.detachHooks(ctx, writerDB, file, token) +func (s *Source) processBatch(batch sqlapi.MutationBatch, collector metrics.Collector) error { + if batch.Transaction == "" { + return errors.New("sqlite mutation observer emitted a batch without a transaction identity") + } + for i, mutation := range batch.Changes { + change, err := s.changeFromMutation(batch, i, mutation) + if err != nil { + return err + } + s.subs.publish(change) + if collector != nil { + collector.CounterInc(changesCounter, metrics.Labels{"source": s.name, "op": change.Op}) + } } - s.releaseResources() - return nil } -func (s *Source) detachHooks(ctx context.Context, writerDB *sql.DB, file string, token uint64) { - releaseCapture(file, token) - - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) - defer cancel() - - conn, err := writerDB.Conn(cleanupCtx) +func (s *Source) changeFromMutation(batch sqlapi.MutationBatch, index int, mutation sqlapi.Mutation) (config.Change, error) { + op := strings.ToLower(strings.TrimSpace(mutation.Op)) + if op != "insert" && op != "update" && op != "delete" && !(batch.Snapshot && op == "snapshot") { + return config.Change{}, fmt.Errorf("sqlite mutation observer emitted unsupported operation %q", mutation.Op) + } + if mutation.Table == "" { + return config.Change{}, errors.New("sqlite mutation observer emitted a mutation without a table") + } + if len(mutation.Columns) == 0 && (len(mutation.Before) != 0 || len(mutation.After) != 0) { + return config.Change{}, fmt.Errorf("sqlite mutation observer emitted %s.%s without captured columns", mutation.Schema, mutation.Table) + } + before, err := valuesByColumn(mutation.Columns, mutation.Before) if err != nil { - return + return config.Change{}, fmt.Errorf("sqlite %s.%s before image: %w", mutation.Schema, mutation.Table, err) } - _ = conn.Raw(func(dc any) error { return applyOwnerOnRaw(dc, file) }) - _ = conn.Close() + after, err := valuesByColumn(mutation.Columns, mutation.After) + if err != nil { + return config.Change{}, fmt.Errorf("sqlite %s.%s after image: %w", mutation.Schema, mutation.Table, err) + } + + cursor := batch.Transaction + "/" + strconv.Itoa(index) + schema := normalizeSchema(mutation.Schema) + return config.Change{ + Before: before, + After: after, + Source: s.name, + SourceID: s.id, + Op: op, + Schema: schema, + Table: mutation.Table, + Relation: mutation.Table, + Cursor: cursor, + Transaction: batch.Transaction, + }, nil } -func (s *Source) releaseResources() { - s.mu.Lock() - readDB := s.readDB - res := s.poolRes - s.readDB = nil - s.poolRes = nil - s.mu.Unlock() - - if readDB != nil { - _ = readDB.Close() +func valuesByColumn(columns []string, values []any) (map[string]any, error) { + if values == nil { + return nil, nil } - if res != nil { - res.Release() + if len(columns) != len(values) { + return nil, fmt.Errorf("captured %d values for %d columns", len(values), len(columns)) } -} - -func (s *Source) run(ctx context.Context, status chan any, runDone chan struct{}, mc metrics.Collector) { - defer close(runDone) - defer close(status) - - ticker := time.NewTicker(s.statusInterval) - defer ticker.Stop() - - faultCh := s.faultCh - for { - select { - case batch := <-s.commits: - if !s.faulted.Load() { - s.process(ctx, batch, mc) - } - case <-faultCh: - s.emitFault(ctx) - faultCh = nil - case <-ticker.C: - s.onTick(mc) - case <-ctx.Done(): - if !s.faulted.Load() { - s.drainRemaining(ctx, mc) - } - - return + out := make(map[string]any, len(values)) + for i, column := range columns { + column = strings.TrimSpace(column) + if column == "" { + return nil, fmt.Errorf("captured column %d has an empty name", i) } - } -} - -func (s *Source) drainRemaining(ctx context.Context, mc metrics.Collector) { - for { - select { - case batch := <-s.commits: - s.process(ctx, batch, mc) - default: - return + if _, exists := out[column]; exists { + return nil, fmt.Errorf("captured duplicate column %q", column) } + out[column] = cloneValue(values[i]) } + return out, nil } -func (s *Source) emitFault(ctx context.Context) { - msg := "sqlite cdc source faulted" - if p := s.faultMsg.Load(); p != nil { - msg = *p +func cloneValue(value any) any { + bytes, ok := value.([]byte) + if !ok { + return value } - - s.log.Error("sqlite cdc source faulted", zap.String("source", s.name), zap.String("reason", msg)) - s.subs.publish(ctx, config.Change{Source: s.name, Op: "error", Error: msg}) + return append([]byte(nil), bytes...) } -func (s *Source) process(ctx context.Context, batch []capturedChange, mc metrics.Collector) { - s.refreshSchemaVersion(ctx) - for _, ch := range batch { - cols := s.columnsFor(ctx, ch.table) - if ch.ncols > 0 && len(cols) > 0 && len(cols) != ch.ncols { - s.invalidateColumns(ch.table) - cols = s.columnsFor(ctx, ch.table) - } - - op := opString(ch.op) - seq := s.seq.Add(1) - change := config.Change{ - Source: s.name, - Op: op, - Schema: normalizeSchema(ch.schema), - Table: ch.table, - Relation: ch.table, - Before: mapRow(cols, ch.old), - After: mapRow(cols, ch.new), - LSN: strconv.FormatUint(seq, 10), - } - s.subs.publish(ctx, change) - if mc != nil { - mc.CounterInc(changesCounter, metrics.Labels{"source": s.name, "op": op}) - } +func normalizeSchema(schema string) string { + schema = strings.TrimSpace(schema) + if schema == "" { + return "main" } + return schema } -func (s *Source) refreshSchemaVersion(ctx context.Context) { - var ver int64 - if err := s.readDB.QueryRowContext(ctx, "PRAGMA schema_version").Scan(&ver); err != nil { +func (s *Source) fail(err error) { + if err == nil { + err = ErrSourceClosed + } + s.mu.Lock() + if s.state == config.SourceStateStopped || s.stopping { + s.mu.Unlock() return } + if s.state == config.SourceStateFaulted { + s.mu.Unlock() + return + } + s.state = config.SourceStateFaulted + s.sourceErr = err + stream := s.observer + s.observer = nil + s.observerSource = nil + snapshotSubscriptions := make([]*subscription, 0, len(s.snapshotSubs)) + snapshotSubs := make([]sqlapi.MutationStream, 0, len(s.snapshotSubs)) + for sub, snapshotStream := range s.snapshotSubs { + snapshotSubscriptions = append(snapshotSubscriptions, sub) + snapshotSubs = append(snapshotSubs, snapshotStream) + delete(s.snapshotSubs, sub) + } + s.closeStatusLocked() + s.mu.Unlock() - prev := s.schemaVer.Swap(ver) - if prev != 0 && prev != ver { - s.colMu.Lock() - s.cols = make(map[string][]columnInfo) - s.colMu.Unlock() + s.subs.closeWithError(err) + for _, sub := range snapshotSubscriptions { + sub.closeWithError(err) } + for _, snapshotStream := range snapshotSubs { + _ = snapshotStream.Close() + } + if stream != nil { + _ = stream.Close() + } + s.log.Error("sqlite cdc source faulted", zap.String("source", s.name), zap.Error(err)) } -func (s *Source) invalidateColumns(table string) { - s.colMu.Lock() - delete(s.cols, table) - s.colMu.Unlock() +func (s *Source) closeStatusLocked() { + if s.status != nil && !s.statusClosed { + close(s.status) + s.statusClosed = true + } } -func (s *Source) onTick(mc metrics.Collector) { - if mc == nil { - return +// Stop closes the observer stream and all subscriptions. It never closes the +// DB observer itself: that belongs to the SQL resource generation. +func (s *Source) Stop(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() } - if info, err := os.Stat(s.file + "-wal"); err == nil { - mc.GaugeSet(walGauge, float64(info.Size()), metrics.Labels{"source": s.name}) + s.mu.Lock() + if s.state == config.SourceStateStopped { + s.mu.Unlock() + return nil } -} - -func (s *Source) columnsFor(ctx context.Context, table string) []columnInfo { - s.colMu.RLock() - cols, ok := s.cols[table] - s.colMu.RUnlock() - if ok { - return cols + s.stopping = true + s.state = config.SourceStateStopped + startCancel := s.startCancel + runCancel := s.runCancel + startDone := s.startDone + runDone := s.runDone + stream := s.observer + snapshotStreams := make([]sqlapi.MutationStream, 0, len(s.snapshotSubs)) + snapshotSubscriptions := make([]*subscription, 0, len(s.snapshotSubs)) + for sub, snapshotStream := range s.snapshotSubs { + snapshotSubscriptions = append(snapshotSubscriptions, sub) + snapshotStreams = append(snapshotStreams, snapshotStream) + delete(s.snapshotSubs, sub) } + s.mu.Unlock() - cols, err := resolveColumns(ctx, s.readDB, table) - if err != nil { - s.log.Warn("resolve columns failed; emitting positional column names", - zap.String("table", table), zap.Error(err)) + if startCancel != nil { + startCancel() + } + if runCancel != nil { + runCancel() + } + if stream != nil { + _ = stream.Close() + } + for _, sub := range snapshotSubscriptions { + sub.closeWithError(nil) + } + for _, snapshotStream := range snapshotStreams { + _ = snapshotStream.Close() + } - return nil + waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + if err := waitDone(waitCtx, startDone); err != nil { + s.subs.closeWithError(err) + return err + } + if err := waitDone(waitCtx, runDone); err != nil { + s.subs.closeWithError(err) + return err + } + snapshotDone := make(chan struct{}) + go func() { + s.snapshotWG.Wait() + close(snapshotDone) + }() + if err := waitDone(waitCtx, snapshotDone); err != nil { + s.subs.closeWithError(err) + return err } - s.colMu.Lock() - s.cols[table] = cols - s.colMu.Unlock() + s.subs.closeAll() + s.mu.Lock() + s.state = config.SourceStateStopped + s.stopping = false + s.observer = nil + s.observerSource = nil + s.startCancel = nil + s.runCancel = nil + s.closeStatusLocked() + s.mu.Unlock() + return nil +} - return cols +func waitDone(ctx context.Context, done <-chan struct{}) error { + if done == nil { + return nil + } + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } } -func normalizeSchema(schema string) string { - if schema == "" { - return "main" +// Subscribe exposes committed changes. Cursor resume remains unsupported +// because this process-local generation has no durable checkpoint; snapshots +// use the SQL-owned atomic fence and handoff stream per subscriber. +func (s *Source) Subscribe(ctx context.Context, opts config.StreamOptions) (config.Stream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + if opts.After != "" { + return nil, config.ErrUnsupported } - return schema + s.mu.RLock() + if s.state != config.SourceStateRunning || s.stopping { + s.mu.RUnlock() + return nil, config.ErrSourceNotReady + } + if opts.Snapshot || s.snapshot { + observer := s.observerSource + s.mu.RUnlock() + return s.subscribeSnapshot(ctx, observer, opts) + } + // Hold the source read lock while registering the subscription. Stop takes + // the write lock before closing the current subscriber set, so a stream + // cannot be inserted after lifecycle cleanup has already passed. + sub := s.subs.subscribe(s.name, opts) + s.mu.RUnlock() + return sub, nil } -func opString(op int) string { - switch op { - case cdcInsert: - return "insert" - case cdcUpdate: - return "update" - case cdcDelete: - return "delete" - default: - return "unknown" +func (s *Source) subscribeSnapshot(ctx context.Context, observer sqlapi.CommittedMutationSource, opts config.StreamOptions) (config.Stream, error) { + if observer == nil { + return nil, fmt.Errorf("%w: sqlite snapshot observer is unavailable", config.ErrUnsupported) + } + tables, noTables := intersectTables(s.tables, opts.Tables) + buffer := opts.Buffer + if buffer <= 0 { + buffer = defaultStreamBuffer + } + if buffer > maxStreamBuffer { + buffer = maxStreamBuffer + } + if noTables { + // An empty table intersection means no rows can match. Passing an empty + // list to the SQL observer would mean "all tables", so return an + // already-complete stream instead. + sub := newSubscription(s.name, opts, buffer) + sub.Close() + return sub, nil } -} -func openReadConn(file string) (*sql.DB, error) { - db, err := sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)+"&_query_only=ON") + stream, err := observer.Snapshot(ctx, sqlapi.SnapshotOptions{ + Tables: tables, + }) if err != nil { - return nil, fmt.Errorf("open read connection: %w", err) + return nil, err + } + if stream == nil { + return nil, errors.New("sqlite snapshot observer returned a nil stream") } - db.SetMaxOpenConns(2) - db.SetMaxIdleConns(2) + sub := newSubscription(s.name, opts, buffer) + + s.mu.Lock() + if s.state != config.SourceStateRunning || s.stopping || s.observerSource != observer { + s.mu.Unlock() + _ = stream.Close() + return nil, config.ErrSourceNotReady + } + s.snapshotSubs[sub] = stream + s.snapshotWG.Add(1) + s.mu.Unlock() - return db, nil + go s.runSnapshot(ctx, stream, sub) + return sub, nil } -func approxRowSize(vals []any) int { - size := 0 - for _, v := range vals { - switch t := v.(type) { - case []byte: - size += len(t) - case string: - size += len(t) - default: - size += 8 +func (s *Source) runSnapshot(ctx context.Context, stream sqlapi.SnapshotStream, sub *subscription) { + defer s.snapshotWG.Done() + defer func() { + s.mu.Lock() + delete(s.snapshotSubs, sub) + s.mu.Unlock() + }() + // The SQL observer owns the snapshot read transaction and scan worker. A + // subscriber can end for any reason (upstream error, downstream overflow, + // cancellation, or normal close), so every return path must release that + // upstream stream. Close is idempotent for the SQL observer stream. + defer func() { _ = stream.Close() }() + + watermark := stream.Watermark() + for { + select { + case <-ctx.Done(): + sub.closeWithError(ctx.Err()) + return + case <-sub.done: + return + case batch, ok := <-stream.Changes(): + if !ok { + err := stream.Err() + sub.closeWithError(err) + return + } + for i, mutation := range batch.Changes { + change, err := s.changeFromMutation(batch, i, mutation) + if err != nil { + sub.closeWithError(err) + return + } + change.Cursor = snapshotCursor(watermark, batch.Transaction, i) + if batch.Snapshot { + change.Op = "snapshot" + } + if batch.Snapshot { + if !sub.matchesSnapshot(change) { + continue + } + } else if !sub.matches(change) { + continue + } + sub.send(change) + if sub.isClosed() { + return + } + } } } +} - return size +func snapshotCursor(watermark, transaction string, index int) string { + parts := make([]string, 0, 3) + if watermark != "" { + parts = append(parts, watermark) + } + if transaction != "" { + parts = append(parts, transaction) + } + parts = append(parts, strconv.Itoa(index)) + return strings.Join(parts, "/") } -func openSnapshotConn(file string) (*sql.DB, error) { - db, err := sql.Open("sqlite3", "file:"+file+"?mode=rwc&_busy_timeout="+strconv.Itoa(auxBusyTimeoutMillisec)+"&_query_only=ON") - if err != nil { - return nil, fmt.Errorf("open snapshot connection: %w", err) +func intersectTables(source, requested []string) ([]string, bool) { + if len(source) == 0 { + return append([]string(nil), requested...), false + } + if len(requested) == 0 { + return append([]string(nil), source...), false + } + matched := make([]string, 0, len(requested)) + for _, want := range requested { + for _, allowed := range source { + if tableNamesEqual(want, allowed) { + matched = append(matched, want) + break + } + } } + return matched, len(matched) == 0 +} - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) +func tableNamesEqual(left, right string) bool { + left = strings.ToLower(strings.TrimSpace(left)) + right = strings.ToLower(strings.TrimSpace(right)) + if left == right { + return true + } + left = tableNameOnly(left) + right = tableNameOnly(right) + return left != "" && left == right +} - return db, nil +func tableNameOnly(value string) string { + if index := strings.LastIndexByte(value, '.'); index >= 0 { + return value[index+1:] + } + return value } + +var _ config.Source = (*Source)(nil) +var _ interface { + Start(context.Context) (<-chan any, error) + Stop(context.Context) error +} = (*Source)(nil) diff --git a/service/cdc/sqlite/source_test.go b/service/cdc/sqlite/source_test.go new file mode 100644 index 000000000..d1b735fd6 --- /dev/null +++ b/service/cdc/sqlite/source_test.go @@ -0,0 +1,563 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + sqlapi "github.com/wippyai/runtime/api/service/sql" + sqlconfig "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +type testResourceRegistry struct { + observer sqlapi.CommittedMutationSource + releases atomic.Int32 +} + +func (r *testResourceRegistry) Acquire(context.Context, registry.ID, resource.AccessMode) (resource.Resource[any], error) { + return &testDBResource{owner: r, value: sqlservice.DBResource{ + Type: sqlconfig.SQLite, + Observer: r.observer, + }}, nil +} + +func (*testResourceRegistry) List() ([]registry.ID, error) { return nil, nil } +func (*testResourceRegistry) Exists(registry.ID) bool { return true } + +type testDBResource struct { + owner *testResourceRegistry + value sqlservice.DBResource + once sync.Once +} + +func (r *testDBResource) Get() (any, error) { return r.value, nil } +func (r *testDBResource) Release() { + r.once.Do(func() { r.owner.releases.Add(1) }) +} + +type testObserver struct { + mu sync.Mutex + stream *testMutationStream + snapshot *testSnapshotStream + closed bool + closeN atomic.Int32 + subOpts sqlapi.MutationOptions +} + +func (o *testObserver) Subscribe(ctx context.Context, opts sqlapi.MutationOptions) (sqlapi.MutationStream, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + o.mu.Lock() + defer o.mu.Unlock() + if o.closed { + return nil, errors.New("test observer closed") + } + o.subOpts = opts + o.stream = newTestMutationStream() + return o.stream, nil +} + +func (o *testObserver) Snapshot(ctx context.Context, _ sqlapi.SnapshotOptions) (sqlapi.SnapshotStream, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + o.mu.Lock() + defer o.mu.Unlock() + if o.closed { + return nil, errors.New("test observer closed") + } + o.snapshot = &testSnapshotStream{ + testMutationStream: newTestMutationStream(), + watermark: "watermark-1", + } + return o.snapshot, nil +} + +func (o *testObserver) Close() error { + o.mu.Lock() + if o.closed { + o.mu.Unlock() + return nil + } + o.closed = true + stream := o.stream + snapshot := o.snapshot + o.mu.Unlock() + o.closeN.Add(1) + if stream != nil { + stream.closeWithError(errors.New("test SQL generation closed")) + } + if snapshot != nil { + snapshot.closeWithError(errors.New("test SQL generation closed")) + } + return nil +} + +func (o *testObserver) currentStream(t *testing.T) *testMutationStream { + t.Helper() + o.mu.Lock() + stream := o.stream + o.mu.Unlock() + require.NotNil(t, stream) + return stream +} + +func (o *testObserver) currentSnapshot(t *testing.T) *testSnapshotStream { + t.Helper() + o.mu.Lock() + stream := o.snapshot + o.mu.Unlock() + require.NotNil(t, stream) + return stream +} + +type testMutationStream struct { + changes chan sqlapi.MutationBatch + mu sync.Mutex + err error + closed bool + closeN atomic.Int32 +} + +type testSnapshotStream struct { + *testMutationStream + watermark string +} + +func (s *testSnapshotStream) Watermark() string { return s.watermark } + +func newTestMutationStream() *testMutationStream { + return &testMutationStream{changes: make(chan sqlapi.MutationBatch, 16)} +} + +func (s *testMutationStream) Changes() <-chan sqlapi.MutationBatch { return s.changes } + +func (s *testMutationStream) Err() error { + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err +} + +func (s *testMutationStream) Close() error { + s.closeWithError(nil) + return nil +} + +func (s *testMutationStream) closeWithError(err error) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.closed = true + s.err = err + close(s.changes) + s.mu.Unlock() + s.closeN.Add(1) +} + +func (s *testMutationStream) push(batch sqlapi.MutationBatch) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.closed { + s.changes <- batch + } +} + +func newTestSource(t *testing.T, observer *testObserver, opts sourceOptions) *Source { + t.Helper() + resources := &testResourceRegistry{observer: observer} + if opts.res == nil { + opts.res = resources + } + if opts.id.Name == "" { + opts.id = registry.NewID("app", "cdc") + } + if opts.name == "" { + opts.name = opts.id.String() + } + source, err := buildSource(opts) + require.NoError(t, err) + return source.(*Source) +} + +func receiveChange(t *testing.T, stream cdcapi.Stream) cdcapi.Change { + t.Helper() + select { + case change, ok := <-stream.Changes(): + if !ok { + if errStream, isErrStream := stream.(cdcapi.ErrStream); isErrStream { + require.Failf(t, "snapshot/live stream closed", "stream error: %v", errStream.Err()) + } + require.Fail(t, "snapshot/live stream closed") + } + return change + case <-time.After(time.Second): + t.Fatal("timed out waiting for SQLite CDC change") + return cdcapi.Change{} + } +} + +func requireNoChangeSource(t *testing.T, stream cdcapi.Stream) { + t.Helper() + select { + case change, ok := <-stream.Changes(): + if !ok { + t.Fatalf("stream closed while expecting no change") + } + t.Fatalf("unexpected change: %#v", change) + case <-time.After(50 * time.Millisecond): + } +} + +func waitStreamClosed(t *testing.T, stream cdcapi.Stream) error { + t.Helper() + deadline := time.After(time.Second) + for { + select { + case _, ok := <-stream.Changes(): + if !ok { + if errStream, ok := stream.(cdcapi.ErrStream); ok { + return errStream.Err() + } + return nil + } + case <-deadline: + t.Fatal("timed out waiting for SQLite CDC stream close") + return nil + } + } +} + +func TestSourceForwardsCommittedMutationWithStableShape(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{tables: []string{"users"}}) + status, err := source.Start(context.Background()) + require.NoError(t, err) + require.Equal(t, "sqlite cdc source started", <-status) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Ops: []string{"update"}}) + require.NoError(t, err) + defer stream.Close() + + assert.Equal(t, []string{"users"}, observer.subOpts.Tables) + observer.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "17", + Changes: []sqlapi.Mutation{{ + Schema: "main", + Table: "users", + Columns: []string{"id", "name"}, + Before: []any{int64(1), []byte("old")}, + After: []any{int64(1), []byte("new")}, + Op: "update", + }}, + }) + + change := receiveChange(t, stream) + assert.Equal(t, source.name, change.Source) + assert.Equal(t, source.id, change.SourceID) + assert.Equal(t, "update", change.Op) + assert.Equal(t, "main", change.Schema) + assert.Equal(t, "users", change.Relation) + assert.Equal(t, "17/0", change.Cursor) + assert.Equal(t, "17", change.Transaction) + assert.Equal(t, int64(1), change.After["id"]) + assert.Equal(t, []byte("old"), change.Before["name"]) + assert.Equal(t, []byte("new"), change.After["name"]) + + info := source.Info() + assert.True(t, info.Capabilities.Snapshot, "a tagged SQL observer exposes the snapshot handoff capability") + assert.False(t, info.Capabilities.Durable) + assert.False(t, info.Capabilities.Replayable) + assert.False(t, info.Capabilities.CapturesExternalWrites) + assert.True(t, info.Capabilities.BeforeImages) +} + +func TestSourceRejectsResumeButSupportsSnapshotHandoff(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + _, err = source.Subscribe(context.Background(), cdcapi.StreamOptions{After: "17/0"}) + assert.ErrorIs(t, err, cdcapi.ErrUnsupported) + snapshot, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + snapshot.Close() +} + +func TestSourceSnapshotSubscriberOwnsAtomicHandoff(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + live, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + defer live.Close() + snapshot, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{ + Snapshot: true, + }) + require.NoError(t, err) + defer snapshot.Close() + + observer.currentSnapshot(t).push(sqlapi.MutationBatch{ + Transaction: "snapshot-1", + Snapshot: true, + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id", "name"}, + After: []any{int64(1), "existing"}, Op: "insert", + }}, + }) + observer.currentSnapshot(t).push(sqlapi.MutationBatch{ + Transaction: "live-1", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id", "name"}, + After: []any{int64(2), "new"}, Op: "insert", + }}, + }) + + snapshotChange := receiveChange(t, snapshot) + assert.Equal(t, "snapshot", snapshotChange.Op) + assert.Equal(t, "watermark-1/snapshot-1/0", snapshotChange.Cursor) + assert.Equal(t, "existing", snapshotChange.After["name"]) + liveChange := receiveChange(t, snapshot) + assert.Equal(t, "insert", liveChange.Op) + assert.Equal(t, "new", liveChange.After["name"]) + requireNoChangeSource(t, live) +} + +func TestSourceDefaultSnapshotAppliesPerSubscriber(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{snapshot: true}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + observer.currentSnapshot(t).push(sqlapi.MutationBatch{ + Transaction: "snapshot-default", + Snapshot: true, + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id"}, After: []any{int64(7)}, Op: "insert", + }}, + }) + assert.Equal(t, "snapshot", receiveChange(t, stream).Op) +} + +func TestSourceSnapshotClosesUpstreamWhenItEnds(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + upstream := observer.currentSnapshot(t) + expected := errors.New("snapshot worker failed") + upstream.closeWithError(expected) + + assert.ErrorIs(t, waitStreamClosed(t, stream), expected) + assert.Eventually(t, func() bool { return upstream.closeN.Load() == 1 }, time.Second, time.Millisecond) +} + +func TestSourceSnapshotOverflowClosesUpstream(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{ + Snapshot: true, + Buffer: 1, + }) + require.NoError(t, err) + upstream := observer.currentSnapshot(t) + change := func(id int64) sqlapi.MutationBatch { + return sqlapi.MutationBatch{ + Transaction: "snapshot", + Snapshot: true, + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "users", Columns: []string{"id"}, + After: []any{id}, Op: "insert", + }}, + } + } + upstream.push(change(1)) + subscriber := stream.(*subscription) + assert.Eventually(t, func() bool { + subscriber.mu.Lock() + defer subscriber.mu.Unlock() + return len(subscriber.changes) == 1 + }, time.Second, time.Millisecond) + upstream.push(change(2)) + + assert.Eventually(t, func() bool { return upstream.closeN.Load() == 1 }, time.Second, time.Millisecond) + assert.ErrorIs(t, subscriber.Err(), errSubscriberOverflow) + stream.Close() +} + +func TestSourceStopsWithoutClosingSQLGeneration(t *testing.T) { + observer := &testObserver{} + resources := &testResourceRegistry{observer: observer} + source := newTestSource(t, observer, sourceOptions{res: resources}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + assert.Equal(t, int32(1), resources.releases.Load(), "the ordinary resource borrow ends after Subscribe") + + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + snapshot, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, int32(0), observer.closeN.Load(), "CDC does not own the SQL observer") + assert.NoError(t, waitStreamClosed(t, stream)) + assert.NoError(t, waitStreamClosed(t, snapshot)) +} + +func TestSourceCanRestartAfterStop(t *testing.T) { + observer := &testObserver{} + resources := &testResourceRegistry{observer: observer} + source := newTestSource(t, observer, sourceOptions{res: resources}) + + _, err := source.Start(context.Background()) + require.NoError(t, err) + first, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, cdcapi.SourceStateStopped, source.Info().State) + assert.NoError(t, waitStreamClosed(t, first)) + + _, err = source.Start(context.Background()) + require.NoError(t, err, "a supervisor stop is restartable; only resource-generation disposal is terminal") + second, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + observer.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "2", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(2)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(2), receiveChange(t, second).After["id"]) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, int32(2), resources.releases.Load()) +} + +func TestSourceFaultsWhenSQLGenerationCloses(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + require.NoError(t, observer.Close()) + assert.Error(t, waitStreamClosed(t, stream)) + assert.Equal(t, cdcapi.SourceStateFaulted, source.Info().State) + assert.NotEmpty(t, source.Info().Error) + assert.NoError(t, source.Stop(context.Background())) +} + +func TestSourceContextCancellationClosesSubscribers(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + ctx, cancel := context.WithCancel(context.Background()) + _, err := source.Start(ctx) + require.NoError(t, err) + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + cancel() + assert.ErrorIs(t, waitStreamClosed(t, stream), context.Canceled) + assert.Equal(t, cdcapi.SourceStateFaulted, source.Info().State) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceCanRestartAgainstReplacementSQLGeneration(t *testing.T) { + firstObserver := &testObserver{} + resources := &testResourceRegistry{observer: firstObserver} + source := newTestSource(t, firstObserver, sourceOptions{res: resources}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + first, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + require.NoError(t, firstObserver.Close()) + assert.Error(t, waitStreamClosed(t, first)) + require.NoError(t, source.Stop(context.Background())) + + secondObserver := &testObserver{} + resources.observer = secondObserver + _, err = source.Start(context.Background()) + require.NoError(t, err) + second, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + secondObserver.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "replacement-1", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(9)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(9), receiveChange(t, second).After["id"]) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceClosesOnlyOverflowedSubscriber(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, source.Stop(context.Background())) }() + + laggard, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 1}) + require.NoError(t, err) + reader, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 8}) + require.NoError(t, err) + + batch := sqlapi.MutationBatch{Transaction: "1", Changes: []sqlapi.Mutation{ + {Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(1)}, Op: "insert"}, + {Schema: "main", Table: "t", Columns: []string{"id"}, After: []any{int64(2)}, Op: "insert"}, + }} + observer.currentStream(t).push(batch) + + assert.Equal(t, "insert", receiveChange(t, reader).Op) + assert.ErrorIs(t, waitStreamClosed(t, laggard), errSubscriberOverflow) + assert.Equal(t, "insert", receiveChange(t, reader).Op) +} + +func TestSourceMalformedMutationFaultsClosedStream(t *testing.T) { + observer := &testObserver{} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + stream, err := source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + observer.currentStream(t).push(sqlapi.MutationBatch{Transaction: "1", Changes: []sqlapi.Mutation{{ + Table: "t", Columns: nil, After: []any{int64(1)}, Op: "insert", + }}}) + assert.Error(t, waitStreamClosed(t, stream)) + assert.Equal(t, cdcapi.SourceStateFaulted, source.Info().State) + assert.NoError(t, source.Stop(context.Background())) +} From 25a8fcf52507454307f0269329121da5e435e549 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:51:12 -0400 Subject: [PATCH 15/47] refactor(sql): own SQLite mutation observation per pool --- .github/workflows/ci-cd.yml | 26 + Makefile | 8 + api/service/sql/config.go | 29 +- api/service/sql/config_test.go | 16 + api/service/sql/errors.go | 20 +- api/service/sql/observer.go | 85 + boot/components/dispatchers/cdc_dispatcher.go | 6 +- boot/components/service/storage/sql.go | 5 +- service/cdc/sqlite/cdc_bench_test.go | 194 -- service/cdc/sqlite/driver.go | 75 + service/cdc/sqlite/driver_test.go | 16 + service/cdc/sqlite/helpers_test.go | 36 +- service/cdc/sqlite/hook.go | 203 -- service/cdc/sqlite/hook_registry_test.go | 41 - service/cdc/sqlite/integration_live_test.go | 224 ++ service/cdc/sqlite/integration_lua_test.go | 232 -- service/cdc/sqlite/integration_test.go | 296 --- service/cdc/sqlite/manager.go | 268 --- service/cdc/sqlite/manager_test.go | 74 - .../cdc/sqlite/redesign_integration_test.go | 329 --- service/cdc/sqlite/snapshot.go | 183 -- service/cdc/sqlite/source_stub.go | 2 +- service/cdc/sqlite/source_tagged_test.go | 6 + service/cdc/sqlite/subscribers.go | 211 +- service/cdc/sqlite/subscribers_snapshot.go | 44 - service/cdc/sqlite/subscribers_test.go | 41 +- service/sql/conn.go | 154 +- service/sql/conn_test.go | 18 + service/sql/driver.go | 35 - service/sql/driver_test.go | 28 +- service/sql/engine.go | 104 +- service/sql/engine/all/all.go | 20 +- service/sql/engine/sqlite/observer.go | 1918 +++++++++++++++++ service/sql/engine/sqlite/observer_stub.go | 27 + service/sql/engine/sqlite/observer_test.go | 630 ++++++ service/sql/engine/sqlite/sqlite.go | 37 +- service/sql/engine/sqlite/sqlite_test.go | 2 +- service/sql/engine/standard/standard.go | 31 +- service/sql/engine/standard/standard_test.go | 8 +- service/sql/engines_stub_test.go | 31 +- service/sql/factory.go | 27 +- service/sql/factory_test.go | 4 +- service/sql/manager.go | 37 +- service/sql/manager_test.go | 5 +- test.sh | 4 +- 45 files changed, 3544 insertions(+), 2246 deletions(-) create mode 100644 api/service/sql/observer.go delete mode 100644 service/cdc/sqlite/cdc_bench_test.go create mode 100644 service/cdc/sqlite/driver.go create mode 100644 service/cdc/sqlite/driver_test.go delete mode 100644 service/cdc/sqlite/hook.go delete mode 100644 service/cdc/sqlite/hook_registry_test.go create mode 100644 service/cdc/sqlite/integration_live_test.go delete mode 100644 service/cdc/sqlite/integration_lua_test.go delete mode 100644 service/cdc/sqlite/integration_test.go delete mode 100644 service/cdc/sqlite/manager.go delete mode 100644 service/cdc/sqlite/manager_test.go delete mode 100644 service/cdc/sqlite/redesign_integration_test.go delete mode 100644 service/cdc/sqlite/snapshot.go delete mode 100644 service/cdc/sqlite/subscribers_snapshot.go create mode 100644 service/sql/engine/sqlite/observer.go create mode 100644 service/sql/engine/sqlite/observer_stub.go create mode 100644 service/sql/engine/sqlite/observer_test.go diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 967b934be..00c57a639 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -86,3 +86,29 @@ jobs: SKIP_CLOUDSTORAGE_TESTS: "1" SKIP_DOCKER_TESTS: "1" run: make test + + test-sqlite-cdc: + name: Test SQLite CDC (Linux, CGO, race) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: '1.26.5' + cache: true + + - name: Install dependencies + run: go mod download + + - name: Run tagged SQLite CDC tests + env: + CGO_ENABLED: 1 + GORACE: "halt_on_error=1" + SKIP_TEMPORAL_TESTS: "1" + SKIP_CLOUDSTORAGE_TESTS: "1" + SKIP_DOCKER_TESTS: "1" + run: make test-cdc-sqlite diff --git a/Makefile b/Makefile index b57a48524..ffd0bf184 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,14 @@ test: go test ./boot/... -v -race -short go test --tags "fts5 sqlite_vec treesitter sqlite_preupdate_hook" ./cmd/... -v -race -short +# The default service test intentionally stays untagged so the SQLite stub and +# the non-CGO portability path remain covered. The real preupdate-hook source +# is exercised by the Linux CGO CI job through this target. +.PHONY: test-cdc-sqlite +test-cdc-sqlite: + CGO_ENABLED=1 go test ./service/sql/... ./service/cdc/sqlite -v -race -tags sqlite_preupdate_hook + CGO_ENABLED=1 go test ./service/cdc/sqlite -v -race -timeout 300s -tags "integration sqlite_preupdate_hook" + test-system: go test ./internal/... -v -race go test ./api/... -v -race diff --git a/api/service/sql/config.go b/api/service/sql/config.go index 230001078..2f40fef88 100644 --- a/api/service/sql/config.go +++ b/api/service/sql/config.go @@ -34,6 +34,13 @@ const ( // DefaultMaxLifetime is the default maximum lifetime of a connection DefaultMaxLifetime = 1 * time.Hour + + // DefaultMaxMutationChanges bounds the in-memory candidate row count held + // by the SQLite observer for one transaction. + DefaultMaxMutationChanges = 100000 + // DefaultMaxMutationBytes bounds the in-memory candidate value bytes held + // by the SQLite observer for one transaction. + DefaultMaxMutationBytes = 64 * 1024 * 1024 ) // EngineConfig is the contract every engine configuration satisfies, letting the @@ -66,10 +73,12 @@ type ( // SQLiteConfig defines SQLite-specific configuration SQLiteConfig struct { - Options map[string]string `json:"options"` - File string `json:"file"` - Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` - Pool PoolConfig `json:"pool"` + Options map[string]string `json:"options"` + File string `json:"file"` + Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` + Pool PoolConfig `json:"pool"` + MaxMutationChanges int `json:"max_mutation_changes,omitempty"` + MaxMutationBytes int `json:"max_mutation_bytes,omitempty"` } ) @@ -112,6 +121,12 @@ func (c *SQLiteConfig) InitDefaults() { // Initialize lifecycle defaults c.Lifecycle.InitDefaults() + if c.MaxMutationChanges == 0 { + c.MaxMutationChanges = DefaultMaxMutationChanges + } + if c.MaxMutationBytes == 0 { + c.MaxMutationBytes = DefaultMaxMutationBytes + } } // LifecycleConfig returns the supervisor lifecycle settings for the database. @@ -170,6 +185,12 @@ func (c *SQLiteConfig) Validate() error { if c.Pool.MaxLifetime <= 0 { return ErrInvalidMaxLifetime } + if c.MaxMutationChanges < 0 { + return ErrInvalidMaxMutationChanges + } + if c.MaxMutationBytes < 0 { + return ErrInvalidMaxMutationBytes + } return nil } diff --git a/api/service/sql/config_test.go b/api/service/sql/config_test.go index ef47c55bb..3b50d58de 100644 --- a/api/service/sql/config_test.go +++ b/api/service/sql/config_test.go @@ -285,9 +285,25 @@ func TestSQLiteConfig_InitDefaults(t *testing.T) { assert.Equal(t, DefaultMaxOpen, config.Pool.MaxOpen) assert.Equal(t, DefaultMaxIdle, config.Pool.MaxIdle) assert.Equal(t, DefaultMaxLifetime, config.Pool.MaxLifetime) + assert.Equal(t, DefaultMaxMutationChanges, config.MaxMutationChanges) + assert.Equal(t, DefaultMaxMutationBytes, config.MaxMutationBytes) assert.NotNil(t, config.Options) } +func TestSQLiteConfig_RejectsNegativeMutationLimits(t *testing.T) { + base := SQLiteConfig{File: ":memory:", Pool: PoolConfig{MaxLifetime: time.Hour}} + base.MaxMutationChanges = -1 + assert.ErrorIs(t, base.Validate(), ErrInvalidMaxMutationChanges) + base.MaxMutationChanges = 0 + base.MaxMutationBytes = -1 + assert.ErrorIs(t, base.Validate(), ErrInvalidMaxMutationBytes) + + base.MaxMutationChanges = -1 + base.MaxMutationBytes = 0 + base.InitDefaults() + assert.Equal(t, -1, base.MaxMutationChanges, "negative limits must not be silently defaulted") +} + func TestPoolConfig_UnmarshalJSON_InvalidDuration(t *testing.T) { jsonData := `{"max_lifetime":"invalid"}` var config PoolConfig diff --git a/api/service/sql/errors.go b/api/service/sql/errors.go index b8792bdfb..ca3b5101f 100644 --- a/api/service/sql/errors.go +++ b/api/service/sql/errors.go @@ -5,13 +5,15 @@ package sql import apierror "github.com/wippyai/runtime/api/error" var ( - ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) - ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) - ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) - ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) - ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) - ErrInvalidMaxOpen = apierror.New(apierror.Invalid, "max open connections must be non-negative").WithRetryable(apierror.False) - ErrInvalidMaxIdle = apierror.New(apierror.Invalid, "max idle connections must be non-negative").WithRetryable(apierror.False) - ErrInvalidMaxLifetime = apierror.New(apierror.Invalid, "max lifetime must be greater than 0").WithRetryable(apierror.False) - ErrFileRequired = apierror.New(apierror.Invalid, "file path is required").WithRetryable(apierror.False) + ErrHostRequired = apierror.New(apierror.Invalid, "host is required").WithRetryable(apierror.False) + ErrInvalidPort = apierror.New(apierror.Invalid, "port must be greater than 0").WithRetryable(apierror.False) + ErrDatabaseRequired = apierror.New(apierror.Invalid, "database is required").WithRetryable(apierror.False) + ErrUsernameRequired = apierror.New(apierror.Invalid, "username is required").WithRetryable(apierror.False) + ErrPasswordRequired = apierror.New(apierror.Invalid, "password is required").WithRetryable(apierror.False) + ErrInvalidMaxOpen = apierror.New(apierror.Invalid, "max open connections must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxIdle = apierror.New(apierror.Invalid, "max idle connections must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxLifetime = apierror.New(apierror.Invalid, "max lifetime must be greater than 0").WithRetryable(apierror.False) + ErrFileRequired = apierror.New(apierror.Invalid, "file path is required").WithRetryable(apierror.False) + ErrInvalidMaxMutationChanges = apierror.New(apierror.Invalid, "max mutation changes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxMutationBytes = apierror.New(apierror.Invalid, "max mutation bytes must be non-negative").WithRetryable(apierror.False) ) diff --git a/api/service/sql/observer.go b/api/service/sql/observer.go new file mode 100644 index 000000000..ba25b72b5 --- /dev/null +++ b/api/service/sql/observer.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sql + +import "context" + +// CommittedMutationSource is an optional capability exposed by a SQL resource. +// It reports mutations observed by the database engine after a transaction +// boundary. The interface deliberately contains no driver-specific handles or +// SQL connection types; consumers can use the same capability with different +// database engines. +// +// A source is owned by the SQL resource generation that created it. Closing the +// generation closes the source and all of its streams. +type CommittedMutationSource interface { + Subscribe(context.Context, MutationOptions) (MutationStream, error) + // Snapshot establishes the commit fence before starting its read view. The + // returned stream emits snapshot batches first and then live committed + // batches after the returned watermark, without exposing a database handle + // to the consumer. + Snapshot(context.Context, SnapshotOptions) (SnapshotStream, error) + Close() error +} + +// MutationOptions controls an observation stream. Filtering is intentionally +// expressed in the neutral mutation vocabulary so a consumer does not need to +// know which SQL driver produced the stream. +type MutationOptions struct { + Tables []string + Operations []string + MaxChanges int + MaxBytes int +} + +// SnapshotOptions selects tables and the maximum number of rows in one +// snapshot batch. A non-positive BatchSize uses the engine default. +type SnapshotOptions struct { + Tables []string + BatchSize int + MaxChanges int + MaxBytes int +} + +// MutationStream delivers committed mutation batches in commit order. +type MutationStream interface { + Changes() <-chan MutationBatch + Err() error + Close() error +} + +// SnapshotStream is an atomic snapshot/live handoff. Snapshot batches are +// marked with MutationBatch.Snapshot; once they are exhausted, the stream +// carries live batches in commit order. Watermark identifies the fence at +// which the read view was established and is process-local unless an engine +// documents a durable position. +type SnapshotStream interface { + MutationStream + Watermark() string +} + +// MutationBatch is the atomic unit emitted by an observation source. A batch +// belongs to one database transaction; an empty batch is not emitted. +type MutationBatch struct { + Transaction string + Snapshot bool + Changes []Mutation +} + +// Mutation is a driver-neutral row mutation. Values retain the database/sql +// driver's native scalar representation. Column names are captured with the +// mutation so schema changes after capture cannot relabel an earlier row. +type Mutation struct { + Schema string + Table string + Columns []string + // OldRowID is the row identifier before the change. It is zero for an + // insert; RowID is the identifier after the change and is zero for a + // delete. Drivers that cannot provide a stable row identifier must fail + // closed rather than emit an ambiguous mutation. + OldRowID int64 + RowID int64 + Before []any + After []any + Op string +} diff --git a/boot/components/dispatchers/cdc_dispatcher.go b/boot/components/dispatchers/cdc_dispatcher.go index 79467bc93..c4c0971e4 100644 --- a/boot/components/dispatchers/cdc_dispatcher.go +++ b/boot/components/dispatchers/cdc_dispatcher.go @@ -7,13 +7,13 @@ import ( "github.com/wippyai/runtime/api/boot" dispatcherapi "github.com/wippyai/runtime/api/dispatcher" - "github.com/wippyai/runtime/service/cdc/postgres" + "github.com/wippyai/runtime/service/cdc" ) const CDCDefaultWorkers = 4 func CDC() boot.Component { - var d *postgres.Dispatcher + var d *cdc.Dispatcher return boot.New(boot.P{ Name: CDCDispatcherName, @@ -24,7 +24,7 @@ func CDC() boot.Component { return ctx, ErrDispatcherNotFound } - d = postgres.NewDispatcher(postgres.WithWorkers(CDCDefaultWorkers)) + d = cdc.NewDispatcher(cdc.WithWorkers(CDCDefaultWorkers)) d.RegisterAll(reg.Register) return ctx, nil }, diff --git a/boot/components/service/storage/sql.go b/boot/components/service/storage/sql.go index b59786e91..3bcc3ec30 100644 --- a/boot/components/service/storage/sql.go +++ b/boot/components/service/storage/sql.go @@ -13,9 +13,7 @@ import ( bootpkg "github.com/wippyai/runtime/boot" bootsystem "github.com/wippyai/runtime/boot/components/system" "github.com/wippyai/runtime/service/sql" - - // Register the built-in SQL engines (postgres, mysql, sqlite) with the manager. - _ "github.com/wippyai/runtime/service/sql/engine/all" + "github.com/wippyai/runtime/service/sql/engine/all" ) func SQL() boot.Component { @@ -34,6 +32,7 @@ func SQL() boot.Component { bus, logger.Named("sql"), envRegistry, + sql.WithDriver(all.Drivers()...), ) if err != nil { return ctx, NewSQLManagerError(err) diff --git a/service/cdc/sqlite/cdc_bench_test.go b/service/cdc/sqlite/cdc_bench_test.go deleted file mode 100644 index 47ad687cf..000000000 --- a/service/cdc/sqlite/cdc_bench_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build sqlite_preupdate_hook - -package sqlite - -import ( - "context" - "database/sql" - "path/filepath" - "sort" - "testing" - "time" - - "github.com/wippyai/runtime/api/registry" - "github.com/wippyai/runtime/api/resource" - config "github.com/wippyai/runtime/api/service/cdc" - sqlconfig "github.com/wippyai/runtime/api/service/sql" - sqlservice "github.com/wippyai/runtime/service/sql" -) - -var benchCtx = context.Background() - -type benchRegistry struct{ db *sql.DB } - -func (r *benchRegistry) Acquire(context.Context, registry.ID, resource.AccessMode) (resource.Resource[any], error) { - return &benchResource{res: sqlservice.DBResource{DB: r.db, Type: sqlconfig.SQLite}}, nil -} -func (r *benchRegistry) List() ([]registry.ID, error) { return nil, nil } -func (r *benchRegistry) Exists(registry.ID) bool { return true } - -type benchResource struct{ res sqlservice.DBResource } - -func (b *benchResource) Get() (any, error) { return b.res, nil } -func (b *benchResource) Release() {} - -func benchPool(b *testing.B, driver string) *sql.DB { - b.Helper() - file := filepath.Join(b.TempDir(), "bench.db") - db, err := sql.Open(driver, "file:"+file+"?mode=rwc") - if err != nil { - b.Fatal(err) - } - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) - if _, err := db.ExecContext(benchCtx, "PRAGMA journal_mode=WAL"); err != nil { - b.Fatal(err) - } - b.Cleanup(func() { _ = db.Close() }) - return db -} - -func benchStartSource(b *testing.B, db *sql.DB) *Source { - b.Helper() - h, err := buildSource(sourceOptions{ - res: &benchRegistry{db: db}, - dbResource: registry.NewID("app", "db"), - name: "bench-src", - statusInterval: "1h", - }) - if err != nil { - b.Fatal(err) - } - src := h.(*Source) - if _, err := src.Start(context.Background()); err != nil { - b.Fatal(err) - } - b.Cleanup(func() { _ = src.Stop(context.Background()) }) - return src -} - -func drainStream(stream config.ChangeStream) func() { - stop := make(chan struct{}) - go func() { - for { - select { - case <-stop: - return - case _, ok := <-stream.Changes(): - if !ok { - return - } - } - } - }() - return func() { close(stop) } -} - -func reportLatencies(b *testing.B, lat []time.Duration) { - if len(lat) == 0 { - return - } - sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] }) - p := func(q float64) time.Duration { - idx := int(q * float64(len(lat)-1)) - return lat[idx] - } - b.ReportMetric(float64(p(0.50).Nanoseconds()), "p50-ns/commit") - b.ReportMetric(float64(p(0.99).Nanoseconds()), "p99-ns/commit") - b.ReportMetric(float64(p(0.999).Nanoseconds()), "p999-ns/commit") -} - -func benchWrite(b *testing.B, db *sql.DB, payload string) { - if _, err := db.ExecContext(benchCtx, "CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY, v TEXT)"); err != nil { - b.Fatal(err) - } - lat := make([]time.Duration, 0, b.N) - b.ResetTimer() - for i := 0; i < b.N; i++ { - start := time.Now() - if _, err := db.ExecContext(benchCtx, "INSERT INTO t (v) VALUES (?)", payload); err != nil { - b.Fatal(err) - } - lat = append(lat, time.Since(start)) - } - b.StopTimer() - reportLatencies(b, lat) -} - -func BenchmarkWriteHooksDisabled(b *testing.B) { - db := benchPool(b, "sqlite3") - benchWrite(b, db, "payload") -} - -func BenchmarkWriteHooksEnabledNoSubscribers(b *testing.B) { - db := benchPool(b, sqliteCDCDriver) - benchStartSource(b, db) - benchWrite(b, db, "payload") -} - -func BenchmarkWriteOneSubscriber(b *testing.B) { - db := benchPool(b, sqliteCDCDriver) - src := benchStartSource(b, db) - stop := drainStream(src.Subscribe(config.StreamOptions{Buffer: 1024})) - defer stop() - benchWrite(b, db, "payload") -} - -func BenchmarkWriteLargeBlobOneSubscriber(b *testing.B) { - db := benchPool(b, sqliteCDCDriver) - src := benchStartSource(b, db) - stop := drainStream(src.Subscribe(config.StreamOptions{Buffer: 1024})) - defer stop() - blob := make([]byte, 64*1024) - if _, err := db.ExecContext(benchCtx, "CREATE TABLE t (id INTEGER PRIMARY KEY, v BLOB)"); err != nil { - b.Fatal(err) - } - lat := make([]time.Duration, 0, b.N) - b.ResetTimer() - for i := 0; i < b.N; i++ { - start := time.Now() - if _, err := db.ExecContext(benchCtx, "INSERT INTO t (v) VALUES (?)", blob); err != nil { - b.Fatal(err) - } - lat = append(lat, time.Since(start)) - } - b.StopTimer() - reportLatencies(b, lat) -} - -func BenchmarkWriteSaturatedSubscriber(b *testing.B) { - db := benchPool(b, sqliteCDCDriver) - src := benchStartSource(b, db) - _ = src.Subscribe(config.StreamOptions{Buffer: 1}) - benchWrite(b, db, "payload") -} - -func BenchmarkLargeTransaction(b *testing.B) { - db := benchPool(b, sqliteCDCDriver) - src := benchStartSource(b, db) - stop := drainStream(src.Subscribe(config.StreamOptions{Buffer: 4096})) - defer stop() - if _, err := db.ExecContext(benchCtx, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); err != nil { - b.Fatal(err) - } - const rowsPerTxn = 1000 - b.ResetTimer() - for i := 0; i < b.N; i++ { - tx, err := db.BeginTx(benchCtx, nil) - if err != nil { - b.Fatal(err) - } - for j := 0; j < rowsPerTxn; j++ { - if _, err := tx.ExecContext(benchCtx, "INSERT INTO t (v) VALUES (?)", "payload"); err != nil { - b.Fatal(err) - } - } - if err := tx.Commit(); err != nil { - b.Fatal(err) - } - } - b.StopTimer() - b.ReportMetric(float64(rowsPerTxn), "rows/txn") -} diff --git a/service/cdc/sqlite/driver.go b/service/cdc/sqlite/driver.go new file mode 100644 index 000000000..dca05dbeb --- /dev/null +++ b/service/cdc/sqlite/driver.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "context" + + "go.uber.org/zap" + + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/resource" + config "github.com/wippyai/runtime/api/service/cdc" + "github.com/wippyai/runtime/api/supervisor" + cdcservice "github.com/wippyai/runtime/service/cdc" + entryutil "github.com/wippyai/runtime/system/entry" +) + +// managedSource is the narrow internal contract shared by build-tagged +// implementations. Keeping it in the untagged driver file makes the package +// fail closed without the SQLite preupdate build tag rather than exposing a +// half-working source. +type managedSource interface { + config.Source + supervisor.Service +} + +type sourceOptions struct { + res resource.Registry + log *zap.Logger + id registry.ID + dbResource registry.ID + name string + statusInterval string + tables []string + snapshot bool + lifecycle supervisor.LifecycleConfig +} + +// Driver wires the SQLite CDC implementation into the driver-neutral CDC +// manager. It owns no process-global SQL driver registration. +type Driver struct{} + +func NewDriver() cdcservice.Driver { return Driver{} } + +func (Driver) Kind() registry.Kind { return config.SQLite } + +func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice.Dependencies) (cdcservice.ManagedSource, error) { + if deps.Resources == nil { + return nil, ErrResourceRegRequired + } + cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, deps.Transcoder, entry) + if err != nil { + return nil, NewInvalidConfigError(err) + } + if err := cfg.Validate(); err != nil { + return nil, NewInvalidConfigError(err) + } + log := deps.Logger + if log == nil { + log = zap.NewNop() + } + return buildSource(sourceOptions{ + res: deps.Resources, + log: log.With(zap.String("id", entry.ID.String())), + id: entry.ID, + dbResource: registry.ParseID(cfg.DBResource), + name: entry.ID.String(), + statusInterval: cfg.StatusInterval, + tables: cfg.Tables, + snapshot: cfg.Snapshot, + lifecycle: cfg.Lifecycle, + }) +} + +var _ cdcservice.Driver = Driver{} diff --git a/service/cdc/sqlite/driver_test.go b/service/cdc/sqlite/driver_test.go new file mode 100644 index 000000000..92d769e07 --- /dev/null +++ b/service/cdc/sqlite/driver_test.go @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MPL-2.0 + +package sqlite + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + config "github.com/wippyai/runtime/api/service/cdc" +) + +func TestDriverKind(t *testing.T) { + assert.Equal(t, config.SQLite, Driver{}.Kind()) + assert.Equal(t, config.SQLite, NewDriver().Kind()) +} diff --git a/service/cdc/sqlite/helpers_test.go b/service/cdc/sqlite/helpers_test.go index cd0c325e6..f0520c3d6 100644 --- a/service/cdc/sqlite/helpers_test.go +++ b/service/cdc/sqlite/helpers_test.go @@ -8,34 +8,30 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestSchemaAllowed(t *testing.T) { - assert.True(t, schemaAllowed("")) - assert.True(t, schemaAllowed("main")) - assert.True(t, schemaAllowed("MAIN")) - assert.False(t, schemaAllowed("temp")) - assert.False(t, schemaAllowed("attached")) -} - func TestNormalizeSchema(t *testing.T) { assert.Equal(t, "main", normalizeSchema("")) assert.Equal(t, "temp", normalizeSchema("temp")) assert.Equal(t, "audit", normalizeSchema("audit")) } -func TestOpString(t *testing.T) { - assert.Equal(t, "insert", opString(cdcInsert)) - assert.Equal(t, "update", opString(cdcUpdate)) - assert.Equal(t, "delete", opString(cdcDelete)) - assert.Equal(t, "unknown", opString(9999)) +func TestValuesByColumnRequiresCapturedShape(t *testing.T) { + got, err := valuesByColumn([]string{"id", "name"}, []any{int64(1), "one"}) + require.NoError(t, err) + assert.Equal(t, map[string]any{"id": int64(1), "name": "one"}, got) + + _, err = valuesByColumn([]string{"id"}, []any{int64(1), "extra"}) + assert.Error(t, err) + _, err = valuesByColumn([]string{"id", "id"}, []any{int64(1), int64(2)}) + assert.Error(t, err) } -func TestApproxRowSize(t *testing.T) { - assert.Equal(t, 0, approxRowSize(nil)) - assert.Equal(t, 8, approxRowSize([]any{int64(1)})) - assert.Equal(t, 3, approxRowSize([]any{[]byte{1, 2, 3}})) - assert.Equal(t, 5, approxRowSize([]any{"hello"})) - assert.Equal(t, 8+3+5, approxRowSize([]any{1.5, []byte("abc"), "hello"})) - assert.Equal(t, 8, approxRowSize([]any{nil})) +func TestValuesByColumnCopiesBytes(t *testing.T) { + value := []byte{1, 2, 3} + got, err := valuesByColumn([]string{"blob"}, []any{value}) + require.NoError(t, err) + value[0] = 9 + assert.Equal(t, []byte{1, 2, 3}, got["blob"]) } diff --git a/service/cdc/sqlite/hook.go b/service/cdc/sqlite/hook.go deleted file mode 100644 index 389fe212b..000000000 --- a/service/cdc/sqlite/hook.go +++ /dev/null @@ -1,203 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build sqlite_preupdate_hook - -package sqlite - -import ( - "path/filepath" - "sync" - - "database/sql" - - "github.com/mattn/go-sqlite3" - - apierror "github.com/wippyai/runtime/api/error" - sqlconfig "github.com/wippyai/runtime/api/service/sql" - sqlservice "github.com/wippyai/runtime/service/sql" -) - -const sqliteCDCDriver = "sqlite3_wippy" - -const ( - cdcInsert = sqlite3.SQLITE_INSERT - cdcUpdate = sqlite3.SQLITE_UPDATE - cdcDelete = sqlite3.SQLITE_DELETE -) - -var ( - errNotSQLiteConn = apierror.New(apierror.Invalid, "underlying connection is not a SQLite connection").WithRetryable(apierror.False) - errCDCMemoryUnsupported = apierror.New(apierror.Invalid, "sqlite cdc requires a file-backed database").WithRetryable(apierror.False) - errCaptureOwned = apierror.New(apierror.Conflict, "sqlite cdc capture already owned for this database").WithRetryable(apierror.True) -) - -type cdcSink interface { - PreUpdate(op int, schema, table string, rowid int64, ncols int, old, new []any, scanErr error) - Commit() - Rollback() -} - -type captureOwner struct { - sink cdcSink - token uint64 -} - -var ( - cdcMu sync.Mutex - captures = make(map[string]captureOwner) - captureNo uint64 -) - -func init() { - sql.Register(sqliteCDCDriver, &sqlite3.SQLiteDriver{ConnectHook: cdcConnectHook}) - sqlservice.RegisterDriver(sqlconfig.SQLite, sqliteCDCDriver) -} - -func cdcConnectHook(conn *sqlite3.SQLiteConn) error { - file := normalizeCDCPath(conn.GetFilename("main")) - if file == "" { - return nil - } - - cdcMu.Lock() - defer cdcMu.Unlock() - if owner, ok := captures[file]; ok { - bindCDCHooks(conn, owner.sink) - } - - return nil -} - -func claimCapture(file string, sink cdcSink) (uint64, error) { - cdcMu.Lock() - defer cdcMu.Unlock() - if _, ok := captures[file]; ok { - return 0, errCaptureOwned - } - - captureNo++ - captures[file] = captureOwner{sink: sink, token: captureNo} - - return captureNo, nil -} - -func releaseCapture(file string, token uint64) { - cdcMu.Lock() - defer cdcMu.Unlock() - if owner, ok := captures[file]; ok && owner.token == token { - delete(captures, file) - } -} - -func installHooksOnRaw(raw any, sink cdcSink) (string, uint64, error) { - conn, ok := raw.(*sqlite3.SQLiteConn) - if !ok { - return "", 0, errNotSQLiteConn - } - - file := normalizeCDCPath(conn.GetFilename("main")) - if file == "" { - return "", 0, errCDCMemoryUnsupported - } - - token, err := claimCapture(file, sink) - if err != nil { - return file, 0, err - } - - bindCDCHooks(conn, sink) - - return file, token, nil -} - -func applyOwnerOnRaw(raw any, file string) error { - conn, ok := raw.(*sqlite3.SQLiteConn) - if !ok { - return errNotSQLiteConn - } - - cdcMu.Lock() - defer cdcMu.Unlock() - if owner, ok := captures[file]; ok { - bindCDCHooks(conn, owner.sink) - } else { - clearHooks(conn) - } - - return nil -} - -func clearHooks(conn *sqlite3.SQLiteConn) { - conn.RegisterPreUpdateHook(nil) - conn.RegisterCommitHook(nil) - conn.RegisterRollbackHook(nil) -} - -func normalizeCDCPath(path string) string { - if path == "" { - return "" - } - - if resolved, err := filepath.EvalSymlinks(path); err == nil { - path = resolved - } - - abs, err := filepath.Abs(path) - if err != nil { - return filepath.Clean(path) - } - - return abs -} - -func bindCDCHooks(conn *sqlite3.SQLiteConn, sink cdcSink) { - conn.RegisterPreUpdateHook(func(d sqlite3.SQLitePreUpdateData) { - count := d.Count() - var oldRow, newRow []any - var rowid int64 - var scanErr error - switch d.Op { - case sqlite3.SQLITE_INSERT: - newRow, scanErr = scanPreUpdateRow(&d, count, true) - rowid = d.NewRowID - case sqlite3.SQLITE_DELETE: - oldRow, scanErr = scanPreUpdateRow(&d, count, false) - rowid = d.OldRowID - case sqlite3.SQLITE_UPDATE: - oldRow, scanErr = scanPreUpdateRow(&d, count, false) - if scanErr == nil { - newRow, scanErr = scanPreUpdateRow(&d, count, true) - } - rowid = d.NewRowID - } - - sink.PreUpdate(d.Op, d.DatabaseName, d.TableName, rowid, count, oldRow, newRow, scanErr) - }) - conn.RegisterCommitHook(func() int { - sink.Commit() - - return 0 - }) - conn.RegisterRollbackHook(func() { - sink.Rollback() - }) -} - -func scanPreUpdateRow(d *sqlite3.SQLitePreUpdateData, count int, isNew bool) ([]any, error) { - if count <= 0 { - return nil, nil - } - - vals := make([]any, count) - var err error - if isNew { - err = d.New(vals...) - } else { - err = d.Old(vals...) - } - if err != nil { - return nil, err - } - - return vals, nil -} diff --git a/service/cdc/sqlite/hook_registry_test.go b/service/cdc/sqlite/hook_registry_test.go deleted file mode 100644 index 320e46624..000000000 --- a/service/cdc/sqlite/hook_registry_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build sqlite_preupdate_hook - -package sqlite - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -type fakeSink struct{} - -func (fakeSink) PreUpdate(int, string, string, int64, int, []any, []any, error) {} -func (fakeSink) Commit() {} -func (fakeSink) Rollback() {} - -func TestCaptureRegistrySingleOwnerAndTokenGuard(t *testing.T) { - const file = "/tmp/wippy-cdc-registry-test.db" - - a := fakeSink{} - b := fakeSink{} - - tok, err := claimCapture(file, a) - require.NoError(t, err) - t.Cleanup(func() { releaseCapture(file, tok) }) - - _, err = claimCapture(file, b) - require.ErrorIs(t, err, errCaptureOwned, "a second owner must be refused") - - releaseCapture(file, tok+1000) - _, err = claimCapture(file, b) - require.ErrorIs(t, err, errCaptureOwned, "release with a stale token must not evict the owner") - - releaseCapture(file, tok) - tok2, err := claimCapture(file, b) - require.NoError(t, err, "after the real owner releases, a new owner can claim") - require.NotEqual(t, tok, tok2) - releaseCapture(file, tok2) -} diff --git a/service/cdc/sqlite/integration_live_test.go b/service/cdc/sqlite/integration_live_test.go new file mode 100644 index 000000000..de870f920 --- /dev/null +++ b/service/cdc/sqlite/integration_live_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build integration && sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wippyai/runtime/api/registry" + cdcapi "github.com/wippyai/runtime/api/service/cdc" + sqlapi "github.com/wippyai/runtime/api/service/sql" + sqlconfig "github.com/wippyai/runtime/api/service/sql" + sqliteengine "github.com/wippyai/runtime/service/sql/engine/sqlite" +) + +type integrationDB struct { + db *sql.DB + observer sqlapi.CommittedMutationSource + resources *testResourceRegistry + source *Source +} + +func openIntegrationDB(t *testing.T, opts sourceOptions) *integrationDB { + t.Helper() + ctx := context.Background() + file := filepath.Join(t.TempDir(), "cdc.db") + cfg := &sqlconfig.SQLiteConfig{File: file} + cfg.InitDefaults() + driver := sqliteengine.NewDriver() + opened, err := driver.Open(ctx, cfg) + require.NoError(t, err) + require.NoError(t, driver.Prepare(ctx, opened.DB, cfg)) + driver.Tune(opened.DB, cfg) + require.NotNil(t, opened.Observer) + + resources := &testResourceRegistry{observer: opened.Observer} + if opts.res == nil { + opts.res = resources + } + if opts.id.Name == "" { + opts.id = registry.NewID("app", "sqlite-cdc") + } + if opts.name == "" { + opts.name = opts.id.String() + } + sourceValue, err := buildSource(opts) + require.NoError(t, err) + source := sourceValue.(*Source) + + result := &integrationDB{db: opened.DB, observer: opened.Observer, resources: resources, source: source} + t.Cleanup(func() { + _ = source.Stop(context.Background()) + _ = opened.Observer.Close() + _ = opened.DB.Close() + }) + return result +} + +func requireNoChange(t *testing.T, stream cdcapi.Stream) { + t.Helper() + select { + case change, ok := <-stream.Changes(): + if !ok { + if errStream, isErrStream := stream.(cdcapi.ErrStream); isErrStream { + t.Fatalf("stream closed unexpectedly: %v", errStream.Err()) + } + t.Fatal("stream closed unexpectedly") + } + t.Fatalf("unexpected CDC change: %#v", change) + case <-time.After(100 * time.Millisecond): + } +} + +func TestIntegrationLiveCommitAndFilters(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{tables: []string{"users"}}) + _, err := db.db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = db.db.Exec(`CREATE TABLE audit (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{Ops: []string{"insert"}}) + require.NoError(t, err) + defer stream.Close() + + _, err = db.db.Exec(`INSERT INTO users (id, value) VALUES (1, 'one')`) + require.NoError(t, err) + change := receiveChange(t, stream) + assert.Equal(t, "insert", change.Op) + assert.Equal(t, "users", change.Table) + assert.Equal(t, int64(1), change.After["id"]) + + _, err = db.db.Exec(`UPDATE users SET value = 'two' WHERE id = 1`) + require.NoError(t, err) + requireNoChange(t, stream) + _, err = db.db.Exec(`INSERT INTO audit (id, value) VALUES (1, 'ignored')`) + require.NoError(t, err) + requireNoChange(t, stream) +} + +func startSourceForIntegration(source *Source) error { + _, err := source.Start(context.Background()) + return err +} + +func TestIntegrationRollbackAndSavepointPublishOnlyCommittedRows(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + defer stream.Close() + + tx, err := db.db.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'kept')`) + require.NoError(t, err) + _, err = tx.Exec(`SAVEPOINT nested`) + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (2, 'discarded')`) + require.NoError(t, err) + _, err = tx.Exec(`ROLLBACK TO SAVEPOINT nested`) + require.NoError(t, err) + _, err = tx.Exec(`RELEASE SAVEPOINT nested`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + change := receiveChange(t, stream) + assert.Equal(t, int64(1), change.After["id"]) + assert.Equal(t, []byte("kept"), change.After["value"]) + requireNoChange(t, stream) + + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (3, 'rolled back')`) + require.NoError(t, err) + change = receiveChange(t, stream) + assert.Equal(t, int64(3), change.After["id"]) +} + +func TestIntegrationFailedStatementFailsClosed(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + defer stream.Close() + + tx, err := db.db.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT OR FAIL INTO items (id, value) VALUES (1, 'same'), (2, 'same')`) + assert.Error(t, err) + require.NoError(t, tx.Commit()) + + var count int + require.NoError(t, db.db.QueryRow(`SELECT count(*) FROM items`).Scan(&count)) + assert.Equal(t, 1, count, "SQLite must retain the applied prefix") + streamErr := waitStreamClosed(t, stream) + require.Error(t, streamErr) + assert.Contains(t, streamErr.Error(), "cannot determine statement outcome") +} + +func TestIntegrationSubscriberOverflowDoesNotRollbackApplication(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 1}) + require.NoError(t, err) + + for i := 1; i <= 32; i++ { + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (?, 'value')`, i) + require.NoError(t, err) + } + err = waitStreamClosed(t, stream) + assert.ErrorIs(t, err, errSubscriberOverflow) + var count int + require.NoError(t, db.db.QueryRow(`SELECT count(*) FROM items`).Scan(&count)) + assert.Equal(t, 32, count) +} + +func TestIntegrationSQLGenerationCloseFaultsSource(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + require.NoError(t, db.observer.Close()) + assert.Error(t, waitStreamClosed(t, stream)) + assert.Equal(t, cdcapi.SourceStateFaulted, db.source.Info().State) +} + +func TestIntegrationSnapshotHandoffIsPerSubscriber(t *testing.T) { + db := openIntegrationDB(t, sourceOptions{}) + _, err := db.db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (1, 'existing')`) + require.NoError(t, err) + require.NoError(t, startSourceForIntegration(db.source)) + stream, err := db.source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + require.NoError(t, err) + defer stream.Close() + + snapshot := receiveChange(t, stream) + assert.Equal(t, "snapshot", snapshot.Op) + assert.Equal(t, int64(1), snapshot.After["id"]) + assert.NotEmpty(t, snapshot.Cursor) + + _, err = db.db.Exec(`INSERT INTO items (id, value) VALUES (2, 'live')`) + require.NoError(t, err) + live := receiveChange(t, stream) + assert.Equal(t, "insert", live.Op) + assert.Equal(t, int64(2), live.After["id"]) +} diff --git a/service/cdc/sqlite/integration_lua_test.go b/service/cdc/sqlite/integration_lua_test.go deleted file mode 100644 index c471125d8..000000000 --- a/service/cdc/sqlite/integration_lua_test.go +++ /dev/null @@ -1,232 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build integration && sqlite_preupdate_hook - -package sqlite - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - lua "github.com/wippyai/go-lua" - "go.uber.org/zap" - - ctxapi "github.com/wippyai/runtime/api/context" - "github.com/wippyai/runtime/api/dispatcher" - "github.com/wippyai/runtime/api/event" - "github.com/wippyai/runtime/api/payload" - "github.com/wippyai/runtime/api/pid" - "github.com/wippyai/runtime/api/process" - "github.com/wippyai/runtime/api/registry" - "github.com/wippyai/runtime/api/relay" - apiruntime "github.com/wippyai/runtime/api/runtime" - "github.com/wippyai/runtime/api/security" - cdcapi "github.com/wippyai/runtime/api/service/cdc" - apisup "github.com/wippyai/runtime/api/supervisor" - "github.com/wippyai/runtime/runtime/lua/engine" - luapayload "github.com/wippyai/runtime/runtime/lua/engine/payload" - cdcmod "github.com/wippyai/runtime/runtime/lua/modules/cdc" - pgcdc "github.com/wippyai/runtime/service/cdc/postgres" - "github.com/wippyai/runtime/system/eventbus" - systempayload "github.com/wippyai/runtime/system/payload" - sysrelay "github.com/wippyai/runtime/system/relay" - "github.com/wippyai/runtime/system/scheduler" - "github.com/wippyai/runtime/system/scheduler/pool/inline" - syssup "github.com/wippyai/runtime/system/supervisor" -) - -type signalingStreamer struct { - inner cdcapi.SourceStreamer - ready chan struct{} - once sync.Once -} - -func (s *signalingStreamer) Stream(ctx context.Context, source string, opts cdcapi.StreamOptions) (cdcapi.ChangeStream, cdcapi.SourceInfo, error) { - stream, info, err := s.inner.Stream(ctx, source, opts) - if err == nil { - s.once.Do(func() { close(s.ready) }) - } - return stream, info, err -} - -func TestLuaSeesRealRunningSQLiteSourceAndItsChanges(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) - require.NoError(t, err) - - bus := eventbus.NewBus() - sup := syssup.NewSupervisor(bus, zap.NewNop()) - supCtx, supCancel := context.WithCancel(context.Background()) - defer supCancel() - require.NoError(t, sup.Start(supCtx)) - - transcoder := systempayload.NewTranscoder() - luapayload.Register(transcoder) - - manager, err := NewManager(transcoder, bus, zap.NewNop(), &fakeRegistry{db: db}) - require.NoError(t, err) - - entryID := registry.NewID("test", "cdc-lua-e2e") - src, err := buildSource(sourceOptions{ - res: &fakeRegistry{db: db}, - dbResource: registry.NewID("app", "db"), - name: entryID.String(), - statusInterval: "1h", - }) - require.NoError(t, err) - srcImpl := src.(*Source) - manager.sources[entryID] = src - manager.storeInfo(registry.Entry{ID: entryID, Kind: cdcapi.SQLite}, &cdcapi.SQLiteConfig{DBResource: "app:db"}) - - lc := apisup.LifecycleConfig{AutoStart: true} - lc.InitDefaults() - bus.Send(supCtx, event.Event{System: registry.System, Kind: registry.TxBegin, Path: "tx"}) - bus.Send(supCtx, event.Event{ - System: apisup.System, - Kind: apisup.ServiceRegister, - Path: entryID.String(), - Data: &apisup.Entry{Service: src, Config: lc}, - }) - bus.Send(supCtx, event.Event{System: registry.System, Kind: registry.TxCommit, Path: "tx"}) - - require.Eventually(t, func() bool { - return srcImpl.Epoch() != "" - }, 15*time.Second, 50*time.Millisecond, "supervisor must auto-start the registered source") - defer func() { - stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = srcImpl.Stop(stopCtx) - stopCancel() - }() - - streamer := &signalingStreamer{inner: manager, ready: make(chan struct{})} - root := security.SetStrictMode(ctxapi.NewRootContext(), false) - payload.WithTranscoder(root, transcoder) - root = cdcapi.WithSourceInspector(root, manager) - root = cdcapi.WithSourceStreamer(root, streamer) - - node := sysrelay.NewNode("cdc-lua-sqlite-node") - root = relay.WithNode(root, node) - - runCtx, runCancel := context.WithTimeout(root, 30*time.Second) - defer runCancel() - - dispReg := scheduler.NewRegistry() - cdcDisp := pgcdc.NewDispatcher(pgcdc.WithWorkers(1)) - require.NoError(t, cdcDisp.Start(runCtx)) - defer func() { require.NoError(t, cdcDisp.Stop(context.Background())) }() - cdcDisp.RegisterAll(func(id dispatcher.CommandID, h dispatcher.Handler) { - dispReg.Register(id, h) - }) - - const expectedEmail = "lua-sqlite-e2e@wippy.ai" - const hostID = "test.cdc.sqlite.lua" - factory := func() (process.Process, error) { - cfg := engine.FactoryConfig{ - ScriptName: "cdc_sqlite_lua_e2e", - Script: ` -local cdc = require("cdc") - -local function main() - local rows, err = cdc.list_sources() - if err ~= nil then return nil, "list_sources error: " .. tostring(err) end - if #rows ~= 1 then return nil, "expected 1 source, got " .. tostring(#rows) end - - local r = rows[1] - if r.engine ~= "sqlite" then return nil, "wrong engine: " .. tostring(r.engine) end - - local stream, stream_err = cdc.stream("test:cdc-lua-e2e", { - tables = {"users"}, - ops = {"insert"}, - buffer = 8, - }) - if stream_err ~= nil then return nil, "stream error: " .. tostring(stream_err) end - - local ch = stream:channel() - local change, ok = ch:receive() - stream:release() - if ok ~= true then return nil, "stream closed before change" end - if change.op ~= "insert" then return nil, "wrong op: " .. tostring(change.op) end - if change.source ~= "test:cdc-lua-e2e" then return nil, "wrong source: " .. tostring(change.source) end - if change.schema ~= "main" then return nil, "wrong schema: " .. tostring(change.schema) end - if change.table ~= "users" then return nil, "wrong table: " .. tostring(change.table) end - if change.after == nil then return nil, "missing after table" end - if change.after.email ~= "` + expectedEmail + `" then - return nil, "wrong email: " .. tostring(change.after.email) - end - - return change.after.email -end - -return { main = main } -`, - ModuleBinders: append(engine.CoreBinders(), func(l *lua.LState) error { - engine.LoadModuleDef(l, cdcmod.Module) - return nil - }), - } - return engine.NewFactory(cfg)() - } - - pool, err := inline.New(factory, dispReg) - require.NoError(t, err) - defer pool.Stop() - require.NoError(t, node.RegisterHost(hostID, pool)) - - frameCtx, frame := ctxapi.OpenFrameContext(runCtx) - defer ctxapi.ReleaseFrameContext(frame) - testPID := pid.PID{Host: hostID, UniqID: "cdc-lua-e2e"} - testPID = testPID.Precomputed() - require.NoError(t, apiruntime.SetFramePID(frameCtx, testPID)) - - resultCh := make(chan *apiruntime.Result, 1) - errCh := make(chan error, 1) - go func() { - result, err := pool.Call(frameCtx, "main", nil) - if err != nil { - errCh <- err - return - } - resultCh <- result - }() - - select { - case <-streamer.ready: - case err := <-errCh: - require.NoError(t, err) - case result := <-resultCh: - t.Fatalf("Lua returned before subscribing: value=%v err=%v", func() any { - if result != nil && result.Value != nil { - return result.Value.Data() - } - return nil - }(), func() any { - if result != nil { - return result.Error - } - return nil - }()) - case <-runCtx.Done(): - t.Fatal("timed out waiting for Lua CDC stream subscription") - } - - _, err = db.Exec(`INSERT INTO users (email) VALUES (?)`, expectedEmail) - require.NoError(t, err) - - var result *apiruntime.Result - select { - case result = <-resultCh: - case err := <-errCh: - require.NoError(t, err) - case <-runCtx.Done(): - t.Fatal("timed out waiting for Lua to receive CDC change") - } - require.NotNil(t, result) - require.NoError(t, result.Error) - require.NotNil(t, result.Value) - got, ok := result.Value.Data().(lua.LString) - require.True(t, ok, "expected Lua string result, got %T", result.Value.Data()) - require.Equal(t, expectedEmail, string(got)) -} diff --git a/service/cdc/sqlite/integration_test.go b/service/cdc/sqlite/integration_test.go deleted file mode 100644 index a0d3eab2b..000000000 --- a/service/cdc/sqlite/integration_test.go +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build integration && sqlite_preupdate_hook - -package sqlite - -import ( - "context" - "database/sql" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/wippyai/runtime/api/registry" - "github.com/wippyai/runtime/api/resource" - config "github.com/wippyai/runtime/api/service/cdc" - sqlconfig "github.com/wippyai/runtime/api/service/sql" - sqlservice "github.com/wippyai/runtime/service/sql" -) - -type fakeResource struct{ res sqlservice.DBResource } - -func (f *fakeResource) Get() (any, error) { return f.res, nil } -func (f *fakeResource) Release() {} - -type fakeRegistry struct{ db *sql.DB } - -func (r *fakeRegistry) Acquire(context.Context, registry.ID, resource.AccessMode) (resource.Resource[any], error) { - return &fakeResource{res: sqlservice.DBResource{DB: r.db, Type: sqlconfig.SQLite}}, nil -} -func (r *fakeRegistry) List() ([]registry.ID, error) { return nil, nil } -func (r *fakeRegistry) Exists(registry.ID) bool { return true } - -func openPool(t *testing.T) (*sql.DB, string) { - t.Helper() - file := filepath.Join(t.TempDir(), "app.db") - db, err := sql.Open("sqlite3_wippy", "file:"+file+"?mode=rwc") - require.NoError(t, err) - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) - _, err = db.Exec("PRAGMA journal_mode=WAL") - require.NoError(t, err) - return db, file -} - -func newSource(t *testing.T, db *sql.DB, opts sourceOptions) *Source { - t.Helper() - opts.res = &fakeRegistry{db: db} - opts.dbResource = registry.NewID("app", "db") - if opts.name == "" { - opts.name = "test-src" - } - if opts.statusInterval == "" { - opts.statusInterval = "1s" - } - h, err := buildSource(opts) - require.NoError(t, err) - return h.(*Source) -} - -func waitChange(t *testing.T, ch <-chan config.Change) config.Change { - t.Helper() - select { - case c := <-ch: - return c - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for change") - return config.Change{} - } -} - -func TestIntegrationInsertUpdateDelete(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT, balance REAL)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - - _, err = db.Exec(`INSERT INTO users (id, email, balance) VALUES (1, 'a@b.com', 42.5)`) - require.NoError(t, err) - ins := waitChange(t, stream.Changes()) - assert.Equal(t, "insert", ins.Op) - assert.Equal(t, "users", ins.Table) - assert.Equal(t, "a@b.com", ins.After["email"]) - assert.Equal(t, 42.5, ins.After["balance"]) - assert.Equal(t, int64(1), ins.After["id"]) - assert.Nil(t, ins.Before) - - _, err = db.Exec(`UPDATE users SET balance = 99.0 WHERE id = 1`) - require.NoError(t, err) - upd := waitChange(t, stream.Changes()) - assert.Equal(t, "update", upd.Op) - assert.Equal(t, 42.5, upd.Before["balance"]) - assert.Equal(t, 99.0, upd.After["balance"]) - - _, err = db.Exec(`DELETE FROM users WHERE id = 1`) - require.NoError(t, err) - del := waitChange(t, stream.Changes()) - assert.Equal(t, "delete", del.Op) - assert.Equal(t, "a@b.com", del.Before["email"]) - assert.Nil(t, del.After) -} - -func TestIntegrationValueFidelity(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT, qty INTEGER, price REAL, blob BLOB, note TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - - _, err = db.Exec(`INSERT INTO items (id, name, qty, price, blob, note) VALUES (1, 'widget', 7, 1.25, X'00ff10', NULL)`) - require.NoError(t, err) - - c := waitChange(t, stream.Changes()) - assert.Equal(t, "widget", c.After["name"]) - assert.Equal(t, int64(7), c.After["qty"]) - assert.Equal(t, 1.25, c.After["price"]) - assert.Equal(t, []byte{0x00, 0xff, 0x10}, c.After["blob"]) - assert.Nil(t, c.After["note"]) -} - -func TestIntegrationRollbackDiscarded(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - - tx, err := db.Begin() - require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO t (id, v) VALUES (1, 'rolled-back')`) - require.NoError(t, err) - require.NoError(t, tx.Rollback()) - - _, err = db.Exec(`INSERT INTO t (id, v) VALUES (2, 'committed')`) - require.NoError(t, err) - - c := waitChange(t, stream.Changes()) - assert.Equal(t, "insert", c.Op) - assert.Equal(t, "committed", c.After["v"]) - assert.Equal(t, int64(2), c.After["id"]) -} - -func TestIntegrationSnapshotBootstrap(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{Snapshot: true}) - defer stream.Close() - - snap := waitChange(t, stream.Changes()) - assert.Equal(t, "snapshot", snap.Op) - assert.Equal(t, "existing@b.com", snap.After["email"]) - - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (2, 'new@b.com')`) - require.NoError(t, err) - live := waitChange(t, stream.Changes()) - assert.Equal(t, "insert", live.Op) - assert.Equal(t, "new@b.com", live.After["email"]) -} - -func TestIntegrationRestartResnapshots(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) - require.NoError(t, err) - - first := newSource(t, db, sourceOptions{name: "src"}) - _, err = first.Start(context.Background()) - require.NoError(t, err) - s1 := first.Subscribe(config.StreamOptions{Snapshot: true}) - snap := waitChange(t, s1.Changes()) - require.Equal(t, "snapshot", snap.Op) - s1.Close() - require.NoError(t, first.Stop(context.Background())) - - epoch1 := first.Epoch() - - second := newSource(t, db, sourceOptions{name: "src"}) - _, err = second.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = second.Stop(context.Background()) }() - - assert.NotEqual(t, epoch1, second.Epoch(), "each Start must mint a fresh session epoch") - - s2 := second.Subscribe(config.StreamOptions{Snapshot: true}) - defer s2.Close() - - got := waitChange(t, s2.Changes()) - assert.Equal(t, "snapshot", got.Op, "honest v1: no durable checkpoint, so a fresh snapshot subscriber re-sees existing state") - assert.Equal(t, "existing@b.com", got.After["email"]) -} - -func TestIntegrationLaggardDoesNotStallWrites(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - laggard := src.Subscribe(config.StreamOptions{Buffer: 1}) - defer laggard.Close() - - done := make(chan error, 1) - go func() { - for i := 0; i < 500; i++ { - if _, e := db.Exec(`INSERT INTO t (v) VALUES ('x')`); e != nil { - done <- e - return - } - } - done <- nil - }() - - select { - case e := <-done: - require.NoError(t, e) - case <-time.After(10 * time.Second): - t.Fatal("writes stalled: a non-reading subscriber blocked the writer") - } - - reader := src.Subscribe(config.StreamOptions{}) - defer reader.Close() - _, err = db.Exec(`INSERT INTO t (v) VALUES ('final')`) - require.NoError(t, err) - - deadline := time.After(10 * time.Second) - for { - select { - case got := <-reader.Changes(): - if got.Op == "insert" && got.After["v"] == "final" { - return - } - case <-deadline: - t.Fatal("did not observe the 'final' insert on a fresh subscriber") - } - } -} - -func TestIntegrationTableAllowlist(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{tables: []string{"users"}}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - - _, err = db.Exec(`INSERT INTO orders (id, v) VALUES (1, 'ignored')`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO users (id, v) VALUES (1, 'captured')`) - require.NoError(t, err) - - c := waitChange(t, stream.Changes()) - assert.Equal(t, "users", c.Table) - assert.Equal(t, "captured", c.After["v"]) -} diff --git a/service/cdc/sqlite/manager.go b/service/cdc/sqlite/manager.go deleted file mode 100644 index 9df617838..000000000 --- a/service/cdc/sqlite/manager.go +++ /dev/null @@ -1,268 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -package sqlite - -import ( - "context" - "sync" - - "github.com/wippyai/runtime/api/event" - "github.com/wippyai/runtime/api/payload" - "github.com/wippyai/runtime/api/registry" - "github.com/wippyai/runtime/api/resource" - config "github.com/wippyai/runtime/api/service/cdc" - "github.com/wippyai/runtime/api/supervisor" - entryutil "github.com/wippyai/runtime/system/entry" - "go.uber.org/zap" -) - -type sourceHandle interface { - supervisor.Service - Subscribe(opts config.StreamOptions) config.ChangeStream - closeSubscriptions() - Epoch() string - Faulted() (bool, string) -} - -type sourceOptions struct { - res resource.Registry - log *zap.Logger - dbResource registry.ID - name string - statusInterval string - tables []string - snapshot bool -} - -type Manager struct { - dtt payload.Transcoder - bus event.Bus - res resource.Registry - log *zap.Logger - sources map[registry.ID]sourceHandle - infos map[registry.ID]config.SourceInfo - infosByName map[string]registry.ID - mu sync.Mutex -} - -func NewManager(dtt payload.Transcoder, bus event.Bus, log *zap.Logger, res resource.Registry) (*Manager, error) { - if dtt == nil { - return nil, ErrTranscoderRequired - } - if bus == nil { - return nil, ErrEventBusRequired - } - if res == nil { - return nil, ErrResourceRegRequired - } - if log == nil { - log = zap.NewNop() - } - return &Manager{ - dtt: dtt, - bus: bus, - res: res, - log: log, - sources: make(map[registry.ID]sourceHandle), - infos: make(map[registry.ID]config.SourceInfo), - infosByName: make(map[string]registry.ID), - }, nil -} - -func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - - if entry.Kind != config.SQLite { - return NewUnsupportedEntryKindError(entry.Kind) - } - if _, exists := m.sources[entry.ID]; exists { - return NewServiceExistsError(entry.ID) - } - - cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } - if err := cfg.Validate(); err != nil { - return NewInvalidConfigError(err) - } - - src, err := buildSource(m.sourceOptions(entry, cfg)) - if err != nil { - return NewSourceCreationError(err) - } - - m.sources[entry.ID] = src - m.storeInfo(entry, cfg) - m.register(ctx, entry, src, cfg.Lifecycle) - return nil -} - -func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - - if entry.Kind != config.SQLite { - return NewUnsupportedEntryKindError(entry.Kind) - } - if _, exists := m.sources[entry.ID]; !exists { - return NewServiceNotFoundError(entry.ID) - } - - cfg, err := entryutil.DecodeEntryConfig[config.SQLiteConfig](ctx, m.dtt, entry) - if err != nil { - return NewInvalidConfigError(err) - } - if err := cfg.Validate(); err != nil { - return NewInvalidConfigError(err) - } - - src, err := buildSource(m.sourceOptions(entry, cfg)) - if err != nil { - return NewSourceCreationError(err) - } - - if old := m.sources[entry.ID]; old != nil { - old.closeSubscriptions() - } - m.removeInfo(entry.ID) - m.unregister(ctx, entry) - - m.sources[entry.ID] = src - m.storeInfo(entry, cfg) - m.register(ctx, entry, src, cfg.Lifecycle) - return nil -} - -func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - - src, exists := m.sources[entry.ID] - if !exists { - return NewServiceNotFoundError(entry.ID) - } - src.closeSubscriptions() - m.removeInfo(entry.ID) - m.unregister(ctx, entry) - delete(m.sources, entry.ID) - return nil -} - -func (m *Manager) sourceOptions(entry registry.Entry, cfg *config.SQLiteConfig) sourceOptions { - return sourceOptions{ - res: m.res, - log: m.log.With(zap.String("id", entry.ID.String())), - name: entry.ID.String(), - dbResource: registry.ParseID(cfg.DBResource), - tables: cfg.Tables, - statusInterval: cfg.StatusInterval, - snapshot: cfg.Snapshot, - } -} - -func (m *Manager) storeInfo(entry registry.Entry, cfg *config.SQLiteConfig) { - info := config.SourceInfo{ - Name: entry.ID.String(), - Engine: "sqlite", - DBResource: cfg.DBResource, - Tables: append([]string(nil), cfg.Tables...), - Snapshot: cfg.Snapshot, - } - m.infos[entry.ID] = info - m.infosByName[info.Name] = entry.ID -} - -func (m *Manager) removeInfo(id registry.ID) { - if info, ok := m.infos[id]; ok { - if current, present := m.infosByName[info.Name]; present && current == id { - delete(m.infosByName, info.Name) - } - delete(m.infos, id) - } -} - -func (m *Manager) List() []config.SourceInfo { - m.mu.Lock() - defer m.mu.Unlock() - - out := make([]config.SourceInfo, 0, len(m.infos)) - for id, info := range m.infos { - out = append(out, m.enrich(id, info)) - } - return out -} - -func (m *Manager) Get(name string) (config.SourceInfo, bool) { - m.mu.Lock() - defer m.mu.Unlock() - - if id, ok := m.infosByName[name]; ok { - if info, present := m.infos[id]; present { - return m.enrich(id, info), true - } - } - return config.SourceInfo{}, false -} - -func (m *Manager) enrich(id registry.ID, info config.SourceInfo) config.SourceInfo { - src := m.sources[id] - if src == nil { - return info - } - - info.Epoch = src.Epoch() - if faulted, reason := src.Faulted(); faulted { - info.Faulted = true - info.Error = reason - } - return info -} - -func (m *Manager) Stream(_ context.Context, name string, opts config.StreamOptions) (config.ChangeStream, config.SourceInfo, error) { - m.mu.Lock() - src, info, ok := m.lookupSourceLocked(name) - m.mu.Unlock() - if !ok { - return nil, config.SourceInfo{}, NewServiceNotFoundError(registry.ParseID(name)) - } - - info.Epoch = src.Epoch() - if faulted, reason := src.Faulted(); faulted { - info.Faulted = true - info.Error = reason - } - return src.Subscribe(opts), info, nil -} - -func (m *Manager) lookupSourceLocked(name string) (sourceHandle, config.SourceInfo, bool) { - if id, ok := m.infosByName[name]; ok { - if src := m.sources[id]; src != nil { - return src, m.infos[id], true - } - } - return nil, config.SourceInfo{}, false -} - -func (m *Manager) register(ctx context.Context, entry registry.Entry, src sourceHandle, lifecycle supervisor.LifecycleConfig) { - m.bus.Send(ctx, event.Event{ - System: supervisor.System, - Kind: supervisor.ServiceRegister, - Path: entry.ID.String(), - Data: &supervisor.Entry{ - Service: src, - Config: lifecycle, - }, - }) - m.log.Info("added sqlite cdc source", zap.String("id", entry.ID.String()), zap.String("kind", entry.Kind)) -} - -func (m *Manager) unregister(ctx context.Context, entry registry.Entry) { - m.bus.Send(ctx, event.Event{ - System: supervisor.System, - Kind: supervisor.ServiceRemove, - Path: entry.ID.String(), - }) - m.log.Info("removed sqlite cdc source", zap.String("id", entry.ID.String())) -} diff --git a/service/cdc/sqlite/manager_test.go b/service/cdc/sqlite/manager_test.go deleted file mode 100644 index c4dca30dc..000000000 --- a/service/cdc/sqlite/manager_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -package sqlite - -import ( - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/wippyai/runtime/api/registry" - config "github.com/wippyai/runtime/api/service/cdc" -) - -func newInspectorManager() *Manager { - return &Manager{ - sources: map[registry.ID]sourceHandle{}, - infos: map[registry.ID]config.SourceInfo{}, - infosByName: map[string]registry.ID{}, - } -} - -func TestNewManagerValidation(t *testing.T) { - _, err := NewManager(nil, nil, nil, nil) - assert.ErrorIs(t, err, ErrTranscoderRequired) -} - -func TestManagerStoreAndList(t *testing.T) { - m := newInspectorManager() - idA := registry.NewID("test", "id-a") - idB := registry.NewID("test", "id-b") - - m.storeInfo(registry.Entry{ID: idA, Kind: config.SQLite}, &config.SQLiteConfig{ - DBResource: "app:db", - Tables: []string{"users"}, - Snapshot: true, - }) - m.storeInfo(registry.Entry{ID: idB, Kind: config.SQLite}, &config.SQLiteConfig{ - DBResource: "app:db2", - }) - - infos := m.List() - require.Len(t, infos, 2) - names := []string{infos[0].Name, infos[1].Name} - sort.Strings(names) - assert.Equal(t, []string{idA.String(), idB.String()}, names) - - got, ok := m.Get(idA.String()) - require.True(t, ok) - assert.Equal(t, "sqlite", got.Engine) - assert.Equal(t, "app:db", got.DBResource) - assert.Equal(t, []string{"users"}, got.Tables) - assert.True(t, got.Snapshot) -} - -func TestManagerGetMiss(t *testing.T) { - m := newInspectorManager() - _, ok := m.Get("missing") - assert.False(t, ok) -} - -func TestManagerRemoveInfo(t *testing.T) { - m := newInspectorManager() - id := registry.NewID("test", "id-a") - m.storeInfo(registry.Entry{ID: id, Kind: config.SQLite}, &config.SQLiteConfig{DBResource: "app:db"}) - - m.removeInfo(id) - - _, ok := m.Get(id.String()) - assert.False(t, ok) - assert.Empty(t, m.List()) - assert.NotContains(t, m.infosByName, id.String()) -} diff --git a/service/cdc/sqlite/redesign_integration_test.go b/service/cdc/sqlite/redesign_integration_test.go deleted file mode 100644 index ac9a46405..000000000 --- a/service/cdc/sqlite/redesign_integration_test.go +++ /dev/null @@ -1,329 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build integration && sqlite_preupdate_hook - -package sqlite - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - config "github.com/wippyai/runtime/api/service/cdc" -) - -func TestIntegrationLateSubscriberGetsSnapshotThenLive(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{Snapshot: true}) - defer stream.Close() - - snap := waitChange(t, stream.Changes()) - assert.Equal(t, "snapshot", snap.Op) - assert.Equal(t, "main", snap.Schema) - assert.Equal(t, "existing@b.com", snap.After["email"]) - - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (2, 'live@b.com')`) - require.NoError(t, err) - - live := waitChange(t, stream.Changes()) - assert.Equal(t, "insert", live.Op) - assert.Equal(t, "live@b.com", live.After["email"]) -} - -func TestIntegrationOpFilteredSubscriberStillGetsSnapshot(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'existing@b.com')`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{Snapshot: true, Ops: []string{"insert"}}) - defer stream.Close() - - snap := waitChange(t, stream.Changes()) - assert.Equal(t, "snapshot", snap.Op, "op filter must not drop snapshot rows") - assert.Equal(t, "existing@b.com", snap.After["email"]) -} - -func TestIntegrationSecondSourceRefused(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src1 := newSource(t, db, sourceOptions{name: "src-1"}) - _, err = src1.Start(context.Background()) - require.NoError(t, err) - - src2 := newSource(t, db, sourceOptions{name: "src-2"}) - ctx2, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) - defer cancel() - _, err = src2.Start(ctx2) - require.Error(t, err, "a second capture owner on the same database must be refused") - - require.NoError(t, src1.Stop(context.Background())) - - src3 := newSource(t, db, sourceOptions{name: "src-3"}) - _, err = src3.Start(context.Background()) - require.NoError(t, err, "after the first owner stops, a new source may claim capture") - require.NoError(t, src3.Stop(context.Background())) -} - -func TestIntegrationOverflowFaultsWithoutStallingWriter(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - src.maxRows = 5 - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - - tx, err := db.Begin() - require.NoError(t, err) - for i := 0; i < 20; i++ { - _, err = tx.Exec(`INSERT INTO t (v) VALUES ('x')`) - require.NoError(t, err) - } - require.NoError(t, tx.Commit(), "the application commit must succeed even when CDC overflows") - - c := waitChange(t, stream.Changes()) - assert.Equal(t, "error", c.Op) - assert.NotEmpty(t, c.Error) - - faulted, reason := src.Faulted() - assert.True(t, faulted) - assert.NotEmpty(t, reason) - - var n int - require.NoError(t, db.QueryRow(`SELECT count(*) FROM t`).Scan(&n)) - assert.Equal(t, 20, n, "all application rows must be durably written despite the CDC fault") -} - -func TestIntegrationSubscribeAfterFaultGetsTerminalError(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - src.maxRows = 2 - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - tx, err := db.Begin() - require.NoError(t, err) - for i := 0; i < 5; i++ { - _, err = tx.Exec(`INSERT INTO t (v) VALUES ('x')`) - require.NoError(t, err) - } - require.NoError(t, tx.Commit()) - - require.Eventually(t, func() bool { - faulted, _ := src.Faulted() - return faulted - }, 5*time.Second, 10*time.Millisecond) - - late := src.Subscribe(config.StreamOptions{}) - defer late.Close() - c := waitChange(t, late.Changes()) - assert.Equal(t, "error", c.Op, "a subscriber joining a faulted source must receive a terminal error") -} - -func TestIntegrationAlterTableColumnsNotStale(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, a TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - - _, err = db.Exec(`INSERT INTO t (id, a) VALUES (1, 'first')`) - require.NoError(t, err) - c1 := waitChange(t, stream.Changes()) - assert.Equal(t, "first", c1.After["a"]) - - _, err = db.Exec(`ALTER TABLE t ADD COLUMN b TEXT`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO t (id, a, b) VALUES (2, 'second', 'added')`) - require.NoError(t, err) - - c2 := waitChange(t, stream.Changes()) - assert.Equal(t, "second", c2.After["a"]) - assert.Equal(t, "added", c2.After["b"], "column cache must be invalidated after ALTER TABLE") -} - -func TestIntegrationTempTableNotCapturedAsMain(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE main_t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - stream := src.Subscribe(config.StreamOptions{}) - defer stream.Close() - - _, err = db.Exec(`CREATE TEMP TABLE tmp_t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO tmp_t (id, v) VALUES (1, 'temp-only')`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO main_t (id, v) VALUES (1, 'main-row')`) - require.NoError(t, err) - - c := waitChange(t, stream.Changes()) - assert.Equal(t, "main", c.Schema) - assert.Equal(t, "main_t", c.Table) - assert.Equal(t, "main-row", c.After["v"], "writes to the temp database must not be reported as main") -} - -func TestIntegrationStopCleansUpWithExpiredContext(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{name: "src-a"}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - - expired, cancel := context.WithCancel(context.Background()) - cancel() - require.NoError(t, src.Stop(expired), "Stop must complete cleanup even when the caller context is already cancelled") - - src2 := newSource(t, db, sourceOptions{name: "src-b"}) - _, err = src2.Start(context.Background()) - require.NoError(t, err, "hooks must be released after Stop so a new source can claim capture") - require.NoError(t, src2.Stop(context.Background())) -} - -func TestIntegrationMultipleSubscribersFilters(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - subAll := src.Subscribe(config.StreamOptions{}) - defer subAll.Close() - subUsers := src.Subscribe(config.StreamOptions{Tables: []string{"users"}}) - defer subUsers.Close() - - _, err = db.Exec(`INSERT INTO orders (id, v) VALUES (1, 'o')`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO users (id, v) VALUES (1, 'u')`) - require.NoError(t, err) - - only := waitChange(t, subUsers.Changes()) - assert.Equal(t, "users", only.Table, "the users-filtered subscriber must skip the orders write") - assert.Equal(t, "u", only.After["v"]) - - first := waitChange(t, subAll.Changes()) - assert.Equal(t, "orders", first.Table) - second := waitChange(t, subAll.Changes()) - assert.Equal(t, "users", second.Table) -} - -func TestIntegrationSnapshotCoversMultipleTables(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)`) - require.NoError(t, err) - _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, total REAL)`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO users (id, email) VALUES (1, 'a@b.com')`) - require.NoError(t, err) - _, err = db.Exec(`INSERT INTO orders (id, total) VALUES (1, 99.5)`) - require.NoError(t, err) - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - sub := src.Subscribe(config.StreamOptions{Snapshot: true}) - defer sub.Close() - - seen := map[string]bool{} - for i := 0; i < 2; i++ { - c := waitChange(t, sub.Changes()) - require.Equal(t, "snapshot", c.Op) - seen[c.Table] = true - } - assert.True(t, seen["users"], "snapshot must cover the users table") - assert.True(t, seen["orders"], "snapshot must cover the orders table") -} - -func TestIntegrationSnapshotWithConcurrentWritesNoGap(t *testing.T) { - db, _ := openPool(t) - _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) - require.NoError(t, err) - - const preRows = 20 - const liveRows = 20 - for i := 1; i <= preRows; i++ { - _, err = db.Exec(`INSERT INTO t (id, v) VALUES (?, 'pre')`, i) - require.NoError(t, err) - } - - src := newSource(t, db, sourceOptions{}) - _, err = src.Start(context.Background()) - require.NoError(t, err) - defer func() { _ = src.Stop(context.Background()) }() - - sub := src.Subscribe(config.StreamOptions{Snapshot: true}) - defer sub.Close() - - go func() { - for i := preRows + 1; i <= preRows+liveRows; i++ { - _, _ = db.Exec(`INSERT INTO t (id, v) VALUES (?, 'live')`, i) - } - }() - - seen := map[int64]bool{} - deadline := time.After(15 * time.Second) - for len(seen) < preRows+liveRows { - select { - case c := <-sub.Changes(): - if id, ok := c.After["id"].(int64); ok { - seen[id] = true - } - case <-deadline: - t.Fatalf("did not observe all rows without a gap; saw %d/%d", len(seen), preRows+liveRows) - } - } - for i := int64(1); i <= preRows+liveRows; i++ { - assert.Truef(t, seen[i], "row %d missing: snapshot+live must cover every row with no gap", i) - } -} diff --git a/service/cdc/sqlite/snapshot.go b/service/cdc/sqlite/snapshot.go deleted file mode 100644 index e2c4faf0c..000000000 --- a/service/cdc/sqlite/snapshot.go +++ /dev/null @@ -1,183 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build sqlite_preupdate_hook - -package sqlite - -import ( - "context" - "database/sql" - "strings" - - config "github.com/wippyai/runtime/api/service/cdc" -) - -func (s *Source) bootstrapSubscription(ctx context.Context, sub *subscription) { - defer sub.finishSnapshot() - - snapDB, err := openSnapshotConn(s.file) - if err != nil { - sub.fail("snapshot open: " + err.Error()) - return - } - defer func() { _ = snapDB.Close() }() - - tx, err := s.fenceAndBegin(ctx, snapDB) - if err != nil { - if ctx.Err() != nil { - return - } - sub.fail("snapshot begin: " + err.Error()) - return - } - defer func() { _ = tx.Rollback() }() - - tables, err := s.snapshotTables(ctx, tx, sub) - if err != nil { - if ctx.Err() != nil { - return - } - sub.fail("snapshot tables: " + err.Error()) - return - } - - for _, table := range tables { - if err := s.streamSnapshotTable(ctx, tx, sub, table); err != nil { - if ctx.Err() != nil || sub.isClosed() { - return - } - sub.fail("snapshot table " + table + ": " + err.Error()) - return - } - } -} - -func (s *Source) fenceAndBegin(ctx context.Context, snapDB *sql.DB) (*sql.Tx, error) { - s.mu.Lock() - writerDB := s.writerDB - s.mu.Unlock() - if writerDB == nil { - return nil, ErrSourceClosed - } - - wc, err := writerDB.Conn(ctx) - if err != nil { - return nil, err - } - defer func() { _ = wc.Close() }() - - if err := wc.PingContext(ctx); err != nil { - return nil, err - } - - tx, err := snapDB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) - if err != nil { - return nil, err - } - - var count int64 - if err := tx.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master").Scan(&count); err != nil { - _ = tx.Rollback() - return nil, err - } - - return tx, nil -} - -func (s *Source) snapshotTables(ctx context.Context, tx *sql.Tx, sub *subscription) ([]string, error) { - rows, err := tx.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - var tables []string - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - return nil, err - } - if s.tableAllowed(name) && sub.tableAllowed(name) { - tables = append(tables, name) - } - } - - return tables, rows.Err() -} - -func (s *Source) streamSnapshotTable(ctx context.Context, tx *sql.Tx, sub *subscription, table string) error { - cols := s.columnsFor(ctx, table) - - rows, err := tx.QueryContext(ctx, "SELECT * FROM "+quoteIdent(table)) //nolint:gosec // quoted identifier sourced from sqlite_master; SQLite cannot bind table names - if err != nil { - return err - } - defer func() { _ = rows.Close() }() - - names, err := rows.Columns() - if err != nil { - return err - } - if len(cols) == 0 { - cols = columnsFromNames(names) - } - - for rows.Next() { - vals := make([]any, len(names)) - ptrs := make([]any, len(names)) - for i := range vals { - ptrs[i] = &vals[i] - } - if err := rows.Scan(ptrs...); err != nil { - return err - } - - change := config.Change{ - Source: s.name, - Op: "snapshot", - Schema: "main", - Table: table, - Relation: table, - After: mapRow(cols, vals), - } - if !sub.sendSnapshot(ctx, change) { - return nil - } - } - - return rows.Err() -} - -func resolveColumns(ctx context.Context, db *sql.DB, table string) ([]columnInfo, error) { - rows, err := db.QueryContext(ctx, "PRAGMA table_info("+quoteIdent(table)+")") - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - var cols []columnInfo - for rows.Next() { - var cid, notnull, pk int - var name, declType string - var dflt any - if err := rows.Scan(&cid, &name, &declType, ¬null, &dflt, &pk); err != nil { - return nil, err - } - cols = append(cols, columnInfo{name: name, text: textAffinity(declType)}) - } - - return cols, rows.Err() -} - -func columnsFromNames(names []string) []columnInfo { - cols := make([]columnInfo, len(names)) - for i, n := range names { - cols[i] = columnInfo{name: n} - } - - return cols -} - -func quoteIdent(name string) string { - return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` -} diff --git a/service/cdc/sqlite/source_stub.go b/service/cdc/sqlite/source_stub.go index 8feb9cd2a..a49e9b428 100644 --- a/service/cdc/sqlite/source_stub.go +++ b/service/cdc/sqlite/source_stub.go @@ -4,6 +4,6 @@ package sqlite -func buildSource(_ sourceOptions) (sourceHandle, error) { +func buildSource(_ sourceOptions) (managedSource, error) { return nil, ErrPreupdateTagRequired } diff --git a/service/cdc/sqlite/source_tagged_test.go b/service/cdc/sqlite/source_tagged_test.go index c98672d36..24b8d2810 100644 --- a/service/cdc/sqlite/source_tagged_test.go +++ b/service/cdc/sqlite/source_tagged_test.go @@ -25,3 +25,9 @@ func TestBuildSourceRejectsBadInterval(t *testing.T) { _, err := buildSource(sourceOptions{name: "x", statusInterval: "nope"}) assert.Error(t, err) } + +func TestBuildSourceRetainsSnapshotPolicyForPerSubscriberHandoff(t *testing.T) { + h, err := buildSource(sourceOptions{name: "x", snapshot: true}) + require.NoError(t, err) + assert.True(t, h.(*Source).snapshot) +} diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go index 13721da50..9b222c715 100644 --- a/service/cdc/sqlite/subscribers.go +++ b/service/cdc/sqlite/subscribers.go @@ -3,7 +3,7 @@ package sqlite import ( - "context" + "errors" "strings" "sync" "sync/atomic" @@ -16,9 +16,11 @@ const ( maxStreamBuffer = 65536 ) +var errSubscriberOverflow = errors.New("sqlite cdc subscriber backlog overflow") + type subscribers struct { - m map[uint64]*subscription mu sync.RWMutex + m map[uint64]*subscription next uint64 } @@ -26,7 +28,7 @@ func newSubscribers() *subscribers { return &subscribers{m: make(map[uint64]*subscription)} } -func (s *subscribers) subscribe(sourceName string, opts config.StreamOptions, wantSnapshot bool) *subscription { +func (s *subscribers) subscribe(sourceName string, opts config.StreamOptions) *subscription { buffer := opts.Buffer if buffer <= 0 { buffer = defaultStreamBuffer @@ -37,30 +39,25 @@ func (s *subscribers) subscribe(sourceName string, opts config.StreamOptions, wa s.mu.Lock() s.next++ - sub := &subscription{ - parent: s, - id: s.next, - sourceName: sourceName, - in: make(chan config.Change, buffer), - out: make(chan config.Change, buffer), - done: make(chan struct{}), - termCh: make(chan struct{}), - tables: filterSet(opts.Tables), - ops: filterSet(opts.Ops), - wantSnapshot: wantSnapshot, - } - if wantSnapshot { - sub.snap = make(chan config.Change) - } + sub := newSubscription(sourceName, opts, buffer) + sub.parent = s + sub.id = s.next s.m[sub.id] = sub s.mu.Unlock() - - go sub.run() - return sub } -func (s *subscribers) publish(_ context.Context, change config.Change) { +func newSubscription(sourceName string, opts config.StreamOptions, buffer int) *subscription { + return &subscription{ + sourceName: sourceName, + changes: make(chan config.Change, buffer), + done: make(chan struct{}), + tables: filterSet(opts.Tables), + ops: filterSet(opts.Ops), + } +} + +func (s *subscribers) publish(change config.Change) { s.mu.RLock() matched := make([]*subscription, 0, len(s.m)) for _, sub := range s.m { @@ -69,7 +66,6 @@ func (s *subscribers) publish(_ context.Context, change config.Change) { } } s.mu.RUnlock() - for _, sub := range matched { sub.send(change) } @@ -82,6 +78,10 @@ func (s *subscribers) remove(id uint64) { } func (s *subscribers) closeAll() { + s.closeWithError(nil) +} + +func (s *subscribers) closeWithError(err error) { s.mu.Lock() subs := make([]*subscription, 0, len(s.m)) for id, sub := range s.m { @@ -89,114 +89,81 @@ func (s *subscribers) closeAll() { delete(s.m, id) } s.mu.Unlock() - for _, sub := range subs { - sub.Close() + sub.closeWithError(err) } } type subscription struct { - parent *subscribers - in chan config.Change - out chan config.Change - snap chan config.Change - done chan struct{} - termCh chan struct{} - tables map[string]struct{} - ops map[string]struct{} - term atomic.Pointer[config.Change] - sourceName string - id uint64 - closeOnce sync.Once - failOnce sync.Once - closed atomic.Bool - wantSnapshot bool -} - -func (s *subscription) Changes() <-chan config.Change { - return s.out -} - -func (s *subscription) Close() { - s.closeOnce.Do(func() { - s.closed.Store(true) - if s.parent != nil { - s.parent.remove(s.id) - } - close(s.done) - }) -} + parent *subscribers + changes chan config.Change + done chan struct{} + tables map[string]struct{} + ops map[string]struct{} + sourceName string + id uint64 -func (s *subscription) fail(reason string) { - s.failOnce.Do(func() { - c := config.Change{Source: s.sourceName, Op: "error", Error: reason} - s.term.Store(&c) - s.closed.Store(true) - if s.parent != nil { - s.parent.remove(s.id) - } - close(s.termCh) - }) + mu sync.Mutex + closed bool + err error + // closedFlag lets the fan-out path reject work without taking the lock in + // the common case. The lock is still held while sending/closing so a send + // cannot race close(changes). + closedFlag atomic.Bool } -func (s *subscription) run() { - defer close(s.out) +func (s *subscription) Changes() <-chan config.Change { return s.changes } + +func (s *subscription) Close() { s.closeWithError(nil) } + +func (s *subscription) Err() error { + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err +} - if s.wantSnapshot && !s.pump(s.snap, true) { - s.flushTerm() +func (s *subscription) send(change config.Change) { + if s.closedFlag.Load() { return } - if !s.pump(s.in, false) { - s.flushTerm() + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return } -} - -func (s *subscription) pump(src <-chan config.Change, snapshotPhase bool) bool { - for { - select { - case <-s.done: - return false - case <-s.termCh: - return false - case change, ok := <-src: - if !ok { - return snapshotPhase - } - select { - case <-s.done: - return false - case <-s.termCh: - return false - case s.out <- change: - } - } + select { + case s.changes <- change: + default: + s.closeLocked(errSubscriberOverflow) } } -func (s *subscription) flushTerm() { - if t := s.term.Load(); t != nil { - select { - case s.out <- *t: - case <-s.done: - } +func (s *subscription) closeWithError(err error) { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.closeLocked(err) + s.mu.Unlock() + if s.parent != nil { + s.parent.remove(s.id) } } -func (s *subscription) send(change config.Change) { - if s.closed.Load() { +func (s *subscription) closeLocked(err error) { + if s.closed { return } - select { - case s.in <- change: - default: - s.fail("sqlite cdc subscriber backlog overflow") - } + s.closed = true + s.closedFlag.Store(true) + s.err = err + close(s.done) + close(s.changes) } func (s *subscription) matches(change config.Change) bool { - if change.Op == "error" || change.Op == "snapshot" { - return true - } if len(s.ops) > 0 { if _, ok := s.ops[strings.ToLower(change.Op)]; !ok { return false @@ -209,28 +176,40 @@ func (s *subscription) matches(change config.Change) bool { if _, ok := s.tables[strings.ToLower(change.Table)]; ok { return true } - return false } - return true } +func (s *subscription) matchesSnapshot(change config.Change) bool { + if len(s.tables) == 0 { + return true + } + if _, ok := s.tables[strings.ToLower(change.Relation)]; ok { + return true + } + _, ok := s.tables[strings.ToLower(change.Table)] + return ok +} + +func (s *subscription) isClosed() bool { return s.closedFlag.Load() } + func filterSet(values []string) map[string]struct{} { if len(values) == 0 { return nil } - out := make(map[string]struct{}, len(values)) - for _, v := range values { - v = strings.ToLower(strings.TrimSpace(v)) - if v != "" { - out[v] = struct{}{} + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if value != "" { + out[value] = struct{}{} } } if len(out) == 0 { return nil } - return out } + +var _ config.Stream = (*subscription)(nil) +var _ config.ErrStream = (*subscription)(nil) diff --git a/service/cdc/sqlite/subscribers_snapshot.go b/service/cdc/sqlite/subscribers_snapshot.go deleted file mode 100644 index aff4e47cf..000000000 --- a/service/cdc/sqlite/subscribers_snapshot.go +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//go:build sqlite_preupdate_hook - -package sqlite - -import ( - "context" - "strings" - - config "github.com/wippyai/runtime/api/service/cdc" -) - -func (s *subscription) isClosed() bool { - return s.closed.Load() -} - -func (s *subscription) finishSnapshot() { - if s.snap != nil { - close(s.snap) - } -} - -func (s *subscription) sendSnapshot(ctx context.Context, change config.Change) bool { - select { - case s.snap <- change: - return true - case <-s.done: - return false - case <-s.termCh: - return false - case <-ctx.Done(): - return false - } -} - -func (s *subscription) tableAllowed(name string) bool { - if len(s.tables) == 0 { - return true - } - _, ok := s.tables[strings.ToLower(name)] - - return ok -} diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go index c081f43ce..be0476f54 100644 --- a/service/cdc/sqlite/subscribers_test.go +++ b/service/cdc/sqlite/subscribers_test.go @@ -3,7 +3,6 @@ package sqlite import ( - "context" "testing" "time" @@ -37,23 +36,23 @@ func TestSubscriptionMatches(t *testing.T) { assert.False(t, byTable.matches(config.Change{Op: "insert", Table: "orders"})) } -func TestSubscriptionMatchesBypassesFiltersForControlEvents(t *testing.T) { +func TestSubscriptionSnapshotMatchesOnlyTables(t *testing.T) { sub := &subscription{ ops: map[string]struct{}{"insert": {}}, tables: map[string]struct{}{"users": {}}, } - assert.True(t, sub.matches(config.Change{Op: "error", Table: "orders"}), "terminal error must reach every subscriber") - assert.True(t, sub.matches(config.Change{Op: "snapshot", Table: "orders"}), "snapshot rows must bypass op/table filters") + assert.False(t, sub.matchesSnapshot(config.Change{Op: "snapshot", Table: "orders"}), "table filter still applies to snapshot rows") + assert.True(t, sub.matchesSnapshot(config.Change{Op: "snapshot", Table: "users"})) assert.False(t, sub.matches(config.Change{Op: "delete", Table: "users"}), "op filter still applies to normal changes") assert.False(t, sub.matches(config.Change{Op: "insert", Table: "orders"}), "table filter still applies to normal changes") } func TestSubscribersPublishAndClose(t *testing.T) { subs := newSubscribers() - stream := subs.subscribe("s", config.StreamOptions{}, false) + stream := subs.subscribe("s", config.StreamOptions{}) - subs.publish(context.Background(), config.Change{Op: "insert", Table: "users", Source: "s"}) + subs.publish(config.Change{Op: "insert", Table: "users", Source: "s"}) select { case change := <-stream.Changes(): @@ -75,35 +74,35 @@ func TestSubscribersPublishAndClose(t *testing.T) { func TestSubscribeBufferClamp(t *testing.T) { subs := newSubscribers() - def := subs.subscribe("s", config.StreamOptions{Buffer: 0}, false) - assert.Equal(t, defaultStreamBuffer, cap(def.in)) + def := subs.subscribe("s", config.StreamOptions{Buffer: 0}) + assert.Equal(t, defaultStreamBuffer, cap(def.changes)) - neg := subs.subscribe("s", config.StreamOptions{Buffer: -5}, false) - assert.Equal(t, defaultStreamBuffer, cap(neg.in)) + neg := subs.subscribe("s", config.StreamOptions{Buffer: -5}) + assert.Equal(t, defaultStreamBuffer, cap(neg.changes)) - exact := subs.subscribe("s", config.StreamOptions{Buffer: 7}, false) - assert.Equal(t, 7, cap(exact.in)) + exact := subs.subscribe("s", config.StreamOptions{Buffer: 7}) + assert.Equal(t, 7, cap(exact.changes)) - huge := subs.subscribe("s", config.StreamOptions{Buffer: maxStreamBuffer + 100}, false) - assert.Equal(t, maxStreamBuffer, cap(huge.in)) + huge := subs.subscribe("s", config.StreamOptions{Buffer: maxStreamBuffer + 100}) + assert.Equal(t, maxStreamBuffer, cap(huge.changes)) } func TestSubscribeAssignsUniqueIncreasingIDs(t *testing.T) { subs := newSubscribers() - a := subs.subscribe("s", config.StreamOptions{}, false) - b := subs.subscribe("s", config.StreamOptions{}, false) + a := subs.subscribe("s", config.StreamOptions{}) + b := subs.subscribe("s", config.StreamOptions{}) assert.Equal(t, uint64(1), a.id) assert.Equal(t, uint64(2), b.id) } func TestPublishNeverBlocksAndClosesLaggard(t *testing.T) { subs := newSubscribers() - stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}, false) + stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) done := make(chan struct{}) go func() { for i := 0; i < 1000; i++ { - subs.publish(context.Background(), config.Change{Op: "insert", Table: "t"}) + subs.publish(config.Change{Op: "insert", Table: "t"}) } close(done) }() @@ -128,11 +127,11 @@ func TestPublishNeverBlocksAndClosesLaggard(t *testing.T) { func TestSubscribersFilterByOp(t *testing.T) { subs := newSubscribers() - stream := subs.subscribe("s", config.StreamOptions{Ops: []string{"delete"}}, false) + stream := subs.subscribe("s", config.StreamOptions{Ops: []string{"delete"}}) defer stream.Close() - subs.publish(context.Background(), config.Change{Op: "insert", Table: "users"}) - subs.publish(context.Background(), config.Change{Op: "delete", Table: "users"}) + subs.publish(config.Change{Op: "insert", Table: "users"}) + subs.publish(config.Change{Op: "delete", Table: "users"}) select { case change := <-stream.Changes(): diff --git a/service/sql/conn.go b/service/sql/conn.go index 372862b3d..2237186fe 100644 --- a/service/sql/conn.go +++ b/service/sql/conn.go @@ -12,19 +12,25 @@ import ( "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/resource" + sqlapi "github.com/wippyai/runtime/api/service/sql" ) // ConnPool represents a database connection pool that acts both as a service // and a resource provider type ConnPool struct { - db *sql.DB - current *dbGeneration - status chan any - config atomic.Pointer[any] - kind registry.Kind - mu sync.RWMutex - wg sync.WaitGroup - closed atomic.Bool + db *sql.DB + current *dbGeneration + status chan any + config atomic.Pointer[any] + kind registry.Kind + driver Driver + mu sync.RWMutex + wg sync.WaitGroup + closed atomic.Bool + stopMu sync.Mutex + stopDone chan struct{} + stopErr error + stopStarted bool } type dbGeneration struct { @@ -35,12 +41,18 @@ type dbGeneration struct { once sync.Once refs atomic.Int32 closing atomic.Bool + observer sqlapi.CommittedMutationSource } -func newDBGeneration(db *sql.DB) *dbGeneration { +func newDBGeneration(db *sql.DB, observers ...sqlapi.CommittedMutationSource) *dbGeneration { + var observer sqlapi.CommittedMutationSource + if len(observers) > 0 { + observer = observers[0] + } return &dbGeneration{ - db: db, - closed: make(chan struct{}), + db: db, + closed: make(chan struct{}), + observer: observer, } } @@ -77,8 +89,13 @@ func (g *dbGeneration) closeWhenIdle() { func (g *dbGeneration) closeNow() { g.once.Do(func() { + if g.observer != nil { + _ = g.observer.Close() + } g.closeMu.Lock() - g.closeErr = g.db.Close() + if g.db != nil { + g.closeErr = g.db.Close() + } g.closeMu.Unlock() close(g.closed) }) @@ -142,36 +159,51 @@ func (p *ConnPool) Start(ctx context.Context) (<-chan any, error) { // Stop implements supervisor.Service func (p *ConnPool) Stop(ctx context.Context) error { - // Try to set closed state - if already closed, return immediately - if !p.closed.CompareAndSwap(false, true) { - return nil - } - - // Wait for all resources to be released - done := make(chan struct{}) - go func() { - p.wg.Wait() - close(done) - }() - + if ctx == nil { + ctx = context.Background() + } + p.stopMu.Lock() + // Serialize the closed transition with Acquire's WaitGroup admission. A + // positive Add must not race with cleanupStop's Wait when the pool has no + // outstanding resources; holding this mutex makes the handoff explicit. + p.closed.Store(true) + if !p.stopStarted { + p.stopStarted = true + p.stopDone = make(chan struct{}) + go p.cleanupStop(p.stopDone) + } + done := p.stopDone + p.stopMu.Unlock() select { case <-ctx.Done(): return ctx.Err() case <-done: - p.mu.Lock() - if p.current == nil && p.db != nil { - p.current = newDBGeneration(p.db) - } - gen := p.current - p.current = nil - p.db = nil - p.mu.Unlock() - if gen == nil { - return nil - } + p.stopMu.Lock() + err := p.stopErr + p.stopMu.Unlock() + return err + } +} + +func (p *ConnPool) cleanupStop(done chan struct{}) { + p.wg.Wait() + p.mu.Lock() + if p.current == nil && p.db != nil { + p.current = newDBGeneration(p.db) + } + gen := p.current + p.current = nil + p.db = nil + p.mu.Unlock() + var err error + if gen != nil { gen.closeWhenIdle() - return gen.waitClosed(ctx) + err = gen.waitClosed(context.Background()) } + p.stopMu.Lock() + p.stopErr = err + p.stopMu.Unlock() + close(done) } // UpdateConfig updates the pool configuration. It delegates engine-specific @@ -186,20 +218,19 @@ func (p *ConnPool) UpdateConfig(cfg any) error { return NewUnsupportedConfigTypeError(p.kind) } - eng, ok := engineFor(p.kind) - if !ok { + if p.driver == nil { return NewUnsupportedConfigTypeError(p.kind) } - return p.updateConfig(context.Background(), eng, ec) + return p.updateConfig(context.Background(), p.driver, ec) } -func (p *ConnPool) updateConfig(ctx context.Context, eng Engine, ec config.EngineConfig) error { +func (p *ConnPool) updateConfig(ctx context.Context, driver Driver, ec config.EngineConfig) error { if p.closed.Load() { return ErrPoolClosed } - if err := eng.ValidateConfigType(ec); err != nil { + if err := driver.ValidateConfigType(ec); err != nil { return err } @@ -207,27 +238,19 @@ func (p *ConnPool) updateConfig(ctx context.Context, eng Engine, ec config.Engin return NewInvalidConfigError(err) } - if p.kind == config.SQLite { - gen := p.currentGeneration() - if gen == nil { - return ErrPoolClosed - } - eng.Tune(gen.db, ec) - var stored any = ec - p.config.Store(&stored) - return nil - } - - newDB, err := openEngineDB(ctx, eng, ec) + opened, err := openDriverDB(ctx, driver, ec) if err != nil { return err } - newGen := newDBGeneration(newDB) + newGen := newDBGeneration(opened.DB, opened.Observer) p.mu.Lock() if p.closed.Load() { p.mu.Unlock() - _ = newDB.Close() + if opened.Observer != nil { + _ = opened.Observer.Close() + } + _ = opened.DB.Close() return ErrPoolClosed } oldGen := p.current @@ -235,7 +258,7 @@ func (p *ConnPool) updateConfig(ctx context.Context, eng Engine, ec config.Engin oldGen = newDBGeneration(p.db) } p.current = newGen - p.db = newDB + p.db = opened.DB p.mu.Unlock() if oldGen != nil { @@ -259,8 +282,15 @@ func (p *ConnPool) Acquire( return nil, NewUnsupportedAccessModeError(string(mode)) } - // Track resource usage before checking closed state to avoid race with Stop() + // Admission is serialized with Stop. This prevents a positive WaitGroup Add + // from racing with cleanupStop's Wait after the counter reaches zero. + p.stopMu.Lock() + if p.closed.Load() { + p.stopMu.Unlock() + return nil, ErrPoolClosed + } p.wg.Add(1) + p.stopMu.Unlock() if p.closed.Load() { p.wg.Done() @@ -293,8 +323,9 @@ type DBConn struct { // DBResource contains both the database connection and its type type DBResource struct { - DB *sql.DB // The database connection - Type registry.Kind // The database type (postgres, mysql, sqlite, etc.) + DB *sql.DB // The database connection + Type registry.Kind // The database type (postgres, mysql, sqlite, etc.) + Observer sqlapi.CommittedMutationSource } // newDBConn creates a new database resource @@ -314,8 +345,9 @@ func (r *DBConn) Get() (any, error) { // Return both the DB and its type return DBResource{ - DB: r.gen.db, - Type: r.dbType, + DB: r.gen.db, + Type: r.dbType, + Observer: r.gen.observer, }, nil } diff --git a/service/sql/conn_test.go b/service/sql/conn_test.go index d41059e05..e0e98f5d8 100644 --- a/service/sql/conn_test.go +++ b/service/sql/conn_test.go @@ -28,6 +28,7 @@ func newTestPool(t *testing.T) *ConnPool { pool := &ConnPool{ kind: apiconfig.SQLite, db: db, + driver: func() Driver { d, _ := testDriverFor(apiconfig.SQLite); return d }(), status: make(chan any, 1), } @@ -179,6 +180,23 @@ func TestConnPool_StopTimeout(t *testing.T) { assert.Equal(t, context.DeadlineExceeded, err) } +func TestConnPool_StopTimeoutStillCleansUp(t *testing.T) { + pool := newTestPool(t) + ctx := context.Background() + _, err := pool.Start(ctx) + require.NoError(t, err) + res, err := pool.Acquire(ctx, testID, resource.ModeNormal) + require.NoError(t, err) + stopCtx, cancel := context.WithTimeout(ctx, 25*time.Millisecond) + err = pool.Stop(stopCtx) + cancel() + require.ErrorIs(t, err, context.DeadlineExceeded) + res.Release() + require.NoError(t, pool.Stop(ctx)) + _, err = pool.Start(ctx) + assert.ErrorIs(t, err, ErrPoolClosed) +} + func TestDBConn_DoubleRelease(t *testing.T) { pool := newTestPool(t) ctx := context.Background() diff --git a/service/sql/driver.go b/service/sql/driver.go index 7cafa44f9..2a7214e65 100644 --- a/service/sql/driver.go +++ b/service/sql/driver.go @@ -1,38 +1,3 @@ // SPDX-License-Identifier: MPL-2.0 package sql - -import ( - "sync" - - "github.com/wippyai/runtime/api/registry" -) - -// driverOverrides lets an out-of-tree extension swap the database/sql driver an -// engine opens, keyed by registry kind. It is the only seam through which engine -// connection behavior (for example, a SQLite build that installs preupdate hooks) -// is altered without modifying the core package. -var ( - driverMu sync.RWMutex - driverOverrides = make(map[registry.Kind]string) -) - -// RegisterDriver overrides the database/sql driver name used for the given kind. -// Extensions call this from an init function, before any pool is created. -func RegisterDriver(kind registry.Kind, name string) { - driverMu.Lock() - driverOverrides[kind] = name - driverMu.Unlock() -} - -// driverName returns the registered override for kind, falling back to def. -func driverName(kind registry.Kind, def string) string { - driverMu.RLock() - name, ok := driverOverrides[kind] - driverMu.RUnlock() - if ok { - return name - } - - return def -} diff --git a/service/sql/driver_test.go b/service/sql/driver_test.go index 9ae3cab82..deb8c27b4 100644 --- a/service/sql/driver_test.go +++ b/service/sql/driver_test.go @@ -53,41 +53,29 @@ func (fixedConfigEngine) ValidateConfigType(config.EngineConfig) error { return nil } -func TestDriverNameFallbackAndOverride(t *testing.T) { +func TestDriverSelectionIsExplicit(t *testing.T) { const kind = registry.Kind("db.sql.drivernametest") - - assert.Equal(t, "default-drv", driverName(kind, "default-drv")) - - RegisterDriver(kind, "override-drv") - assert.Equal(t, "override-drv", driverName(kind, "default-drv")) -} - -func TestCreatePoolAppliesDriverOverride(t *testing.T) { - const kind = registry.Kind("db.sql.overridetest") - RegisterEngine(fixedConfigEngine{kind: kind, driver: "sqlite3"}) - - factory := &DefaultPoolFactory{} + driver := fixedConfigEngine{kind: kind, driver: "sqlite3"} deps := EngineDeps{Log: zap.NewNop()} entry := registry.Entry{ID: registry.NewID("test", "ov"), Kind: kind, Data: payload.New("x")} + _, _, err := NewDefaultPoolFactory().CreatePool(context.Background(), deps, entry) + require.Error(t, err, "an unconfigured factory must not discover a driver globally") + + factory := NewDefaultPoolFactory(driver) pool, _, err := factory.CreatePool(context.Background(), deps, entry) require.NoError(t, err) require.NotNil(t, pool) require.NoError(t, pool.Stop(context.Background())) - - RegisterDriver(kind, "sentinel-missing-driver") - overridden, _, err := factory.CreatePool(context.Background(), deps, entry) - require.Error(t, err) - assert.Nil(t, overridden) } func TestCreatePoolClosesOnPrepareError(t *testing.T) { const kind = registry.Kind("db.sql.preparefailtest") prepErr := errors.New("prepare boom") - RegisterEngine(fixedConfigEngine{kind: kind, driver: "sqlite3", prepareErr: prepErr}) + driver := fixedConfigEngine{kind: kind, driver: "sqlite3", prepareErr: prepErr} entry := registry.Entry{ID: registry.NewID("test", "pf"), Kind: kind, Data: payload.New("x")} - pool, _, err := (&DefaultPoolFactory{}).CreatePool(context.Background(), EngineDeps{Log: zap.NewNop()}, entry) + pool, _, err := NewDefaultPoolFactory(driver).CreatePool(context.Background(), EngineDeps{Log: zap.NewNop()}, entry) require.Error(t, err) assert.Nil(t, pool) diff --git a/service/sql/engine.go b/service/sql/engine.go index 93aac432c..21c806ef4 100644 --- a/service/sql/engine.go +++ b/service/sql/engine.go @@ -5,11 +5,13 @@ package sql import ( "context" "database/sql" + "fmt" envapi "github.com/wippyai/runtime/api/env" "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/registry" config "github.com/wippyai/runtime/api/service/sql" + sqlapi "github.com/wippyai/runtime/api/service/sql" "go.uber.org/zap" ) @@ -21,10 +23,11 @@ type EngineDeps struct { Log *zap.Logger } -// Engine is a self-contained SQL dialect. Each engine knows how to decode its -// configuration, resolve environment overrides, open and tune a pool, and run any -// post-open preparation. Engines register themselves with RegisterEngine, so adding -// a database never touches the factory or manager dispatch surface. +// Engine is a self-contained SQL dialect. Engines are supplied to a Manager (or +// Factory) explicitly, so the SQL service has no process-global engine registry. +// The original contract intentionally remains small: custom engines may keep +// using BuildDSN plus database/sql, while engines that own a physical connector +// can additionally implement DBOpener. type Engine interface { Kind() registry.Kind DriverName() string @@ -36,28 +39,34 @@ type Engine interface { ValidateConfigType(cfg config.EngineConfig) error } -var engines = make(map[registry.Kind]Engine) +// Driver is the explicit-injection name for an Engine. It is an alias so +// existing extensions implementing the original Engine contract remain valid. +type Driver = Engine -// RegisterEngine adds an engine to the registry under its kind. Intended to be -// called from engine package init functions. -func RegisterEngine(e Engine) { - engines[e.Kind()] = e +// DBOpener is the optional physical-handle seam. A driver that implements it +// owns the database connector and any capabilities attached to that physical +// handle (for example SQLite mutation observation). Engines that do not need +// that ownership use the Engine.BuildDSN fallback in openDriverDB. +type DBOpener interface { + Open(ctx context.Context, cfg config.EngineConfig) (OpenedDB, error) } -// engineFor looks up the engine registered for a kind. -func engineFor(kind registry.Kind) (Engine, bool) { - e, ok := engines[kind] - return e, ok +// OpenedDB is the physical database handle created by a Driver. Observer is an +// optional engine capability and is deliberately kept beside the handle so it +// cannot be accidentally shared between unrelated pool generations. +type OpenedDB struct { + DB *sql.DB + Observer sqlapi.CommittedMutationSource } // createPool runs the generic create lifecycle for a known engine. -func createPool(ctx context.Context, deps EngineDeps, eng Engine, entry registry.Entry) (*ConnPool, config.EngineConfig, error) { - cfg, err := eng.DecodeConfig(ctx, deps.Transcoder, entry) +func createPool(ctx context.Context, deps EngineDeps, driver Driver, entry registry.Entry) (*ConnPool, config.EngineConfig, error) { + cfg, err := driver.DecodeConfig(ctx, deps.Transcoder, entry) if err != nil { return nil, nil, NewInvalidConfigError(err) } - if err := eng.ResolveEnv(ctx, deps, cfg); err != nil { + if err := driver.ResolveEnv(ctx, deps, cfg); err != nil { return nil, nil, err } @@ -65,15 +74,16 @@ func createPool(ctx context.Context, deps EngineDeps, eng Engine, entry registry return nil, nil, NewInvalidConfigError(err) } - db, err := openEngineDB(ctx, eng, cfg) + opened, err := openDriverDB(ctx, driver, cfg) if err != nil { return nil, nil, err } pool := &ConnPool{ - kind: eng.Kind(), - db: db, - current: newDBGeneration(db), + kind: driver.Kind(), + driver: driver, + db: opened.DB, + current: newDBGeneration(opened.DB, opened.Observer), status: make(chan any, 1), } @@ -84,39 +94,61 @@ func createPool(ctx context.Context, deps EngineDeps, eng Engine, entry registry } // updatePool runs the generic update lifecycle for a known engine. -func updatePool(ctx context.Context, deps EngineDeps, eng Engine, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) { - cfg, err := eng.DecodeConfig(ctx, deps.Transcoder, entry) +func updatePool(ctx context.Context, deps EngineDeps, driver Driver, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) { + cfg, err := driver.DecodeConfig(ctx, deps.Transcoder, entry) if err != nil { return nil, NewInvalidConfigError(err) } - if err := eng.ResolveEnv(ctx, deps, cfg); err != nil { + if err := driver.ResolveEnv(ctx, deps, cfg); err != nil { return nil, err } - if err := pool.updateConfig(ctx, eng, cfg); err != nil { + if err := pool.updateConfig(ctx, driver, cfg); err != nil { return nil, NewPoolUpdateError(err) } return cfg, nil } -func openEngineDB(ctx context.Context, eng Engine, cfg config.EngineConfig) (*sql.DB, error) { - dsn, err := eng.BuildDSN(cfg) - if err != nil { - return nil, NewInvalidDSNError(err) +func openDriverDB(ctx context.Context, driver Driver, cfg config.EngineConfig) (OpenedDB, error) { + var ( + opened OpenedDB + err error + ) + if opener, ok := driver.(DBOpener); ok { + opened, err = opener.Open(ctx, cfg) + } else { + var dsn string + dsn, err = driver.BuildDSN(cfg) + if err != nil { + return OpenedDB{}, NewInvalidDSNError(err) + } + opened.DB, err = sql.Open(driver.DriverName(), dsn) + if err != nil { + return OpenedDB{}, NewConnectionPoolCreationError(err) + } } - - db, err := sql.Open(driverName(eng.Kind(), eng.DriverName()), dsn) if err != nil { - return nil, NewConnectionPoolCreationError(err) + return OpenedDB{}, err + } + if opened.DB == nil { + if opened.Observer != nil { + _ = opened.Observer.Close() + } + return OpenedDB{}, NewConnectionPoolCreationError( + fmt.Errorf("driver %q returned a nil database", driver.Kind()), + ) } - if err := eng.Prepare(ctx, db, cfg); err != nil { - _ = db.Close() - return nil, err + if err := driver.Prepare(ctx, opened.DB, cfg); err != nil { + _ = opened.DB.Close() + if opened.Observer != nil { + _ = opened.Observer.Close() + } + return OpenedDB{}, err } - eng.Tune(db, cfg) - return db, nil + driver.Tune(opened.DB, cfg) + return opened, nil } diff --git a/service/sql/engine/all/all.go b/service/sql/engine/all/all.go index d63543976..45ce33bad 100644 --- a/service/sql/engine/all/all.go +++ b/service/sql/engine/all/all.go @@ -1,12 +1,20 @@ // SPDX-License-Identifier: MPL-2.0 -// Package all registers every built-in SQL engine via blank import. A composition -// root (for example the storage boot component) imports this package so the standard -// dialects are available; out-of-tree engines register themselves the same way. +// Package all provides the built-in SQL drivers for composition roots that want +// the standard Wippy database set. It does not register anything globally. package all import ( - // Blank imports register the built-in engines with service/sql via init. - _ "github.com/wippyai/runtime/service/sql/engine/sqlite" - _ "github.com/wippyai/runtime/service/sql/engine/standard" + sqlservice "github.com/wippyai/runtime/service/sql" + "github.com/wippyai/runtime/service/sql/engine/sqlite" + "github.com/wippyai/runtime/service/sql/engine/standard" ) + +// Drivers returns the built-in SQL drivers in a deterministic order. +func Drivers() []sqlservice.Driver { + return []sqlservice.Driver{ + standard.NewPostgresDriver(), + standard.NewMySQLDriver(), + sqlite.NewDriver(), + } +} diff --git a/service/sql/engine/sqlite/observer.go b/service/sql/engine/sqlite/observer.go new file mode 100644 index 000000000..d6cd53300 --- /dev/null +++ b/service/sql/engine/sqlite/observer.go @@ -0,0 +1,1918 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + + "github.com/mattn/go-sqlite3" + + sqlapi "github.com/wippyai/runtime/api/service/sql" +) + +var ( + errObserverClosed = errors.New("sqlite mutation observer is closed") + errObserverOverflow = errors.New("sqlite mutation observer backlog overflow") + errObserverAmbiguous = errors.New("sqlite mutation observer cannot determine statement outcome") +) + +const ( + defaultSnapshotBatchSize = 512 + maxSnapshotBatchSize = 4096 + mutationStructuralBytes = 128 + valueStructuralBytes = 24 +) + +// sqliteConnector keeps the SQLite driver and its connection state owned by +// one SQL pool. It intentionally uses sql.OpenDB instead of sql.Register, so +// no process-global driver name or file-path registry is involved. +type sqliteConnector struct { + dsn string + driver *sqlite3.SQLiteDriver + backend *sqliteBackend +} + +func (c *sqliteConnector) Connect(context.Context) (driver.Conn, error) { + raw, err := c.driver.Open(c.dsn) + if err != nil { + return nil, err + } + sqliteConn, ok := raw.(*sqlite3.SQLiteConn) + if !ok { + _ = raw.Close() + return nil, fmt.Errorf("sqlite driver returned %T", raw) + } + + conn := &observedConn{ + raw: raw, + sqlite: sqliteConn, + backend: c.backend, + state: &sqliteConnectionState{ + backend: c.backend, sqlite: sqliteConn, + maxChanges: c.backend.maxChanges, maxBytes: c.backend.maxBytes, + }, + } + // Install hooks for every physical connection when it is created. This + // avoids trying to mutate a connection that may be in use when a stream is + // subscribed; the backend decides whether candidates are retained. + conn.bindIfActive() + return conn, nil +} + +func (c *sqliteConnector) Driver() driver.Driver { return c.driver } + +func openSQLite(_ context.Context, dsn string, limits ...int) (*sql.DB, sqlapi.CommittedMutationSource, error) { + maxChanges, maxBytes := observerLimits(limits) + backend := newSQLiteBackend(maxChanges, maxBytes) + connector := &sqliteConnector{ + dsn: dsn, + driver: &sqlite3.SQLiteDriver{}, + backend: backend, + } + db := sql.OpenDB(connector) + backend.db = db + return db, backend, nil +} + +type sqliteBackend struct { + db *sql.DB + streams map[*mutationStream]struct{} + mu sync.Mutex + fence chan struct{} + maxChanges int + maxBytes int + closed bool + sequence atomic.Uint64 +} + +func newSQLiteBackend(maxChanges, maxBytes int) *sqliteBackend { + fence := make(chan struct{}, 1) + fence <- struct{}{} + return &sqliteBackend{streams: make(map[*mutationStream]struct{}), fence: fence, maxChanges: maxChanges, maxBytes: maxBytes} +} + +func observerLimits(limits []int) (int, int) { + maxChanges, maxBytes := sqlapi.DefaultMaxMutationChanges, sqlapi.DefaultMaxMutationBytes + if len(limits) > 0 && limits[0] > 0 { + maxChanges = limits[0] + } + if len(limits) > 1 && limits[1] > 0 { + maxBytes = limits[1] + } + return maxChanges, maxBytes +} + +func (b *sqliteBackend) acquireFence(ctx context.Context) error { + select { + case <-b.fence: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (b *sqliteBackend) acquireCommitFence() { + <-b.fence +} + +func (b *sqliteBackend) releaseFence() { + b.fence <- struct{}{} +} + +func (b *sqliteBackend) hasObservers() bool { + b.mu.Lock() + active := !b.closed + if active { + active = false + for stream := range b.streams { + stream.mu.Lock() + closed := stream.closed + stream.mu.Unlock() + if !closed { + active = true + break + } + } + } + b.mu.Unlock() + return active +} + +func (b *sqliteBackend) Subscribe(ctx context.Context, opts sqlapi.MutationOptions) (sqlapi.MutationStream, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if opts.MaxChanges <= 0 { + opts.MaxChanges = b.maxChanges + } + if opts.MaxBytes <= 0 { + opts.MaxBytes = b.maxBytes + } + if err := b.validateTables(ctx, opts.Tables); err != nil { + return nil, err + } + stream := newMutationStream(b, opts) + b.mu.Lock() + if b.closed { + b.mu.Unlock() + stream.closeWithError(errObserverClosed) + return nil, errObserverClosed + } + b.streams[stream] = struct{}{} + b.mu.Unlock() + return stream, nil +} + +func (b *sqliteBackend) validateTables(ctx context.Context, requested []string) error { + b.mu.Lock() + db := b.db + closed := b.closed + b.mu.Unlock() + if closed { + return errObserverClosed + } + if db == nil { + return errors.New("sqlite observer has no database") + } + conn, err := db.Conn(ctx) + if err != nil { + return fmt.Errorf("acquire sqlite observer connection: %w", err) + } + defer conn.Close() + return conn.Raw(func(raw any) error { + observed, ok := raw.(*observedConn) + if !ok { + return fmt.Errorf("sqlite observer received %T", raw) + } + state := &sqliteConnectionState{backend: b, sqlite: observed.sqlite} + tables, err := tablesForValidation(observed.sqlite, requested) + if err != nil { + return err + } + for _, table := range tables { + if err := state.validateTable(table.schema, table.name); err != nil { + return err + } + } + return nil + }) +} + +func tablesForValidation(conn *sqlite3.SQLiteConn, requested []string) ([]snapshotTable, error) { + if len(requested) == 0 { + rows, err := conn.Query(`SELECT name FROM main.sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`, nil) + if err != nil { + return nil, err + } + defer rows.Close() + var tables []snapshotTable + values := make([]driver.Value, len(rows.Columns())) + for { + err := rows.Next(values) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + table, ok := values[0].(string) + if !ok { + return nil, fmt.Errorf("sqlite table name has type %T", values[0]) + } + tables = append(tables, snapshotTable{schema: "main", name: table}) + } + return tables, nil + } + tables := make([]snapshotTable, 0, len(requested)) + for _, name := range requested { + parts := strings.SplitN(name, ".", 2) + if len(parts) == 1 { + tables = append(tables, snapshotTable{schema: "main", name: parts[0]}) + } else { + tables = append(tables, snapshotTable{schema: parts[0], name: parts[1]}) + } + } + return tables, nil +} + +func (b *sqliteBackend) Snapshot(ctx context.Context, opts sqlapi.SnapshotOptions) (sqlapi.SnapshotStream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + b.mu.Lock() + closed := b.closed + db := b.db + b.mu.Unlock() + if closed { + return nil, errObserverClosed + } + if db == nil { + return nil, errors.New("sqlite snapshot has no database") + } + // Reserve the physical connection before taking the fence. SQL pools may + // have a single connection; taking the fence first would let a writer hold + // that connection while waiting for the snapshot and deadlock both paths. + conn, err := db.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("acquire sqlite snapshot connection: %w", err) + } + if err := b.acquireFence(ctx); err != nil { + _ = conn.Close() + return nil, err + } + release := true + defer func() { + if release { + b.releaseFence() + } + }() + tx, err := conn.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("begin sqlite snapshot: %w", err) + } + // database/sql may defer BEGIN until the first operation. Force a read + // while the fence is held so the transaction's SQLite read view is fixed + // before later commits can be buffered as live changes. + var schemaVersion int64 + if err := tx.QueryRowContext(ctx, "PRAGMA schema_version").Scan(&schemaVersion); err != nil { + _ = tx.Rollback() + _ = conn.Close() + return nil, fmt.Errorf("establish sqlite snapshot view: %w", err) + } + if err := validateSnapshotTablesTx(ctx, tx, opts.Tables); err != nil { + _ = tx.Rollback() + _ = conn.Close() + return nil, err + } + watermark := strconv.FormatUint(b.sequence.Load(), 10) + scanCtx, cancel := context.WithCancel(ctx) + if opts.MaxChanges <= 0 { + opts.MaxChanges = b.maxChanges + } + if opts.MaxBytes <= 0 { + opts.MaxBytes = b.maxBytes + } + stream := newSnapshotStream(b, opts, watermark, cancel) + b.mu.Lock() + if b.closed { + b.mu.Unlock() + cancel() + _ = tx.Rollback() + _ = conn.Close() + return nil, errObserverClosed + } + b.streams[stream] = struct{}{} + b.mu.Unlock() + // The fence remains held until the stream is registered and its read view + // has been established. New commits therefore receive a sequence greater + // than watermark and are buffered by this stream. + release = false + b.releaseFence() + go b.scanSnapshot(scanCtx, conn, tx, stream, opts) + return stream, nil +} + +func (b *sqliteBackend) scanSnapshot(ctx context.Context, conn *sql.Conn, tx *sql.Tx, stream *mutationStream, opts sqlapi.SnapshotOptions) { + defer conn.Close() + defer func() { + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + stream.finishSnapshot(err) + } + }() + tables, err := snapshotTables(ctx, tx, opts.Tables) + if err == nil { + batchSize := normalizeSnapshotBatchSize(opts.BatchSize) + for _, table := range tables { + if err = scanSnapshotTable(ctx, tx, stream, table.schema, table.name, batchSize); err != nil { + break + } + } + } + if err != nil { + _ = tx.Rollback() + b.remove(stream, err) + return + } + if err = tx.Commit(); err != nil { + b.remove(stream, fmt.Errorf("commit sqlite snapshot: %w", err)) + return + } + stream.finishSnapshot(nil) +} + +func normalizeSnapshotBatchSize(value int) int { + if value <= 0 { + return defaultSnapshotBatchSize + } + if value > maxSnapshotBatchSize { + return maxSnapshotBatchSize + } + return value +} + +type snapshotTable struct { + schema string + name string +} + +func snapshotTables(ctx context.Context, tx *sql.Tx, requested []string) ([]snapshotTable, error) { + if len(requested) == 0 { + rows, err := tx.QueryContext(ctx, `SELECT name FROM main.sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + var tables []snapshotTable + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + tables = append(tables, snapshotTable{schema: "main", name: name}) + } + return tables, rows.Err() + } + tables := make([]snapshotTable, 0, len(requested)) + for _, name := range requested { + parts := strings.SplitN(name, ".", 2) + if len(parts) == 1 { + tables = append(tables, snapshotTable{schema: "main", name: parts[0]}) + } else { + tables = append(tables, snapshotTable{schema: parts[0], name: parts[1]}) + } + } + return tables, nil +} + +func validateSnapshotTablesTx(ctx context.Context, tx *sql.Tx, requested []string) error { + tables, err := snapshotTables(ctx, tx, requested) + if err != nil { + return err + } + for _, table := range tables { + if strings.EqualFold(table.schema, "temp") { + return errors.New("sqlite snapshot does not support TEMP tables") + } + var definition sql.NullString + query := fmt.Sprintf("SELECT sql FROM %s.sqlite_master WHERE type = 'table' AND name = ?", quoteIdentifier(table.schema)) + if err := tx.QueryRowContext(ctx, query, table.name).Scan(&definition); err != nil { + return fmt.Errorf("inspect sqlite snapshot table %s.%s: %w", table.schema, table.name, err) + } + upper := strings.ToUpper(definition.String) + if strings.Contains(upper, "WITHOUT ROWID") { + return fmt.Errorf("sqlite snapshot does not support WITHOUT ROWID table %s.%s", table.schema, table.name) + } + if strings.HasPrefix(strings.TrimSpace(upper), "CREATE VIRTUAL TABLE") { + return fmt.Errorf("sqlite snapshot does not support virtual table %s.%s", table.schema, table.name) + } + } + return nil +} + +func scanSnapshotTable(ctx context.Context, tx *sql.Tx, stream *mutationStream, schema, table string, batchSize int) error { + rows, err := tx.QueryContext(ctx, fmt.Sprintf("SELECT rowid, * FROM %s.%s", quoteIdentifier(schema), quoteIdentifier(table))) + if err != nil { + return fmt.Errorf("scan sqlite snapshot %s.%s: %w", schema, table, err) + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + return err + } + if len(columns) == 0 { + return nil + } + columns = append([]string(nil), columns[1:]...) + changes := make([]sqlapi.Mutation, 0, batchSize) + for rows.Next() { + values := make([]any, len(columns)+1) + dest := make([]any, len(values)) + for i := range values { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return err + } + rowID, ok := values[0].(int64) + if !ok || rowID == 0 { + return fmt.Errorf("sqlite snapshot %s.%s returned invalid rowid %v", schema, table, values[0]) + } + after := append([]any(nil), values[1:]...) + changes = append(changes, sqlapi.Mutation{ + Schema: schema, Table: table, Columns: columns, + RowID: rowID, After: after, Op: "snapshot", + }) + if len(changes) >= batchSize { + if err := stream.pushSnapshot(sqlapi.MutationBatch{Transaction: stream.watermark, Snapshot: true, Changes: append([]sqlapi.Mutation(nil), changes...)}); err != nil { + return err + } + changes = changes[:0] + } + } + if err := rows.Err(); err != nil { + return err + } + if len(changes) > 0 { + return stream.pushSnapshot(sqlapi.MutationBatch{Transaction: stream.watermark, Snapshot: true, Changes: append([]sqlapi.Mutation(nil), changes...)}) + } + return nil +} + +func (b *sqliteBackend) remove(stream *mutationStream, err error) { + b.mu.Lock() + if _, ok := b.streams[stream]; ok { + delete(b.streams, stream) + } + b.mu.Unlock() + + stream.closeWithError(err) +} + +func (b *sqliteBackend) publish(changes []sqlapi.Mutation) { + if len(changes) == 0 { + return + } + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return + } + streams := make([]*mutationStream, 0, len(b.streams)) + for stream := range b.streams { + streams = append(streams, stream) + } + sequence := b.sequence.Add(1) + b.mu.Unlock() + + batch := sqlapi.MutationBatch{ + Transaction: strconv.FormatUint(sequence, 10), + Changes: changes, + } + for _, stream := range streams { + stream.push(batch) + } +} + +func (b *sqliteBackend) fail(err error) { + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return + } + b.closed = true + streams := make([]*mutationStream, 0, len(b.streams)) + for stream := range b.streams { + streams = append(streams, stream) + } + b.streams = make(map[*mutationStream]struct{}) + b.mu.Unlock() + + for _, stream := range streams { + stream.closeWithError(err) + } +} + +func (b *sqliteBackend) Close() error { + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return nil + } + b.closed = true + streams := make([]*mutationStream, 0, len(b.streams)) + for stream := range b.streams { + streams = append(streams, stream) + } + b.streams = make(map[*mutationStream]struct{}) + b.mu.Unlock() + + for _, stream := range streams { + stream.closeWithError(errObserverClosed) + } + return nil +} + +// sqliteConnectionState is attached to one physical SQLite connection. The +// hooks only collect a candidate transaction. Publication happens from the +// driver wrappers after Exec/Commit/Rows completion, when statement rollback +// and savepoint effects are known. +type sqliteConnectionState struct { + backend *sqliteBackend + sqlite *sqlite3.SQLiteConn + pending []sqlapi.Mutation + pendingBytes int + maxChanges int + maxBytes int + savepoints []savepoint + statementMark int + statementSavepointVerb string + statementSavepointName string + rollbackSeen bool + rollbackUnconfirmed bool + commitPending bool + commitEnds []int + confirmedEnds int + fenceHeld bool + ddlInTxn bool + dmlInTxn bool + failed error + statementDDL bool + unsupported error + prepareMeta statementMeta +} + +// statementMeta is collected by SQLite's authorizer while a statement is +// prepared. It avoids interpreting comments, literals, or quoted identifiers +// as executable SQL control words. A prepared statement carries this metadata +// to its later execution; direct Exec/Query paths merge it after the driver's +// native prepare loop returns. +type statementMeta struct { + ddl bool + unsupported error + savepointVerb string + savepointName string + savepointCount int +} + +type savepoint struct { + name string + index int +} + +func (s *sqliteConnectionState) bind(conn *sqlite3.SQLiteConn) { + conn.RegisterPreUpdateHook(s.preUpdate) + conn.RegisterCommitHook(s.commit) + conn.RegisterRollbackHook(s.rollback) + conn.RegisterAuthorizer(s.authorizer) + s.sqlite = conn +} + +func (s *sqliteConnectionState) clear(conn *sqlite3.SQLiteConn) { + conn.RegisterPreUpdateHook(nil) + conn.RegisterCommitHook(nil) + conn.RegisterRollbackHook(nil) + conn.RegisterAuthorizer(nil) +} + +func (s *sqliteConnectionState) authorizer(action int, arg1, arg2, _ string) int { + // Reaching authorizer for another prepared statement proves that any + // earlier commit-hook boundary belongs to a completed statement. This is + // the native boundary signal needed when a later statement in one Exec + // fails before its own pre-update hook runs. + if len(s.commitEnds) > s.confirmedEnds { + s.confirmedEnds = len(s.commitEnds) + } + switch action { + case sqlite3.SQLITE_CREATE_INDEX, + sqlite3.SQLITE_CREATE_TABLE, + sqlite3.SQLITE_CREATE_TEMP_INDEX, + sqlite3.SQLITE_CREATE_TEMP_TABLE, + sqlite3.SQLITE_CREATE_TEMP_TRIGGER, + sqlite3.SQLITE_CREATE_TEMP_VIEW, + sqlite3.SQLITE_CREATE_TRIGGER, + sqlite3.SQLITE_CREATE_VIEW, + sqlite3.SQLITE_DROP_INDEX, + sqlite3.SQLITE_DROP_TABLE, + sqlite3.SQLITE_DROP_TEMP_INDEX, + sqlite3.SQLITE_DROP_TEMP_TABLE, + sqlite3.SQLITE_DROP_TEMP_TRIGGER, + sqlite3.SQLITE_DROP_TEMP_VIEW, + sqlite3.SQLITE_DROP_TRIGGER, + sqlite3.SQLITE_DROP_VIEW, + sqlite3.SQLITE_ALTER_TABLE, + sqlite3.SQLITE_ATTACH, + sqlite3.SQLITE_DETACH: + s.prepareMeta.ddl = true + case sqlite3.SQLITE_CREATE_VTABLE: + s.prepareMeta.ddl = true + s.prepareMeta.unsupported = fmt.Errorf("sqlite mutation observer cannot observe virtual table %s", arg1) + case sqlite3.SQLITE_DROP_VTABLE: + s.prepareMeta.ddl = true + s.prepareMeta.unsupported = fmt.Errorf("sqlite mutation observer cannot observe dropped virtual table %s", arg1) + case sqlite3.SQLITE_SAVEPOINT: + verb, name := authorizerSavepoint(arg1, arg2) + if verb != "" { + s.prepareMeta.savepointCount++ + s.prepareMeta.savepointVerb = verb + s.prepareMeta.savepointName = name + } + } + return sqlite3.SQLITE_OK +} + +func authorizerSavepoint(operation, name string) (string, string) { + switch strings.ToLower(operation) { + case "begin": + return "savepoint", normalizeSavepointName(name) + case "rollback": + return "rollback to", normalizeSavepointName(name) + case "release": + return "release", normalizeSavepointName(name) + default: + return "", "" + } +} + +func (s *sqliteConnectionState) preUpdate(data sqlite3.SQLitePreUpdateData) { + if s.failed != nil || s.statementDDL || strings.HasPrefix(strings.ToLower(data.TableName), "sqlite_") { + return + } + // A later statement can only reach pre-update after the preceding + // autocommit callback returned successfully. Confirm that preceding fence + // boundary before collecting the new candidate. + if len(s.commitEnds) > s.confirmedEnds { + s.confirmedEnds = len(s.commitEnds) + } + count := data.Count() + var before, after []any + var err error + switch data.Op { + case sqlite3.SQLITE_INSERT: + after, err = scanSQLiteRow(&data, count, true) + case sqlite3.SQLITE_UPDATE: + before, err = scanSQLiteRow(&data, count, false) + if err == nil { + after, err = scanSQLiteRow(&data, count, true) + } + case sqlite3.SQLITE_DELETE: + before, err = scanSQLiteRow(&data, count, false) + } + if err != nil { + s.failed = err + return + } + + op := "unknown" + switch data.Op { + case sqlite3.SQLITE_INSERT: + op = "insert" + case sqlite3.SQLITE_UPDATE: + op = "update" + case sqlite3.SQLITE_DELETE: + op = "delete" + } + s.pending = append(s.pending, sqlapi.Mutation{ + Schema: data.DatabaseName, + Table: data.TableName, + OldRowID: data.OldRowID, + RowID: data.NewRowID, + Before: before, + After: after, + Op: op, + }) + s.pendingBytes = saturatingAdd(s.pendingBytes, mutationSize(s.pending[len(s.pending)-1])) + if (s.maxChanges > 0 && len(s.pending) > s.maxChanges) || (s.maxBytes > 0 && s.pendingBytes > s.maxBytes) { + s.failed = errObserverOverflow + return + } + s.dmlInTxn = true +} + +func (s *sqliteConnectionState) commit() int { + // A single go-sqlite3 ExecContext may execute several semicolon-separated + // statements inside one C call. Reuse the fence when SQLite invokes the + // commit hook more than once before the wrapper gets control back; the + // wrapper publishes the combined candidate and releases it exactly once. + if !s.fenceHeld { + s.backend.acquireCommitFence() + s.fenceHeld = true + } + s.commitPending = true + s.commitEnds = append(s.commitEnds, len(s.pending)) + return 0 +} + +func (s *sqliteConnectionState) rollback() { + s.rollbackSeen = true + // A failed ROLLBACK TO/RELEASE SAVEPOINT can invoke SQLite's rollback + // hook even though the surrounding transaction remains active. The + // authorizer metadata identifies that control statement; defer bookkeeping + // until the wrapper sees its actual error instead of discarding the outer + // transaction candidate here. + if s.prepareMeta.savepointVerb != "" || s.statementSavepointVerb != "" { + return + } + if len(s.commitEnds) > s.confirmedEnds { + s.rollbackUnconfirmed = true + } + if len(s.commitEnds) > 0 { + s.commitPending = true + s.failed = nil + return + } + s.pending = nil + s.pendingBytes = 0 + s.savepoints = nil + s.statementMark = 0 + s.statementSavepointVerb = "" + s.statementSavepointName = "" + s.prepareMeta = statementMeta{} + s.commitPending = false + s.commitEnds = nil + s.confirmedEnds = 0 + if s.fenceHeld { + s.fenceHeld = false + s.backend.releaseFence() + } + s.ddlInTxn = false + s.dmlInTxn = false + s.failed = nil + s.statementDDL = false + s.statementSavepointVerb = "" + s.statementSavepointName = "" + s.prepareMeta = statementMeta{} + s.unsupported = nil +} + +func (s *sqliteConnectionState) statementBegin(_ string) { + s.statementBeginWithMeta(statementMeta{}) +} + +func (s *sqliteConnectionState) statementBeginWithMeta(meta statementMeta) { + s.statementMark = len(s.pending) + s.statementDDL = meta.ddl + s.statementSavepointVerb = meta.savepointVerb + s.statementSavepointName = meta.savepointName + s.rollbackSeen = false + s.rollbackUnconfirmed = false + s.prepareMeta = statementMeta{} +} + +func (s *sqliteConnectionState) statementEnd(_ string, err error) { + s.statementEndWithMeta(err, statementMeta{}) +} + +func (s *sqliteConnectionState) statementEndWithMeta(err error, meta statementMeta) { + s.applyStatementMeta(meta) + if s.rollbackUnconfirmed { + s.resolveRollbackOutcome() + } + if err == nil { + s.confirmedEnds = len(s.commitEnds) + } else if !s.rollbackSeen { + // A commit hook boundary is authoritative once the driver call has + // returned without a rollback callback. Keep confirmed prefixes even + // when a later native statement in the same call reports an error. + s.confirmedEnds = len(s.commitEnds) + } + lastBoundary := 0 + if len(s.commitEnds) > 0 { + lastBoundary = s.commitEnds[len(s.commitEnds)-1] + } + residual := len(s.pending) > s.statementMark + if len(s.commitEnds) > 0 { + residual = len(s.pending) > lastBoundary + } + if err != nil && !s.rollbackSeen && residual { + // SQLite's pre-update hook does not expose whether a failed DML + // statement used ABORT or FAIL. Do not infer the conflict mode from + // caller SQL; closing the observer is safer than publishing a partial + // candidate whose commit status is unknown. + s.failAmbiguous() + truncate := s.statementMark + if lastBoundary > truncate { + truncate = lastBoundary + } + if truncate <= len(s.pending) { + s.pending = s.pending[:truncate] + } + s.recomputePendingBytes() + } + s.statementMark = 0 + s.rollbackSeen = false + s.rollbackUnconfirmed = false + if err == nil { + if s.statementDDL { + s.ddlInTxn = true + } + s.applySavepoint() + } + s.statementDDL = false + s.statementSavepointVerb = "" + s.statementSavepointName = "" + if s.unsupported != nil && s.backend.hasObservers() { + s.backend.fail(s.unsupported) + } + s.unsupported = nil + if s.failed != nil { + if s.commitPending { + s.finalize() + } + return + } + if s.commitPending { + s.finalize() + } +} + +// resolveRollbackOutcome runs after SQLite has returned from the physical +// operation. A rollback hook can follow a committed autocommit prefix when a +// later native statement in the same Exec fails, but it can also follow a +// failed physical commit. The hook alone cannot distinguish those cases. The +// wrapper therefore verifies the unconfirmed net rows on the same connection; +// if they are not provably committed, the observer fails closed. +func (s *sqliteConnectionState) resolveRollbackOutcome() { + if !s.rollbackUnconfirmed { + return + } + if s.sqlite == nil || !s.sqlite.AutoCommit() { + s.failAmbiguous() + return + } + base := 0 + if s.confirmedEnds > 0 { + base = s.commitEnds[s.confirmedEnds-1] + } + if base > len(s.pending) { + s.failAmbiguous() + return + } + changes, err := s.netChangesFor(s.pending[base:]) + if err != nil { + s.failAmbiguous() + return + } + committed, err := s.finalRowsMatch(changes) + if err != nil { + s.failAmbiguous() + return + } + if !committed { + if s.confirmedEnds > 0 { + s.pending = s.pending[:base] + s.recomputePendingBytes() + s.commitEnds = s.commitEnds[:s.confirmedEnds] + s.rollbackUnconfirmed = false + return + } + s.failAmbiguous() + return + } + s.confirmedEnds = len(s.commitEnds) + s.rollbackUnconfirmed = false +} + +func (s *sqliteConnectionState) finalRowsMatch(changes []sqlapi.Mutation) (bool, error) { + if len(changes) == 0 { + return true, nil + } + idsByTable := make(map[string][]int64) + seen := make(map[string]map[int64]struct{}) + for _, change := range changes { + rowID := change.RowID + if change.Op == "delete" { + rowID = change.OldRowID + } + if rowID == 0 { + return false, fmt.Errorf("sqlite mutation observer cannot verify %s.%s row", change.Schema, change.Table) + } + tableKey := change.Schema + "\x00" + change.Table + if seen[tableKey] == nil { + seen[tableKey] = make(map[int64]struct{}) + } + if _, ok := seen[tableKey][rowID]; !ok { + seen[tableKey][rowID] = struct{}{} + idsByTable[tableKey] = append(idsByTable[tableKey], rowID) + } + } + + rowsByKey := make(map[mutationKey][]any) + for tableKey, rowIDs := range idsByTable { + parts := strings.SplitN(tableKey, "\x00", 2) + if len(parts) != 2 { + return false, errors.New("sqlite mutation observer table key is invalid") + } + for start := 0; start < len(rowIDs); start += 500 { + end := start + 500 + if end > len(rowIDs) { + end = len(rowIDs) + } + placeholders := make([]string, end-start) + args := make([]driver.Value, end-start) + for i, rowID := range rowIDs[start:end] { + placeholders[i] = "?" + args[i] = rowID + } + query := fmt.Sprintf("SELECT rowid, * FROM %s.%s WHERE rowid IN (%s)", quoteIdentifier(parts[0]), quoteIdentifier(parts[1]), strings.Join(placeholders, ",")) + rawRows, err := s.sqlite.Query(query, args) + if err != nil { + return false, err + } + values := make([]driver.Value, len(rawRows.Columns())) + for { + err = rawRows.Next(values) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + _ = rawRows.Close() + return false, err + } + rowID, ok := values[0].(int64) + if !ok { + _ = rawRows.Close() + return false, fmt.Errorf("sqlite mutation observer returned rowid type %T", values[0]) + } + after := make([]any, len(values)-1) + for i := range after { + after[i] = values[i+1] + } + rowsByKey[mutationKey{schema: parts[0], table: parts[1], rowID: rowID}] = after + } + _ = rawRows.Close() + } + } + + for _, change := range changes { + rowID := change.RowID + if change.Op == "delete" { + rowID = change.OldRowID + } + row, exists := rowsByKey[mutationKey{schema: change.Schema, table: change.Table, rowID: rowID}] + switch change.Op { + case "delete": + if exists { + return false, nil + } + case "insert", "update": + if !exists || !mutationValuesEqual(row, change.After) { + return false, nil + } + default: + return false, fmt.Errorf("sqlite mutation observer cannot verify operation %q", change.Op) + } + } + return true, nil +} + +func mutationValuesEqual(left, right []any) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if !mutationValueEqual(left[i], right[i]) { + return false + } + } + return true +} + +func mutationValueEqual(left, right any) bool { + if leftBytes, ok := left.([]byte); ok { + switch rightValue := right.(type) { + case []byte: + return bytes.Equal(leftBytes, rightValue) + case string: + return string(leftBytes) == rightValue + } + } + if rightBytes, ok := right.([]byte); ok { + if leftString, ok := left.(string); ok { + return leftString == string(rightBytes) + } + } + return reflect.DeepEqual(left, right) +} + +func (s *sqliteConnectionState) applyStatementMeta(meta statementMeta) { + if s.prepareMeta.savepointCount > 1 || meta.savepointCount > 1 { + s.failAmbiguous() + } + if s.prepareMeta.ddl || meta.ddl { + s.statementDDL = true + } + if s.prepareMeta.unsupported != nil { + s.unsupported = s.prepareMeta.unsupported + } + if meta.unsupported != nil { + s.unsupported = meta.unsupported + } + if s.prepareMeta.savepointVerb != "" { + s.statementSavepointVerb = s.prepareMeta.savepointVerb + s.statementSavepointName = s.prepareMeta.savepointName + } + if meta.savepointVerb != "" { + s.statementSavepointVerb = meta.savepointVerb + s.statementSavepointName = meta.savepointName + } + s.prepareMeta = statementMeta{} +} + +func (s *sqliteConnectionState) failAmbiguous() { + s.failed = errObserverAmbiguous + if s.backend.hasObservers() { + s.backend.fail(errObserverAmbiguous) + } +} + +func (s *sqliteConnectionState) finalizeAfterError(_ string, err error) { + s.statementEndWithMeta(err, statementMeta{}) +} + +// finalize runs only after SQLite has returned from the operation that caused +// the commit hook. The hook is therefore a candidate marker; it never emits +// data itself. Net reduction uses the hook images collected on this physical +// connection, after statement/savepoint outcomes are known. +func (s *sqliteConnectionState) finalize() { + if !s.commitPending { + return + } + s.commitPending = false + defer func() { + if s.fenceHeld { + s.fenceHeld = false + s.backend.releaseFence() + } + }() + if !s.backend.hasObservers() { + s.resetTransaction() + return + } + if s.failed != nil { + s.backend.fail(s.failed) + s.resetTransaction() + return + } + if s.ddlInTxn && s.dmlInTxn { + s.backend.fail(errors.New("sqlite mutation observer cannot represent DDL and DML in one transaction")) + s.resetTransaction() + return + } + ends := s.commitEnds + if len(ends) == 0 { + ends = []int{len(s.pending)} + } + start := 0 + for _, end := range ends { + if end < start || end > len(s.pending) { + s.backend.fail(errors.New("sqlite mutation observer commit boundary is invalid")) + s.resetTransaction() + return + } + changes, err := s.netChangesFor(s.pending[start:end]) + if err != nil { + s.backend.fail(err) + s.resetTransaction() + return + } + s.backend.publish(changes) + start = end + } + s.resetTransaction() +} + +func (s *sqliteConnectionState) resetTransaction() { + s.pending = nil + s.pendingBytes = 0 + s.savepoints = nil + s.statementMark = 0 + s.statementSavepointVerb = "" + s.statementSavepointName = "" + s.prepareMeta = statementMeta{} + s.commitPending = false + s.commitEnds = nil + s.confirmedEnds = 0 + s.ddlInTxn = false + s.dmlInTxn = false + s.statementDDL = false + s.unsupported = nil + s.rollbackSeen = false + s.rollbackUnconfirmed = false + s.prepareMeta = statementMeta{} +} + +// Savepoint state is applied only after SQLite reports success. A failed +// ROLLBACK TO/RELEASE must not change the candidate transaction in memory. +func (s *sqliteConnectionState) applySavepoint() { + verb, name := s.statementSavepointVerb, s.statementSavepointName + switch verb { + case "savepoint": + s.savepoints = append(s.savepoints, savepoint{name: name, index: len(s.pending)}) + case "rollback to": + if index, ok := s.findSavepoint(name); ok { + s.pending = s.pending[:index] + s.recomputePendingBytes() + for i := len(s.savepoints) - 1; i >= 0; i-- { + if s.savepoints[i].name == name { + s.savepoints = s.savepoints[:i+1] + break + } + } + } + case "release": + if index, ok := s.findSavepoint(name); ok { + for i := len(s.savepoints) - 1; i >= 0; i-- { + if s.savepoints[i].index == index { + s.savepoints = s.savepoints[:i] + break + } + } + } + } +} + +func (s *sqliteConnectionState) recomputePendingBytes() { + s.pendingBytes = 0 + for _, change := range s.pending { + s.pendingBytes += mutationSize(change) + } +} + +type mutationKey struct { + schema string + table string + rowID int64 +} + +type netMutation struct { + mutation sqlapi.Mutation + first string + last string +} + +// netChanges retains the earliest before-image for each row and the latest +// pre-update after-image. SQLite invokes the hook for every trigger-generated +// row change too, so the latest image is the committed row state without an +// O(N) SELECT round trip during the commit fence. Table metadata is resolved +// once per touched table. +func (s *sqliteConnectionState) netChangesFor(pending []sqlapi.Mutation) ([]sqlapi.Mutation, error) { + if len(pending) == 0 { + return nil, nil + } + if s.sqlite == nil { + return nil, errors.New("sqlite mutation observer has no physical connection") + } + nets := make(map[mutationKey]*netMutation, len(pending)) + order := make([]mutationKey, 0, len(pending)) + aliases := make(map[mutationKey]mutationKey) + columnsByTable := make(map[string][]string) + for _, change := range pending { + if change.OldRowID == 0 && change.RowID == 0 { + return nil, fmt.Errorf("sqlite mutation observer cannot identify %s.%s row", change.Schema, change.Table) + } + tableKey := change.Schema + "\x00" + change.Table + columns, ok := columnsByTable[tableKey] + if !ok { + if err := s.validateTable(change.Schema, change.Table); err != nil { + return nil, err + } + var err error + columns, err = s.tableColumns(change.Schema, change.Table) + if err != nil { + return nil, err + } + columnsByTable[tableKey] = columns + } + key := mutationKey{schema: change.Schema, table: change.Table, rowID: change.OldRowID} + if key.rowID == 0 { + key.rowID = change.RowID + } + if alias, ok := aliases[key]; ok { + key = alias + } + current, ok := nets[key] + if !ok { + current = &netMutation{mutation: change, first: change.Op, last: change.Op} + current.mutation.Columns = columns + nets[key] = current + order = append(order, key) + } else { + current.last = change.Op + current.mutation.Columns = columns + if change.Op == "delete" { + current.mutation.After = nil + } else if change.After != nil { + current.mutation.After = change.After + } + if current.mutation.RowID != change.RowID && change.RowID != 0 { + aliases[mutationKey{schema: change.Schema, table: change.Table, rowID: change.RowID}] = key + current.mutation.RowID = change.RowID + } + if current.mutation.OldRowID == 0 { + current.mutation.OldRowID = change.OldRowID + } + } + if change.RowID != 0 { + aliases[mutationKey{schema: change.Schema, table: change.Table, rowID: change.RowID}] = key + } + } + + result := make([]sqlapi.Mutation, 0, len(order)) + for _, key := range order { + current := nets[key] + if current.first == "insert" && current.last == "delete" { + continue + } + change := current.mutation + switch { + case current.last == "delete": + change.Op = "delete" + change.RowID = 0 + case current.first == "insert": + change.Op = "insert" + default: + change.Op = "update" + } + result = append(result, change) + } + return result, nil +} + +func (s *sqliteConnectionState) findSavepoint(name string) (int, bool) { + for i := len(s.savepoints) - 1; i >= 0; i-- { + if s.savepoints[i].name == name { + return s.savepoints[i].index, true + } + } + return 0, false +} + +func (s *sqliteConnectionState) validateTable(schema, table string) error { + if strings.EqualFold(schema, "temp") { + return errors.New("sqlite mutation observer does not support TEMP tables") + } + query := fmt.Sprintf("SELECT sql FROM %s.sqlite_master WHERE type = 'table' AND name = ?", quoteIdentifier(schema)) + rows, err := s.sqlite.Query(query, []driver.Value{table}) + if err != nil { + return fmt.Errorf("inspect sqlite table %s.%s: %w", schema, table, err) + } + defer rows.Close() + values := make([]driver.Value, len(rows.Columns())) + if err := rows.Next(values); err != nil { + if err == io.EOF { + return fmt.Errorf("sqlite table %s.%s disappeared", schema, table) + } + return err + } + definition := "" + switch value := values[0].(type) { + case string: + definition = value + case []byte: + definition = string(value) + } + upper := strings.ToUpper(definition) + if strings.Contains(upper, "WITHOUT ROWID") { + return fmt.Errorf("sqlite mutation observer does not support WITHOUT ROWID table %s.%s", schema, table) + } + if strings.HasPrefix(strings.TrimSpace(upper), "CREATE VIRTUAL TABLE") { + return fmt.Errorf("sqlite mutation observer does not support virtual table %s.%s", schema, table) + } + return nil +} + +func (s *sqliteConnectionState) tableColumns(schema, table string) ([]string, error) { + query := fmt.Sprintf("SELECT * FROM %s.%s LIMIT 0", quoteIdentifier(schema), quoteIdentifier(table)) + rows, err := s.sqlite.Query(query, nil) + if err != nil { + return nil, fmt.Errorf("read sqlite columns %s.%s: %w", schema, table, err) + } + defer rows.Close() + return append([]string(nil), rows.Columns()...), nil +} + +func quoteIdentifier(value string) string { + return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` +} + +func scanSQLiteRow(data *sqlite3.SQLitePreUpdateData, count int, isNew bool) ([]any, error) { + if count <= 0 { + return nil, nil + } + values := make([]any, count) + var err error + if isNew { + err = data.New(values...) + } else { + err = data.Old(values...) + } + return values, err +} + +func normalizeSavepointName(name string) string { + name = strings.TrimSpace(strings.TrimSuffix(name, ";")) + name = strings.Trim(name, "`\"[]") + return strings.ToLower(name) +} + +// observedConn delegates all normal SQL behavior while ensuring every physical +// connection operation gets a post-operation finalization point. +type observedConn struct { + raw driver.Conn + sqlite *sqlite3.SQLiteConn + backend *sqliteBackend + state *sqliteConnectionState +} + +func (c *observedConn) bindIfActive() { + c.state.bind(c.sqlite) +} + +func (c *observedConn) clearHooks() { c.state.clear(c.sqlite) } + +func (c *observedConn) Prepare(query string) (driver.Stmt, error) { + c.state.prepareMeta = statementMeta{} + stmt, err := c.raw.Prepare(query) + meta := c.state.prepareMeta + c.state.prepareMeta = statementMeta{} + if err != nil { + return nil, err + } + return &observedStmt{raw: stmt, conn: c, query: query, meta: meta}, nil +} + +func (c *observedConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + preparer, ok := c.raw.(driver.ConnPrepareContext) + if !ok { + return nil, driver.ErrSkip + } + c.state.prepareMeta = statementMeta{} + stmt, err := preparer.PrepareContext(ctx, query) + meta := c.state.prepareMeta + c.state.prepareMeta = statementMeta{} + if err != nil { + return nil, err + } + return &observedStmt{raw: stmt, conn: c, query: query, meta: meta}, nil +} + +func (c *observedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + execer, ok := c.raw.(driver.ExecerContext) + if !ok { + return nil, driver.ErrSkip + } + c.state.statementBegin(query) + result, err := execer.ExecContext(ctx, query, args) + c.state.statementEnd(query, err) + return result, err +} + +func (c *observedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := c.raw.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + c.state.statementBegin(query) + rows, err := queryer.QueryContext(ctx, query, args) + if err != nil { + c.state.statementEnd(query, err) + return nil, err + } + return &observedRows{raw: rows, conn: c, query: query}, nil +} + +func (c *observedConn) CheckNamedValue(value *driver.NamedValue) error { + checker, ok := c.raw.(driver.NamedValueChecker) + if !ok { + return driver.ErrSkip + } + return checker.CheckNamedValue(value) +} + +func (c *observedConn) Ping(ctx context.Context) error { + pinger, ok := c.raw.(driver.Pinger) + if !ok { + return nil + } + return pinger.Ping(ctx) +} + +func (c *observedConn) Close() error { + c.state.rollback() + return c.raw.Close() +} + +func (c *observedConn) Begin() (driver.Tx, error) { + tx, err := c.raw.Begin() + if err != nil { + return nil, err + } + return &observedTx{raw: tx, conn: c}, nil +} + +func (c *observedConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + begin, ok := c.raw.(driver.ConnBeginTx) + if !ok { + return c.Begin() + } + tx, err := begin.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + return &observedTx{raw: tx, conn: c}, nil +} + +type observedTx struct { + raw driver.Tx + conn *observedConn +} + +func (t *observedTx) Commit() error { + err := t.raw.Commit() + if t.conn.state.commitPending { + t.conn.state.finalizeAfterError("COMMIT", err) + } + return err +} + +func (t *observedTx) Rollback() error { + err := t.raw.Rollback() + t.conn.state.rollback() + return err +} + +type observedStmt struct { + raw driver.Stmt + conn *observedConn + query string + meta statementMeta +} + +func (s *observedStmt) Close() error { return s.raw.Close() } +func (s *observedStmt) NumInput() int { return s.raw.NumInput() } + +func (s *observedStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + execer, ok := s.raw.(driver.StmtExecContext) + if !ok { + return nil, driver.ErrSkip + } + s.conn.state.statementBeginWithMeta(s.meta) + result, err := execer.ExecContext(ctx, args) + s.conn.state.statementEndWithMeta(err, s.meta) + return result, err +} + +func (s *observedStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := s.raw.(driver.StmtQueryContext) + if !ok { + return nil, driver.ErrSkip + } + s.conn.state.statementBeginWithMeta(s.meta) + rows, err := queryer.QueryContext(ctx, args) + if err != nil { + s.conn.state.statementEndWithMeta(err, s.meta) + return nil, err + } + return &observedRows{raw: rows, conn: s.conn, query: s.query, meta: s.meta}, nil +} + +func (s *observedStmt) ColumnConverter(index int) driver.ValueConverter { + if converter, ok := s.raw.(driver.ColumnConverter); ok { + return converter.ColumnConverter(index) + } + return driver.DefaultParameterConverter +} + +func (s *observedStmt) Exec(args []driver.Value) (driver.Result, error) { + s.conn.state.statementBeginWithMeta(s.meta) + result, err := s.raw.Exec(args) + s.conn.state.statementEndWithMeta(err, s.meta) + return result, err +} + +func (s *observedStmt) Query(args []driver.Value) (driver.Rows, error) { + s.conn.state.statementBeginWithMeta(s.meta) + rows, err := s.raw.Query(args) + if err != nil { + s.conn.state.statementEndWithMeta(err, s.meta) + return nil, err + } + return &observedRows{raw: rows, conn: s.conn, query: s.query, meta: s.meta}, nil +} + +type observedRows struct { + raw driver.Rows + conn *observedConn + query string + meta statementMeta + closed bool + mu sync.Mutex +} + +func (r *observedRows) Columns() []string { return r.raw.Columns() } + +func (r *observedRows) Next(dest []driver.Value) error { + err := r.raw.Next(dest) + if err == io.EOF || err != nil { + r.finish(err) + } + return err +} + +func (r *observedRows) Close() error { + err := r.raw.Close() + r.finish(err) + return err +} + +func (r *observedRows) ColumnTypeDatabaseTypeName(index int) string { + if rows, ok := r.raw.(driver.RowsColumnTypeDatabaseTypeName); ok { + return rows.ColumnTypeDatabaseTypeName(index) + } + return "" +} + +func (r *observedRows) ColumnTypeLength(index int) (int64, bool) { + if rows, ok := r.raw.(driver.RowsColumnTypeLength); ok { + return rows.ColumnTypeLength(index) + } + return 0, false +} + +func (r *observedRows) ColumnTypeNullable(index int) (bool, bool) { + if rows, ok := r.raw.(driver.RowsColumnTypeNullable); ok { + return rows.ColumnTypeNullable(index) + } + return false, false +} + +func (r *observedRows) ColumnTypePrecisionScale(index int) (int64, int64, bool) { + if rows, ok := r.raw.(driver.RowsColumnTypePrecisionScale); ok { + return rows.ColumnTypePrecisionScale(index) + } + return 0, 0, false +} + +func (r *observedRows) ColumnTypeScanType(index int) reflect.Type { + if rows, ok := r.raw.(driver.RowsColumnTypeScanType); ok { + return rows.ColumnTypeScanType(index) + } + return reflect.TypeOf((*any)(nil)).Elem() +} + +func (r *observedRows) finish(err error) { + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return + } + r.closed = true + r.mu.Unlock() + r.conn.state.statementEndWithMeta(err, r.meta) +} + +type mutationStream struct { + backend *sqliteBackend + opts sqlapi.MutationOptions + changes chan sqlapi.MutationBatch + notify chan struct{} + done chan struct{} + mu sync.Mutex + err error + closed bool + snapshotting bool + watermark string + queue []sqlapi.MutationBatch + pending []sqlapi.MutationBatch + queuedChanges int + queuedBytes int + cancel context.CancelFunc + maxChanges int + maxBytes int +} + +func newMutationStream(backend *sqliteBackend, opts sqlapi.MutationOptions) *mutationStream { + stream := &mutationStream{ + backend: backend, + opts: opts, + changes: make(chan sqlapi.MutationBatch), + notify: make(chan struct{}, 1), + done: make(chan struct{}), + maxChanges: opts.MaxChanges, + maxBytes: opts.MaxBytes, + } + go stream.relay() + return stream +} + +func newSnapshotStream(backend *sqliteBackend, opts sqlapi.SnapshotOptions, watermark string, cancel context.CancelFunc) *mutationStream { + stream := newMutationStream(backend, sqlapi.MutationOptions{ + Tables: opts.Tables, MaxChanges: opts.MaxChanges, MaxBytes: opts.MaxBytes, + }) + stream.snapshotting = true + stream.watermark = watermark + stream.cancel = cancel + return stream +} + +func (s *mutationStream) Changes() <-chan sqlapi.MutationBatch { return s.changes } + +func (s *mutationStream) Err() error { + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err +} + +func (s *mutationStream) Close() error { + s.mu.Lock() + cancel := s.cancel + s.mu.Unlock() + if cancel != nil { + cancel() + } + s.backend.remove(s, nil) + return nil +} + +func (s *mutationStream) push(batch sqlapi.MutationBatch) { + batch = filterBatch(batch, s.opts) + if len(batch.Changes) == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return + } + if s.snapshotting { + if !s.enqueuePendingLocked(batch) { + s.closeLocked(errObserverOverflow) + return + } + return + } + if !s.enqueueLocked(batch) { + s.closeLocked(errObserverOverflow) + } +} + +func filterBatch(batch sqlapi.MutationBatch, opts sqlapi.MutationOptions) sqlapi.MutationBatch { + if len(opts.Tables) == 0 && len(opts.Operations) == 0 { + return batch + } + filtered := make([]sqlapi.Mutation, 0, len(batch.Changes)) + for _, change := range batch.Changes { + if len(opts.Tables) > 0 && !matchesTable(change.Schema, change.Table, opts.Tables) { + continue + } + if len(opts.Operations) > 0 && !matchesValue(change.Op, opts.Operations) { + continue + } + filtered = append(filtered, change) + } + batch.Changes = filtered + return batch +} + +func matchesTable(schema, table string, filters []string) bool { + for _, filter := range filters { + if filter == table || filter == schema+"."+table { + return true + } + } + return false +} + +func matchesValue(value string, filters []string) bool { + for _, filter := range filters { + if strings.EqualFold(value, filter) { + return true + } + } + return false +} + +func (s *mutationStream) pushSnapshot(batch sqlapi.MutationBatch) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + if s.err != nil { + return s.err + } + return errObserverClosed + } + if !s.enqueueLocked(batch) { + s.closeLocked(errObserverOverflow) + return errObserverOverflow + } + return nil +} + +func (s *mutationStream) finishSnapshot(err error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return + } + if err != nil { + s.closeLocked(err) + return + } + s.snapshotting = false + s.queue = append(s.queue, s.pending...) + s.pending = nil + s.signalLocked() +} + +func (s *mutationStream) Watermark() string { return s.watermark } + +func (s *mutationStream) enqueueLocked(batch sqlapi.MutationBatch) bool { + changes := len(batch.Changes) + bytes := mutationBatchBytes(batch) + if s.maxChanges > 0 && (changes > s.maxChanges || s.queuedChanges > s.maxChanges-changes) { + return false + } + if s.maxBytes > 0 && (bytes > s.maxBytes || s.queuedBytes > s.maxBytes-bytes) { + return false + } + s.queue = append(s.queue, batch) + s.queuedChanges = saturatingAdd(s.queuedChanges, changes) + s.queuedBytes = saturatingAdd(s.queuedBytes, bytes) + s.signalLocked() + return true +} + +func (s *mutationStream) enqueuePendingLocked(batch sqlapi.MutationBatch) bool { + changes := len(batch.Changes) + bytes := mutationBatchBytes(batch) + if s.maxChanges > 0 && (changes > s.maxChanges || s.queuedChanges > s.maxChanges-changes) { + return false + } + if s.maxBytes > 0 && (bytes > s.maxBytes || s.queuedBytes > s.maxBytes-bytes) { + return false + } + s.pending = append(s.pending, batch) + s.queuedChanges = saturatingAdd(s.queuedChanges, changes) + s.queuedBytes = saturatingAdd(s.queuedBytes, bytes) + s.signalLocked() + return true +} + +func (s *mutationStream) signalLocked() { + select { + case s.notify <- struct{}{}: + default: + } +} + +func (s *mutationStream) relay() { + for { + s.mu.Lock() + if len(s.queue) == 0 { + closed := s.closed + s.mu.Unlock() + if closed { + close(s.changes) + return + } + <-s.notify + continue + } + batch := s.queue[0] + s.mu.Unlock() + + select { + case s.changes <- batch: + case <-s.done: + close(s.changes) + return + } + + s.mu.Lock() + if len(s.queue) > 0 { + s.queue = s.queue[1:] + s.queuedChanges -= len(batch.Changes) + s.queuedBytes -= mutationBatchBytes(batch) + } + s.mu.Unlock() + } +} + +func mutationBatchBytes(batch sqlapi.MutationBatch) int { + bytes := saturatingAdd(mutationStructuralBytes, len(batch.Transaction)) + for _, change := range batch.Changes { + bytes = saturatingAdd(bytes, mutationSize(change)) + } + return bytes +} + +func mutationSize(change sqlapi.Mutation) int { + bytes := mutationStructuralBytes + bytes = saturatingAdd(bytes, len(change.Schema)) + bytes = saturatingAdd(bytes, len(change.Table)) + bytes = saturatingAdd(bytes, len(change.Op)) + for _, column := range change.Columns { + bytes = saturatingAdd(bytes, len(column)) + } + bytes = saturatingAdd(bytes, mutationValuesBytes(change.Before)) + return saturatingAdd(bytes, mutationValuesBytes(change.After)) +} + +func mutationValuesBytes(values []any) int { + bytes := 0 + for _, value := range values { + bytes = saturatingAdd(bytes, valueStructuralBytes) + switch value := value.(type) { + case nil: + bytes = saturatingAdd(bytes, 1) + case []byte: + bytes = saturatingAdd(bytes, len(value)) + case string: + bytes = saturatingAdd(bytes, len(value)) + default: + bytes = saturatingAdd(bytes, 16) + } + } + return bytes +} + +func saturatingAdd(left, right int) int { + if left < 0 || right < 0 { + return int(^uint(0) >> 1) + } + maxInt := int(^uint(0) >> 1) + if left > maxInt-right { + return maxInt + } + return left + right +} + +func (s *mutationStream) closeWithError(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.closeLocked(err) +} + +func (s *mutationStream) closeLocked(err error) { + if s.closed { + return + } + s.closed = true + s.err = err + if s.cancel != nil { + s.cancel() + s.cancel = nil + } + close(s.done) + s.queue = nil + s.pending = nil + s.queuedChanges = 0 + s.queuedBytes = 0 + s.signalLocked() +} + +var _ driver.Connector = (*sqliteConnector)(nil) +var _ driver.Conn = (*observedConn)(nil) +var _ driver.Tx = (*observedTx)(nil) +var _ driver.Stmt = (*observedStmt)(nil) +var _ driver.Rows = (*observedRows)(nil) +var _ sqlapi.CommittedMutationSource = (*sqliteBackend)(nil) +var _ sqlapi.MutationStream = (*mutationStream)(nil) diff --git a/service/sql/engine/sqlite/observer_stub.go b/service/sql/engine/sqlite/observer_stub.go new file mode 100644 index 000000000..83141cfd4 --- /dev/null +++ b/service/sql/engine/sqlite/observer_stub.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build !sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "database/sql" + "errors" + + sqlapi "github.com/wippyai/runtime/api/service/sql" +) + +var errObserverUnavailable = errors.New("sqlite committed-mutation observation is unavailable in this build") + +// openSQLite uses the normal registered SQLite driver in builds without the +// optional pre-update hook. The SQL resource remains fully usable; CDC callers +// receive an explicit unsupported capability instead of a partially working +// capture source. +func openSQLite(_ context.Context, dsn string, _ ...int) (*sql.DB, sqlapi.CommittedMutationSource, error) { + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, nil, err + } + return db, nil, nil +} diff --git a/service/sql/engine/sqlite/observer_test.go b/service/sql/engine/sqlite/observer_test.go new file mode 100644 index 000000000..4f1ef81b4 --- /dev/null +++ b/service/sql/engine/sqlite/observer_test.go @@ -0,0 +1,630 @@ +// SPDX-License-Identifier: MPL-2.0 + +//go:build sqlite_preupdate_hook + +package sqlite + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + config "github.com/wippyai/runtime/api/service/sql" + sqlservice "github.com/wippyai/runtime/service/sql" +) + +func openObservedDB(t *testing.T, file string) (*openedDBForTest, error) { + t.Helper() + opened, err := (engine{}).Open(context.Background(), &config.SQLiteConfig{File: file}) + if err != nil { + return nil, err + } + if err := (engine{}).Prepare(context.Background(), opened.DB, &config.SQLiteConfig{File: file}); err != nil { + _ = opened.DB.Close() + return nil, err + } + (engine{}).Tune(opened.DB, &config.SQLiteConfig{File: file, Pool: config.PoolConfig{MaxLifetime: time.Hour}}) + return &openedDBForTest{opened: opened}, nil +} + +type openedDBForTest struct { + opened sqlservice.OpenedDB +} + +func (o *openedDBForTest) Close() { + if o.opened.Observer != nil { + _ = o.opened.Observer.Close() + } + _ = o.opened.DB.Close() +} + +func TestPerPoolObserverCapturesOwnDatabase(t *testing.T) { + first, err := openObservedDB(t, filepath.Join(t.TempDir(), "first.db")) + require.NoError(t, err) + defer first.Close() + second, err := openObservedDB(t, filepath.Join(t.TempDir(), "second.db")) + require.NoError(t, err) + defer second.Close() + + for _, db := range []*openedDBForTest{first, second} { + _, err := db.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + } + + firstStream, err := first.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = firstStream.Close() }() + secondStream, err := second.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = secondStream.Close() }() + + _, err = first.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1, 'first')`) + require.NoError(t, err) + _, err = second.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (2, 'second')`) + require.NoError(t, err) + + firstBatch := receiveBatch(t, firstStream) + secondBatch := receiveBatch(t, secondStream) + require.Len(t, firstBatch.Changes, 1) + require.Len(t, secondBatch.Changes, 1) + assert.Equal(t, []byte("first"), firstBatch.Changes[0].After[1]) + assert.Equal(t, []byte("second"), secondBatch.Changes[0].After[1]) + + select { + case batch := <-firstStream.Changes(): + t.Fatalf("first pool received unrelated batch: %#v", batch) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverRebindsAfterConnectionExpiry(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "reconnect.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + observed.opened.DB.SetConnMaxLifetime(time.Nanosecond) + time.Sleep(2 * time.Millisecond) + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1, 'rebound')`) + require.NoError(t, err) + + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, []byte("rebound"), batch.Changes[0].After[1]) +} + +func TestObserverClosesWithGeneration(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "close.db")) + require.NoError(t, err) + + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + require.NoError(t, observed.opened.Observer.Close()) + + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer stream did not close with its generation") + } + assert.ErrorIs(t, stream.Err(), errObserverClosed) + _ = observed.opened.DB.Close() +} + +func TestObserverCloseCancelsSnapshotRead(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-close.db")) + require.NoError(t, err) + defer observed.opened.DB.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + for i := 1; i <= 64; i++ { + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (?, ?)`, i, "value") + require.NoError(t, err) + } + snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 1, + }) + require.NoError(t, err) + require.NoError(t, observed.opened.Observer.Close()) + select { + case _, ok := <-snapshot.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("snapshot stream did not close after observer shutdown") + } + assert.ErrorIs(t, snapshot.Err(), errObserverClosed) +} + +func TestObserverPublishesNetTransactionAfterCommit(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "net.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'first')`) + require.NoError(t, err) + _, err = tx.Exec(`UPDATE items SET value = 'second' WHERE id = 1`) + require.NoError(t, err) + _, err = tx.Exec(`UPDATE items SET value = 'final' WHERE id = 1`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + change := batch.Changes[0] + assert.Equal(t, "insert", change.Op) + assert.Nil(t, change.Before) + assert.Equal(t, []byte("final"), change.After[1]) + + tx, err = observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (2, 'transient')`) + require.NoError(t, err) + _, err = tx.Exec(`DELETE FROM items WHERE id = 2`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + select { + case extra := <-stream.Changes(): + t.Fatalf("insert/delete cycle was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverSavepointRollbackDoesNotPublishRolledBackRows(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "savepoint.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'kept')`) + require.NoError(t, err) + _, err = tx.Exec(`SAVEPOINT /* comment */ nested`) + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (2, 'rolled-back')`) + require.NoError(t, err) + _, err = tx.Exec(`ROLLBACK /* comment */ TO SAVEPOINT nested`) + require.NoError(t, err) + _, err = tx.Exec(`RELEASE /* comment */ SAVEPOINT nested`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) + assert.Equal(t, []byte("kept"), batch.Changes[0].After[1]) + select { + case extra := <-stream.Changes(): + t.Fatalf("rolled-back row was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverFailsClosedForAmbiguousPartialStatement(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "partial.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT OR FAIL INTO items (id, value) VALUES (1, 'first'), (2, 'first')`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for ambiguous partial statement") + } + assert.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverSnapshotFencesLiveWrites(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot.db")) + require.NoError(t, err) + defer observed.Close() + + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + for i := 1; i <= 3; i++ { + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (?, ?)`, i, "old") + require.NoError(t, err) + } + + snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 1, MaxChanges: 32, MaxBytes: 1 << 20, + }) + require.NoError(t, err) + defer func() { _ = snapshot.Close() }() + + writeDone := make(chan error, 1) + go func() { + _, writeErr := observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (99, 'live')`) + writeDone <- writeErr + }() + + var snapshotRows []int64 + for len(snapshotRows) < 3 { + batch := receiveBatch(t, snapshot) + require.True(t, batch.Snapshot) + for _, change := range batch.Changes { + snapshotRows = append(snapshotRows, change.RowID) + } + } + require.NoError(t, <-writeDone) + for { + batch := receiveBatch(t, snapshot) + if batch.Snapshot { + continue + } + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(99), batch.Changes[0].RowID) + assert.Equal(t, []byte("live"), batch.Changes[0].After[1]) + break + } + assert.ElementsMatch(t, []int64{1, 2, 3}, snapshotRows) +} + +func TestObserverSnapshotHandoffIncludesInFlightWriterAsLive(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-inflight.db")) + require.NoError(t, err) + defer observed.Close() + observed.opened.DB.SetMaxOpenConns(2) + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (10, 'in-flight')`) + require.NoError(t, err) + + snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{Tables: []string{"items"}, BatchSize: 8}) + require.NoError(t, err) + defer func() { _ = snapshot.Close() }() + require.NoError(t, tx.Commit()) + + select { + case batch := <-snapshot.Changes(): + if batch.Snapshot { + t.Fatalf("uncommitted writer appeared in snapshot: %#v", batch) + } + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(10), batch.Changes[0].RowID) + case <-time.After(time.Second): + t.Fatal("in-flight writer did not arrive as live batch") + } +} + +func TestObserverAbortedStatementDoesNotPublishPartialRows(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "abort.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'same'), (2, 'same')`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok, "ambiguous statement must close the observer stream") + case <-time.After(time.Second): + t.Fatal("ambiguous statement did not close the observer stream") + } + require.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverFailsClosedWithoutSQLConflictInference(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "conflict-lexing.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT /* OR FAIL */ INTO items (id, value) VALUES (1, 'literal'), (2, 'literal')`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(100 * time.Millisecond): + t.Fatal("observer did not fail closed for ambiguous conflict text") + } + assert.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverFailedSavepointCommandDoesNotCorruptCapture(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "failed-savepoint.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'kept')`) + require.NoError(t, err) + _, err = tx.Exec(`ROLLBACK TO SAVEPOINT missing`) + require.Error(t, err) + require.NoError(t, tx.Commit()) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) +} + +func TestObserverFailsClosedForMultipleSavepointsInOneExec(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "multi-savepoint.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + _, err = observed.opened.DB.Exec(`SAVEPOINT first; SAVEPOINT second`) + require.NoError(t, err) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for ambiguous multi-savepoint Exec") + } + assert.ErrorIs(t, stream.Err(), errObserverAmbiguous) +} + +func TestObserverReturningRowsFinalizeOnClose(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "returning.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + rows, err := observed.opened.DB.Query(`INSERT INTO items (id, value) VALUES (1, 'returned') RETURNING id`) + require.NoError(t, err) + var id int64 + require.True(t, rows.Next()) + require.NoError(t, rows.Scan(&id)) + require.Equal(t, int64(1), id) + require.NoError(t, rows.Close()) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) +} + +func TestObserverFiltersLiveMutations(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "filters.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT); CREATE TABLE ignored (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"items"}, Operations: []string{"insert"}}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.Exec(`INSERT INTO ignored (id, value) VALUES (1, 'no')`) + require.NoError(t, err) + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (2, 'yes')`) + require.NoError(t, err) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, "items", batch.Changes[0].Table) + select { + case extra := <-stream.Changes(): + t.Fatalf("filtered mutation was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverOverflowClosesWithoutBlockingCommit(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "overflow.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1, MaxBytes: 1 << 20}) + require.NoError(t, err) + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1, 'one'), (2, 'two')`) + require.NoError(t, err) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("overflow stream did not close") + } + assert.ErrorIs(t, stream.Err(), errObserverOverflow) +} + +func TestObserverKeepsMultiStatementCommitBoundaries(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "boundaries.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (?,'one'); INSERT INTO items (id, value) VALUES (?,'two')`, 1, 2) + require.NoError(t, err) + first := receiveBatch(t, stream) + second := receiveBatch(t, stream) + require.Len(t, first.Changes, 1) + require.Len(t, second.Changes, 1) + assert.Equal(t, int64(1), first.Changes[0].RowID) + assert.Equal(t, int64(2), second.Changes[0].RowID) +} + +func TestObserverKeepsCommittedPrefixBeforeParameterizedLaterError(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "parameterized-boundary-error.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.Exec( + `INSERT INTO items (id, value) VALUES (?,'one'); INSERT INTO items (id, value) VALUES (?,'one')`, + 1, 2, + ) + require.Error(t, err) + select { + case batch := <-stream.Changes(): + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) + case <-time.After(time.Second): + t.Fatal("committed prefix was not published") + } +} + +func TestObserverKeepsCommittedPrefixBeforeParameterizedSyntaxTail(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "parameterized-syntax-tail.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.Exec( + `INSERT INTO items (id, value) VALUES (?, 'one'); INSER INTO items (id, value) VALUES (?, 'two')`, + 1, 2, + ) + require.Error(t, err) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) +} + +func TestObserverPublishesEarlierAutocommitBeforeLaterError(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "boundary-error.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + defer func() { _ = stream.Close() }() + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1,'one')`) + require.NoError(t, err) + _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (2,'one')`) + require.Error(t, err) + batch := receiveBatch(t, stream) + require.Len(t, batch.Changes, 1) + assert.Equal(t, int64(1), batch.Changes[0].RowID) + select { + case extra := <-stream.Changes(): + t.Fatalf("failed later statement was published: %#v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestObserverRejectsUnsupportedVirtualTable(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "virtual.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE VIRTUAL TABLE docs USING fts5(content)`) + if err != nil { + t.Skipf("sqlite build has no fts5 virtual table: %v", err) + } + _, err = observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"docs"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "virtual table") +} + +func TestObserverFailsClosedWhenVirtualTableIsCreatedDynamically(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "virtual-dynamic.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"items"}}) + require.NoError(t, err) + _, err = observed.opened.DB.Exec(`CREATE VIRTUAL TABLE docs USING fts5(content)`) + if err != nil { + t.Skipf("sqlite build has no fts5 virtual table: %v", err) + } + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for dynamic virtual table") + } + assert.Contains(t, stream.Err().Error(), "virtual table") +} + +func TestObserverDetectsDDLThroughAuthorizerWithLeadingComment(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "ddl-comment.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + tx, err := observed.opened.DB.Begin() + require.NoError(t, err) + _, err = tx.Exec(`/* CREATE TABLE */ CREATE TABLE other (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'value')`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("observer did not fail closed for DDL and DML transaction") + } + assert.Error(t, stream.Err()) +} + +func receiveBatch(t *testing.T, stream interface { + Changes() <-chan config.MutationBatch +}) config.MutationBatch { + t.Helper() + select { + case batch := <-stream.Changes(): + return batch + case <-time.After(time.Second): + t.Fatal("timed out waiting for mutation batch") + return config.MutationBatch{} + } +} diff --git a/service/sql/engine/sqlite/sqlite.go b/service/sql/engine/sqlite/sqlite.go index 20da5a918..68aadcdcb 100644 --- a/service/sql/engine/sqlite/sqlite.go +++ b/service/sql/engine/sqlite/sqlite.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MPL-2.0 -// Package sqlite implements the file-backed SQLite engine. It registers itself with -// service/sql through the public engine seam; a CDC-enabled build overrides the -// underlying driver via service/sql.RegisterDriver, which core applies transparently. +// Package sqlite implements the file-backed SQLite SQL driver. It is explicitly +// constructed by the boot graph and owns the connector/connection lifecycle for +// each pool generation. package sqlite import ( @@ -17,15 +17,18 @@ import ( entryutil "github.com/wippyai/runtime/system/entry" ) -// defaultDriver is the stock SQLite driver. A build with the preupdate hook overrides -// it via service/sql.RegisterDriver; the engine itself stays override-agnostic. +// defaultDriver is retained for diagnostics and config validation. Physical +// opens use the connector-owned driver in observer.go when the preupdate build +// tag is enabled, so no process-global driver name is replaced. const defaultDriver = "sqlite3" type engine struct{} -func init() { - sqlservice.RegisterEngine(engine{}) -} +// NewDriver returns a SQLite SQL driver. The concrete return keeps the +// connector-owned Open capability available to composition/integration code; +// it remains assignable to service/sql.Driver wherever only the base engine +// contract is needed. +func NewDriver() engine { return engine{} } func (engine) Kind() registry.Kind { return config.SQLite @@ -48,6 +51,24 @@ func (engine) ResolveEnv(context.Context, sqlservice.EngineDeps, config.EngineCo return nil } +func (engine) Open(ctx context.Context, ec config.EngineConfig) (sqlservice.OpenedDB, error) { + dsn, err := engine{}.BuildDSN(ec) + if err != nil { + return sqlservice.OpenedDB{}, err + } + + cfg, ok := ec.(*config.SQLiteConfig) + if !ok { + return sqlservice.OpenedDB{}, sqlservice.NewInvalidConfigTypeError(fmt.Sprintf("%T", ec), config.SQLite) + } + db, observer, err := openSQLite(ctx, dsn, cfg.MaxMutationChanges, cfg.MaxMutationBytes) + if err != nil { + return sqlservice.OpenedDB{}, sqlservice.NewConnectionPoolCreationError(err) + } + + return sqlservice.OpenedDB{DB: db, Observer: observer}, nil +} + func (engine) BuildDSN(ec config.EngineConfig) (string, error) { cfg, ok := ec.(*config.SQLiteConfig) if !ok { diff --git a/service/sql/engine/sqlite/sqlite_test.go b/service/sql/engine/sqlite/sqlite_test.go index a5223cf8d..0cf2a616c 100644 --- a/service/sql/engine/sqlite/sqlite_test.go +++ b/service/sql/engine/sqlite/sqlite_test.go @@ -23,7 +23,7 @@ func TestKindDriverRegistered(t *testing.T) { assert.Equal(t, config.SQLite, e.Kind()) assert.Equal(t, "sqlite3", e.DriverName()) - _, _, err := (&sqlservice.DefaultPoolFactory{}).CreatePool( + _, _, err := sqlservice.NewDefaultPoolFactory(NewDriver()).CreatePool( context.Background(), sqlservice.EngineDeps{Log: zap.NewNop()}, registry.Entry{ID: registry.NewID("t", "x"), Kind: config.SQLite, Data: nil}, diff --git a/service/sql/engine/standard/standard.go b/service/sql/engine/standard/standard.go index 936b13976..70f6fd1ac 100644 --- a/service/sql/engine/standard/standard.go +++ b/service/sql/engine/standard/standard.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MPL-2.0 -// Package standard implements the network SQL engines (PostgreSQL and MySQL) that -// share DBConfig. It registers itself with service/sql through the public engine -// seam, so the core package carries no knowledge of these dialects. +// Package standard implements the network SQL drivers (PostgreSQL and MySQL) +// that share DBConfig. Drivers are constructed explicitly by the boot graph; +// importing this package does not mutate process-global SQL state. package standard import ( @@ -31,9 +31,14 @@ type engine struct { driver string } -func init() { - sqlservice.RegisterEngine(engine{kind: config.Postgres, driver: "postgres", dsn: buildPostgresDSN}) - sqlservice.RegisterEngine(engine{kind: config.MySQL, driver: "mysql", dsn: buildMySQLDSN}) +// NewPostgresDriver returns the PostgreSQL SQL driver. +func NewPostgresDriver() sqlservice.Driver { + return engine{kind: config.Postgres, driver: "postgres", dsn: buildPostgresDSN} +} + +// NewMySQLDriver returns the MySQL SQL driver. +func NewMySQLDriver() sqlservice.Driver { + return engine{kind: config.MySQL, driver: "mysql", dsn: buildMySQLDSN} } func (e engine) Kind() registry.Kind { @@ -57,6 +62,20 @@ func (engine) ResolveEnv(context.Context, sqlservice.EngineDeps, config.EngineCo return nil } +func (e engine) Open(_ context.Context, ec config.EngineConfig) (sqlservice.OpenedDB, error) { + dsn, err := e.BuildDSN(ec) + if err != nil { + return sqlservice.OpenedDB{}, err + } + + db, err := sql.Open(e.driver, dsn) + if err != nil { + return sqlservice.OpenedDB{}, sqlservice.NewConnectionPoolCreationError(err) + } + + return sqlservice.OpenedDB{DB: db}, nil +} + func (e engine) BuildDSN(ec config.EngineConfig) (string, error) { cfg, ok := ec.(*config.DBConfig) if !ok { diff --git a/service/sql/engine/standard/standard_test.go b/service/sql/engine/standard/standard_test.go index 8ce3946b7..bb980f8df 100644 --- a/service/sql/engine/standard/standard_test.go +++ b/service/sql/engine/standard/standard_test.go @@ -22,14 +22,14 @@ func sqlOpenMemory(*testing.T) (*sql.DB, error) { } func TestRegistered(t *testing.T) { - for _, k := range []registry.Kind{config.Postgres, config.MySQL} { - _, _, err := (&sqlservice.DefaultPoolFactory{}).CreatePool( + for _, driver := range []sqlservice.Driver{NewPostgresDriver(), NewMySQLDriver()} { + _, _, err := sqlservice.NewDefaultPoolFactory(driver).CreatePool( context.Background(), sqlservice.EngineDeps{Log: zap.NewNop()}, - registry.Entry{ID: registry.NewID("t", "x"), Kind: k, Data: nil}, + registry.Entry{ID: registry.NewID("t", "x"), Kind: driver.Kind(), Data: nil}, ) require.Error(t, err) - assert.NotContains(t, err.Error(), "unsupported entry kind", "engine %s must be registered", k) + assert.NotContains(t, err.Error(), "unsupported entry kind", "driver %s must be accepted", driver.Kind()) } } diff --git a/service/sql/engines_stub_test.go b/service/sql/engines_stub_test.go index 73bb12451..99efc414a 100644 --- a/service/sql/engines_stub_test.go +++ b/service/sql/engines_stub_test.go @@ -24,10 +24,21 @@ type stubEngine struct { isSQLite bool } -func init() { - RegisterEngine(stubEngine{kind: config.Postgres, driver: "postgres"}) - RegisterEngine(stubEngine{kind: config.MySQL, driver: "mysql"}) - RegisterEngine(stubEngine{kind: config.SQLite, driver: "sqlite3", isSQLite: true}) +func testDrivers() []Driver { + return []Driver{ + stubEngine{kind: config.Postgres, driver: "postgres"}, + stubEngine{kind: config.MySQL, driver: "mysql"}, + stubEngine{kind: config.SQLite, driver: "sqlite3", isSQLite: true}, + } +} + +func testDriverFor(kind registry.Kind) (Driver, bool) { + for _, driver := range testDrivers() { + if driver.Kind() == kind { + return driver, true + } + } + return nil, false } func (e stubEngine) Kind() registry.Kind { @@ -58,6 +69,18 @@ func (stubEngine) ResolveEnv(context.Context, EngineDeps, config.EngineConfig) e return nil } +func (e stubEngine) Open(_ context.Context, ec config.EngineConfig) (OpenedDB, error) { + dsn, err := e.BuildDSN(ec) + if err != nil { + return OpenedDB{}, err + } + db, err := sql.Open(e.driver, dsn) + if err != nil { + return OpenedDB{}, err + } + return OpenedDB{DB: db}, nil +} + func (e stubEngine) BuildDSN(config.EngineConfig) (string, error) { if e.isSQLite { return ":memory:", nil diff --git a/service/sql/factory.go b/service/sql/factory.go index 3789a26c6..1cc2232f1 100644 --- a/service/sql/factory.go +++ b/service/sql/factory.go @@ -16,30 +16,39 @@ type Factory interface { UpdatePool(ctx context.Context, deps EngineDeps, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) } -// DefaultPoolFactory is the registry-backed Factory used in production. -type DefaultPoolFactory struct{} +// DefaultPoolFactory dispatches entries to the drivers supplied at +// construction. It deliberately has no package-global registry. +type DefaultPoolFactory struct { + drivers map[registry.Kind]Driver +} -// NewDefaultPoolFactory creates a new default pool factory. -func NewDefaultPoolFactory() Factory { - return &DefaultPoolFactory{} +// NewDefaultPoolFactory creates a pool factory with the supplied drivers. +func NewDefaultPoolFactory(drivers ...Driver) Factory { + registered := make(map[registry.Kind]Driver, len(drivers)) + for _, driver := range drivers { + if driver != nil { + registered[driver.Kind()] = driver + } + } + return &DefaultPoolFactory{drivers: registered} } // CreatePool implements Factory.CreatePool. func (f *DefaultPoolFactory) CreatePool(ctx context.Context, deps EngineDeps, entry registry.Entry) (*ConnPool, config.EngineConfig, error) { - eng, ok := engineFor(entry.Kind) + driver, ok := f.drivers[entry.Kind] if !ok { return nil, nil, NewUnsupportedEntryKindError(entry.Kind) } - return createPool(ctx, deps, eng, entry) + return createPool(ctx, deps, driver, entry) } // UpdatePool implements Factory.UpdatePool. func (f *DefaultPoolFactory) UpdatePool(ctx context.Context, deps EngineDeps, pool *ConnPool, entry registry.Entry) (config.EngineConfig, error) { - eng, ok := engineFor(entry.Kind) + driver, ok := f.drivers[entry.Kind] if !ok { return nil, NewUnsupportedEntryKindError(entry.Kind) } - return updatePool(ctx, deps, eng, pool, entry) + return updatePool(ctx, deps, driver, pool, entry) } diff --git a/service/sql/factory_test.go b/service/sql/factory_test.go index d8cc20775..92f667618 100644 --- a/service/sql/factory_test.go +++ b/service/sql/factory_test.go @@ -71,7 +71,7 @@ func depsFor(cfg any) EngineDeps { // TestDefaultPoolFactory_CreatePoolValidation tests pool creation validation through // the registry-backed factory. func TestDefaultPoolFactory_CreatePoolValidation(t *testing.T) { - factory := &DefaultPoolFactory{} + factory := NewDefaultPoolFactory(testDrivers()...) tests := []struct { cfg any @@ -155,7 +155,7 @@ func TestDefaultPoolFactory_CreatePoolSQLiteSuccess(t *testing.T) { } entry := registry.Entry{ID: registry.NewID("test", "lite"), Kind: config.SQLite, Data: payload.New("x")} - pool, ec, err := (&DefaultPoolFactory{}).CreatePool(context.Background(), depsFor(cfg), entry) + pool, ec, err := NewDefaultPoolFactory(testDrivers()...).CreatePool(context.Background(), depsFor(cfg), entry) require.NoError(t, err) require.NotNil(t, pool) assert.Equal(t, config.SQLite, pool.kind) diff --git a/service/sql/manager.go b/service/sql/manager.go index 2836e8221..fed8ca564 100644 --- a/service/sql/manager.go +++ b/service/sql/manager.go @@ -26,14 +26,41 @@ type Manager struct { mu sync.RWMutex } +// Option configures a SQL Manager. Drivers are injected at boot, matching the +// service/net composition pattern; importing a driver package has no side +// effects on other managers or pools. +type Option func(*managerOptions) + +type managerOptions struct { + drivers []Driver +} + +// WithDriver adds one or more concrete SQL drivers to the manager. +func WithDriver(drivers ...Driver) Option { + return func(opts *managerOptions) { + for _, driver := range drivers { + if driver != nil { + opts.drivers = append(opts.drivers, driver) + } + } + } +} + // NewManager creates a new SQL service manager func NewManager( dtt payload.Transcoder, bus event.Bus, log *zap.Logger, envRegistry envapi.Registry, + opts ...Option, ) (*Manager, error) { - return NewManagerWithFactory(dtt, bus, log, envRegistry, NewDefaultPoolFactory()) + var options managerOptions + for _, opt := range opts { + if opt != nil { + opt(&options) + } + } + return NewManagerWithFactory(dtt, bus, log, envRegistry, NewDefaultPoolFactory(options.drivers...)) } // NewManagerWithFactory creates a new SQL service manager with the specified pool factory @@ -77,10 +104,6 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { m.mu.Lock() defer m.mu.Unlock() - if _, ok := engineFor(entry.Kind); !ok { - return NewUnsupportedEntryKindError(entry.Kind) - } - if _, exists := m.services[entry.ID]; exists { return NewServiceExistsError(entry.ID) } @@ -98,10 +121,6 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { m.mu.Lock() defer m.mu.Unlock() - if _, ok := engineFor(entry.Kind); !ok { - return NewUnsupportedEntryKindError(entry.Kind) - } - pool, exists := m.services[entry.ID] if !exists { return NewServiceNotFoundError(entry.ID) diff --git a/service/sql/manager_test.go b/service/sql/manager_test.go index 7f738d947..6de8065da 100644 --- a/service/sql/manager_test.go +++ b/service/sql/manager_test.go @@ -104,6 +104,7 @@ func NewMockConnPool(kind registry.Kind) *ConnPool { pool := &ConnPool{ kind: kind, db: db, + driver: func() Driver { d, _ := testDriverFor(kind); return d }(), status: make(chan any, 1), closed: atomic.Bool{}, } @@ -172,7 +173,7 @@ func (f *TestPoolFactory) CreatePool(ctx context.Context, deps EngineDeps, entry return nil, nil, assert.AnError } - eng, ok := engineFor(entry.Kind) + eng, ok := testDriverFor(entry.Kind) if !ok { return nil, nil, NewUnsupportedEntryKindError(entry.Kind) } @@ -204,7 +205,7 @@ func (f *TestPoolFactory) UpdatePool(ctx context.Context, deps EngineDeps, pool return nil, assert.AnError } - eng, ok := engineFor(entry.Kind) + eng, ok := testDriverFor(entry.Kind) if !ok { return nil, NewUnsupportedEntryKindError(entry.Kind) } diff --git a/test.sh b/test.sh index 20746c9ba..ff5266e79 100755 --- a/test.sh +++ b/test.sh @@ -41,8 +41,8 @@ go test \ ./boot/... \ ./system/registry/... -echo "running sqlite cdc integration tests (local temp file, no docker)" -CGO_ENABLED=1 go test -tags "integration sqlite_preupdate_hook" ./service/cdc/sqlite +echo "running sqlite cdc implementation and integration tests (local temp file, no docker)" +make test-cdc-sqlite if [[ -n "${WIPPY_CDC_IT_REPL_DSN:-}" && -n "${WIPPY_CDC_IT_ADMIN_DSN:-}" ]]; then go test -tags integration ./service/cdc/postgres From 0a4a6b8cadf750d2eac2e8e9b492a96bbd20257f Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 00:57:33 -0400 Subject: [PATCH 16/47] fix(lua): align CDC registry test fields --- runtime/lua/modules/cdc/module_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/lua/modules/cdc/module_test.go b/runtime/lua/modules/cdc/module_test.go index 07b0a7deb..897bdf581 100644 --- a/runtime/lua/modules/cdc/module_test.go +++ b/runtime/lua/modules/cdc/module_test.go @@ -58,8 +58,8 @@ func (f *fakeSource) Subscribe(context.Context, cdcapi.StreamOptions) (cdcapi.St } type fakeRegistry struct { - all []cdcapi.SourceInfo source cdcapi.Source + all []cdcapi.SourceInfo } func (f *fakeRegistry) List() []cdcapi.SourceInfo { return f.all } From ed54fbd38bd4c33b1863d39956c473cddbdf5df9 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:00:51 -0400 Subject: [PATCH 17/47] chore(cdc/sqlite): clean up lint findings --- service/cdc/sqlite/driver.go | 2 +- service/cdc/sqlite/integration_live_test.go | 5 +-- service/cdc/sqlite/source.go | 43 ++++++++++----------- service/cdc/sqlite/source_test.go | 23 ++++------- service/cdc/sqlite/subscribers.go | 9 ++--- 5 files changed, 34 insertions(+), 48 deletions(-) diff --git a/service/cdc/sqlite/driver.go b/service/cdc/sqlite/driver.go index dca05dbeb..7010a2e18 100644 --- a/service/cdc/sqlite/driver.go +++ b/service/cdc/sqlite/driver.go @@ -32,8 +32,8 @@ type sourceOptions struct { name string statusInterval string tables []string - snapshot bool lifecycle supervisor.LifecycleConfig + snapshot bool } // Driver wires the SQLite CDC implementation into the driver-neutral CDC diff --git a/service/cdc/sqlite/integration_live_test.go b/service/cdc/sqlite/integration_live_test.go index de870f920..3414b7ef7 100644 --- a/service/cdc/sqlite/integration_live_test.go +++ b/service/cdc/sqlite/integration_live_test.go @@ -69,10 +69,7 @@ func requireNoChange(t *testing.T, stream cdcapi.Stream) { select { case change, ok := <-stream.Changes(): if !ok { - if errStream, isErrStream := stream.(cdcapi.ErrStream); isErrStream { - t.Fatalf("stream closed unexpectedly: %v", errStream.Err()) - } - t.Fatal("stream closed unexpectedly") + t.Fatalf("stream closed unexpectedly: %v", stream.Err()) } t.Fatalf("unexpected CDC change: %#v", change) case <-time.After(100 * time.Millisecond): diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index 91de056cf..f7fb80b75 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -20,7 +20,6 @@ import ( "github.com/wippyai/runtime/api/resource" config "github.com/wippyai/runtime/api/service/cdc" sqlapi "github.com/wippyai/runtime/api/service/sql" - sqlconfig "github.com/wippyai/runtime/api/service/sql" "github.com/wippyai/runtime/api/supervisor" sqlservice "github.com/wippyai/runtime/service/sql" ) @@ -36,31 +35,29 @@ const ( // the resource long enough to subscribe to its committed-mutation capability; // it never opens another connection and never installs hooks on a raw one. type Source struct { - res resource.Registry - log *zap.Logger - id registry.ID - name string - dbResID registry.ID - tables []string - statusTick time.Duration - lifecycle configLifecycle - snapshot bool - - subs *subscribers - - mu sync.RWMutex - state config.SourceState - generation string sourceErr error - observer sqlapi.MutationStream + res resource.Registry observerSource sqlapi.CommittedMutationSource - snapshotSubs map[*subscription]sqlapi.MutationStream - snapshotWG sync.WaitGroup - startDone chan struct{} + observer sqlapi.MutationStream runDone chan struct{} - startCancel context.CancelFunc runCancel context.CancelFunc status chan any + startCancel context.CancelFunc + subs *subscribers + log *zap.Logger + startDone chan struct{} + snapshotSubs map[*subscription]sqlapi.MutationStream + id registry.ID + dbResID registry.ID + name string + generation string + state config.SourceState + tables []string + lifecycle configLifecycle + snapshotWG sync.WaitGroup + statusTick time.Duration + mu sync.RWMutex + snapshot bool statusClosed bool stopping bool } @@ -269,7 +266,7 @@ func (s *Source) acquireObserver(ctx context.Context) (sqlapi.CommittedMutationS if !ok { return nil, fmt.Errorf("resource %s is not a database", s.name) } - if db.Type != sqlconfig.SQLite { + if db.Type != sqlapi.SQLite { return nil, fmt.Errorf("resource %s is not a sqlite database (kind %s)", s.name, db.Type) } if db.Observer == nil { @@ -342,7 +339,7 @@ func (s *Source) processBatch(batch sqlapi.MutationBatch, collector metrics.Coll func (s *Source) changeFromMutation(batch sqlapi.MutationBatch, index int, mutation sqlapi.Mutation) (config.Change, error) { op := strings.ToLower(strings.TrimSpace(mutation.Op)) - if op != "insert" && op != "update" && op != "delete" && !(batch.Snapshot && op == "snapshot") { + if op != "insert" && op != "update" && op != "delete" && (!batch.Snapshot || op != "snapshot") { return config.Change{}, fmt.Errorf("sqlite mutation observer emitted unsupported operation %q", mutation.Op) } if mutation.Table == "" { diff --git a/service/cdc/sqlite/source_test.go b/service/cdc/sqlite/source_test.go index d1b735fd6..27e898fc5 100644 --- a/service/cdc/sqlite/source_test.go +++ b/service/cdc/sqlite/source_test.go @@ -19,7 +19,6 @@ import ( "github.com/wippyai/runtime/api/resource" cdcapi "github.com/wippyai/runtime/api/service/cdc" sqlapi "github.com/wippyai/runtime/api/service/sql" - sqlconfig "github.com/wippyai/runtime/api/service/sql" sqlservice "github.com/wippyai/runtime/service/sql" ) @@ -30,7 +29,7 @@ type testResourceRegistry struct { func (r *testResourceRegistry) Acquire(context.Context, registry.ID, resource.AccessMode) (resource.Resource[any], error) { return &testDBResource{owner: r, value: sqlservice.DBResource{ - Type: sqlconfig.SQLite, + Type: sqlapi.SQLite, Observer: r.observer, }}, nil } @@ -50,12 +49,12 @@ func (r *testDBResource) Release() { } type testObserver struct { - mu sync.Mutex stream *testMutationStream snapshot *testSnapshotStream - closed bool - closeN atomic.Int32 subOpts sqlapi.MutationOptions + mu sync.Mutex + closeN atomic.Int32 + closed bool } func (o *testObserver) Subscribe(ctx context.Context, opts sqlapi.MutationOptions) (sqlapi.MutationStream, error) { @@ -127,11 +126,11 @@ func (o *testObserver) currentSnapshot(t *testing.T) *testSnapshotStream { } type testMutationStream struct { + err error changes chan sqlapi.MutationBatch mu sync.Mutex - err error - closed bool closeN atomic.Int32 + closed bool } type testSnapshotStream struct { @@ -202,10 +201,7 @@ func receiveChange(t *testing.T, stream cdcapi.Stream) cdcapi.Change { select { case change, ok := <-stream.Changes(): if !ok { - if errStream, isErrStream := stream.(cdcapi.ErrStream); isErrStream { - require.Failf(t, "snapshot/live stream closed", "stream error: %v", errStream.Err()) - } - require.Fail(t, "snapshot/live stream closed") + require.Failf(t, "snapshot/live stream closed", "stream error: %v", stream.Err()) } return change case <-time.After(time.Second): @@ -233,10 +229,7 @@ func waitStreamClosed(t *testing.T, stream cdcapi.Stream) error { select { case _, ok := <-stream.Changes(): if !ok { - if errStream, ok := stream.(cdcapi.ErrStream); ok { - return errStream.Err() - } - return nil + return stream.Err() } case <-deadline: t.Fatal("timed out waiting for SQLite CDC stream close") diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go index 9b222c715..bc9470f0f 100644 --- a/service/cdc/sqlite/subscribers.go +++ b/service/cdc/sqlite/subscribers.go @@ -19,9 +19,9 @@ const ( var errSubscriberOverflow = errors.New("sqlite cdc subscriber backlog overflow") type subscribers struct { - mu sync.RWMutex m map[uint64]*subscription next uint64 + mu sync.RWMutex } func newSubscribers() *subscribers { @@ -95,6 +95,7 @@ func (s *subscribers) closeWithError(err error) { } type subscription struct { + err error parent *subscribers changes chan config.Change done chan struct{} @@ -102,14 +103,12 @@ type subscription struct { ops map[string]struct{} sourceName string id uint64 - - mu sync.Mutex - closed bool - err error + mu sync.Mutex // closedFlag lets the fan-out path reject work without taking the lock in // the common case. The lock is still held while sending/closing so a send // cannot race close(changes). closedFlag atomic.Bool + closed bool } func (s *subscription) Changes() <-chan config.Change { return s.changes } From 1e967f0f90492a0bbb713c357d740f263bd23e13 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:03:27 -0400 Subject: [PATCH 18/47] fix(cdc): align generic CDC structs --- api/service/cdc/command.go | 6 +++--- api/service/cdc/context.go | 39 +++++++++++++++++----------------- service/cdc/dispatcher.go | 24 ++++++++++----------- service/cdc/dispatcher_test.go | 4 ++-- service/cdc/manager_test.go | 14 ++++++------ service/cdc/slot.go | 31 ++++++++++----------------- service/cdc/stream.go | 4 ++-- system/cdc/registry.go | 6 +++--- 8 files changed, 58 insertions(+), 70 deletions(-) diff --git a/api/service/cdc/command.go b/api/service/cdc/command.go index 4715809ab..164e7e6a4 100644 --- a/api/service/cdc/command.go +++ b/api/service/cdc/command.go @@ -19,13 +19,13 @@ const ( ) type StreamOptions struct { + // After is an opaque source cursor. A driver that cannot resume from a + // cursor must return ErrUnsupported rather than silently ignore it. + After string Tables []string Ops []string Buffer int Snapshot bool - // After is an opaque source cursor. A driver that cannot resume from a - // cursor must return ErrUnsupported rather than silently ignore it. - After string } type Change struct { diff --git a/api/service/cdc/context.go b/api/service/cdc/context.go index f76797e8e..1f7501ba8 100644 --- a/api/service/cdc/context.go +++ b/api/service/cdc/context.go @@ -66,30 +66,29 @@ type Registry interface { } type SourceInfo struct { + // The fields in this struct are retained for wire compatibility with + // existing Lua and API consumers. New code must use ID, Kind, State, + // Capabilities and Generation; driver-specific metadata should not be + // added to the common contract. ID registry.ID `json:"id,omitempty"` + Engine string `json:"engine,omitempty"` + Epoch string `json:"epoch,omitempty"` + Error string `json:"error,omitempty"` + Generation string `json:"generation,omitempty"` + Name string `json:"name"` + Slot string `json:"slot"` + Publication string `json:"publication,omitempty"` Kind registry.Kind `json:"kind,omitempty"` State SourceState `json:"state,omitempty"` + File string `json:"file,omitempty"` + DBResource string `json:"db_resource,omitempty"` + Tables []string `json:"tables,omitempty"` Capabilities Capabilities `json:"capabilities,omitempty"` - Generation string `json:"generation,omitempty"` - - // The fields below are retained for wire compatibility with existing Lua - // and API consumers. New code must use ID, Kind, State, Capabilities and - // Generation; driver-specific metadata should not be added to the common - // contract. - Name string `json:"name"` - Slot string `json:"slot"` - Publication string `json:"publication,omitempty"` - Engine string `json:"engine,omitempty"` - File string `json:"file,omitempty"` - DBResource string `json:"db_resource,omitempty"` - Epoch string `json:"epoch,omitempty"` - Error string `json:"error,omitempty"` - Tables []string `json:"tables,omitempty"` - Streaming bool `json:"streaming,omitempty"` - Failover bool `json:"failover,omitempty"` - Temporary bool `json:"temporary,omitempty"` - Snapshot bool `json:"snapshot,omitempty"` - Faulted bool `json:"faulted,omitempty"` + Streaming bool `json:"streaming,omitempty"` + Failover bool `json:"failover,omitempty"` + Temporary bool `json:"temporary,omitempty"` + Snapshot bool `json:"snapshot,omitempty"` + Faulted bool `json:"faulted,omitempty"` } type SourceInspector interface { diff --git a/service/cdc/dispatcher.go b/service/cdc/dispatcher.go index 99afd1d5a..3d379bba0 100644 --- a/service/cdc/dispatcher.go +++ b/service/cdc/dispatcher.go @@ -51,20 +51,18 @@ const ( // configured source manager. The dispatcher owns subscription relays; a // driver owns the source and its stream implementation. type Dispatcher struct { - workers int - log *zap.Logger - - mu sync.Mutex - state dispatcherState - ctx context.Context - cancel context.CancelFunc - jobs chan dispatchJob - sessions map[uint64]*relaySession - nextID uint64 - stopDone chan struct{} - + ctx context.Context + log *zap.Logger + cancel context.CancelFunc + jobs chan dispatchJob + sessions map[uint64]*relaySession + stopDone chan struct{} workersWG sync.WaitGroup relaysWG sync.WaitGroup + workers int + nextID uint64 + mu sync.Mutex + state dispatcherState } type dispatchJob struct { @@ -442,9 +440,9 @@ func complete(receiver dispatcher.ResultReceiver, tag uint64, data any, err erro } type relaySession struct { - id uint64 cancel context.CancelFunc close func() + id uint64 once sync.Once } diff --git a/service/cdc/dispatcher_test.go b/service/cdc/dispatcher_test.go index e14eacd27..18ed33174 100644 --- a/service/cdc/dispatcher_test.go +++ b/service/cdc/dispatcher_test.go @@ -94,10 +94,10 @@ func (n *dispatcherTestNode) Attach(pid.PID, chan *relay.Package) (context.Cance func (n *dispatcherTestNode) Detach(pid.PID) {} type dispatcherTestReceiver struct { - done chan struct{} - once sync.Once data any err error + done chan struct{} + once sync.Once } func (r *dispatcherTestReceiver) CompleteYield(_ uint64, data any, err error) { diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go index 97dad50f5..2482decc4 100644 --- a/service/cdc/manager_test.go +++ b/service/cdc/manager_test.go @@ -28,22 +28,22 @@ func (s *testStream) Close() { close(s.changes) } func (s *testStream) Err() error { return nil } type managedTestSource struct { - info api.SourceInfo startErr error stopErr error - exclusive string stream *testStream + lifecycle *supervisor.LifecycleConfig + exclusive string + info api.SourceInfo startCount atomic.Int32 stopCount atomic.Int32 active atomic.Int32 maxActive atomic.Int32 - lifecycle *supervisor.LifecycleConfig } type disposableTestSource struct { + disposeErr error *managedTestSource disposeCount atomic.Int32 - disposeErr error failedOnce atomic.Bool } @@ -96,8 +96,8 @@ func (s *managedTestSource) LifecycleConfig() supervisor.LifecycleConfig { func (s *managedTestSource) ExclusiveResourceKey() string { return s.exclusive } type testDriver struct { - kind registry.Kind create func(registry.Entry) (ManagedSource, error) + kind registry.Kind } func (d testDriver) Kind() registry.Kind { return d.kind } @@ -107,8 +107,8 @@ func (d testDriver) Create(_ context.Context, entry registry.Entry, _ Dependenci } type recordingBus struct { - mu sync.Mutex events []event.Event + mu sync.Mutex } func (b *recordingBus) Subscribe(context.Context, event.System, chan<- event.Event) (event.SubscriberID, error) { @@ -174,7 +174,7 @@ func TestManagerRoutesCanonicalIDsAndOwnsLifecycle(t *testing.T) { require.Same(t, created[0], slot.current) slot.mu.RUnlock() require.Equal(t, "app:events", m.List()[0].ID.String()) - require.Equal(t, registry.Kind(driver.kind), m.List()[0].Kind) + require.Equal(t, driver.kind, m.List()[0].Kind) require.ErrorIs(t, m.Add(context.Background(), entry), ErrSourceExists) require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: registry.ParseID("app:events")})) diff --git a/service/cdc/slot.go b/service/cdc/slot.go index f85612af0..6162a80cf 100644 --- a/service/cdc/slot.go +++ b/service/cdc/slot.go @@ -34,24 +34,22 @@ const ( // supervisor. A driver replacement changes the delegated generation, never the // supervisor object or registry pointer. type sourceSlot struct { - id registry.ID - kind registry.Kind - - opMu sync.Mutex - mu sync.RWMutex - + runCtx context.Context current ManagedSource + runCancel context.CancelFunc log *zap.Logger + status chan any + retiredHook func(string, uint64) + id registry.ID + kind registry.Kind + retired []retiredSource generation uint64 + mu sync.RWMutex + opMu sync.Mutex state slotState - runCtx context.Context - runCancel context.CancelFunc - status chan any - statusDone bool - replacing bool disposing bool - retired []retiredSource - retiredHook func(string, uint64) + replacing bool + statusDone bool } type retiredSource struct { @@ -619,13 +617,6 @@ func exclusiveResourceKey(source ManagedSource) string { return "" } -func (s *sourceSlot) currentGeneration() uint64 { - s.mu.RLock() - generation := s.generation - s.mu.RUnlock() - return generation -} - func (s *sourceSlot) currentSource() ManagedSource { s.mu.RLock() source := s.current diff --git a/service/cdc/stream.go b/service/cdc/stream.go index 4b2adda51..a99bf0ee2 100644 --- a/service/cdc/stream.go +++ b/service/cdc/stream.go @@ -20,10 +20,10 @@ const ( // resume diagnostics. type stampedStream struct { upstream api.Stream - sourceID registry.ID - generation string out chan api.Change done chan struct{} + sourceID registry.ID + generation string once sync.Once } diff --git a/system/cdc/registry.go b/system/cdc/registry.go index 8b7405cc8..e7ab3f72b 100644 --- a/system/cdc/registry.go +++ b/system/cdc/registry.go @@ -31,8 +31,8 @@ type entry struct { // process-wide aliases such as PostgreSQL slot names into this layer. type Registry struct { log *zap.Logger - mu sync.RWMutex sources map[registry.ID]entry + mu sync.RWMutex } func NewRegistry(log *zap.Logger) *Registry { @@ -119,15 +119,15 @@ func (r *Registry) Get(id registry.ID) (api.Source, bool) { func (r *Registry) List() []api.SourceInfo { r.mu.RLock() items := make([]struct { + source api.Source id registry.ID kind registry.Kind - source api.Source }, 0, len(r.sources)) for id, item := range r.sources { items = append(items, struct { + source api.Source id registry.ID kind registry.Kind - source api.Source }{id: id, kind: item.kind, source: item.source}) } r.mu.RUnlock() From aca095f8fac69aa798ba3c9d8bd442b8549aaa1c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:03:29 -0400 Subject: [PATCH 19/47] fix(cdc/postgres): satisfy lifecycle and layout lint --- service/cdc/postgres/decoder.go | 12 ++------- service/cdc/postgres/decoder_stream_test.go | 2 +- service/cdc/postgres/driver.go | 4 +-- service/cdc/postgres/manager.go | 1 + service/cdc/postgres/service.go | 28 +++++++++++---------- service/cdc/postgres/stream.go | 2 +- 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/service/cdc/postgres/decoder.go b/service/cdc/postgres/decoder.go index 87892d1f8..71765fc1d 100644 --- a/service/cdc/postgres/decoder.go +++ b/service/cdc/postgres/decoder.go @@ -26,14 +26,14 @@ type decodeResult struct { type decoder struct { rels *relationCache buffer map[uint32][]bufferedChange + usage map[uint32]int64 + limits decoderLimits commitLSN pglogrepl.LSN xid uint32 curTopXid uint32 streaming bool inStream bool txActive bool - limits decoderLimits - usage map[uint32]int64 } func newDecoder(limits ...decoderLimits) *decoder { @@ -58,14 +58,6 @@ func newDecoderWithMode(streaming bool, limits ...decoderLimits) *decoder { } } -func (d *decoder) decode(walData []byte, walStart pglogrepl.LSN) ([]RowChange, error) { - result, err := d.decodeResult(walData, walStart) - if err != nil { - return nil, err - } - return result.changes, nil -} - func (d *decoder) decodeResult(walData []byte, walStart pglogrepl.LSN) (decodeResult, error) { var ( msg pglogrepl.Message diff --git a/service/cdc/postgres/decoder_stream_test.go b/service/cdc/postgres/decoder_stream_test.go index dc7127e94..9b11bf81e 100644 --- a/service/cdc/postgres/decoder_stream_test.go +++ b/service/cdc/postgres/decoder_stream_test.go @@ -47,8 +47,8 @@ func TestStreamingDecoderBuffersUntilCommit(t *testing.T) { func TestStreamingDecoderRequiresStopBeforeCommitOrAbort(t *testing.T) { for _, tc := range []struct { - name string msg pglogrepl.Message + name string }{ {name: "commit", msg: &pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x99}}, {name: "abort", msg: &pglogrepl.StreamAbortMessageV2{Xid: 100, SubXid: 100}}, diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index 2ad4242ce..d81dcfc8d 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -81,10 +81,10 @@ func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice. // remains private to this adapter, so no driver-specific shape leaks into the // common manager or dispatcher. type sourceAdapter struct { - mu sync.RWMutex source *Source - lifecycle supervisor.LifecycleConfig exclusiveKey string + lifecycle supervisor.LifecycleConfig + mu sync.RWMutex } func (s *sourceAdapter) Info() config.SourceInfo { diff --git a/service/cdc/postgres/manager.go b/service/cdc/postgres/manager.go index 30228a0cc..ce6b75cd3 100644 --- a/service/cdc/postgres/manager.go +++ b/service/cdc/postgres/manager.go @@ -22,6 +22,7 @@ import ( ) // Manager is the legacy PostgreSQL-specific registry and lifecycle wrapper. +// // Deprecated: use service/cdc.Manager with NewDriver so source identity and // lifecycle are owned by the driver-neutral CDC manager. type Manager struct { diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 7d4b91eab..3c0cb04d2 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -64,9 +64,10 @@ type SourceOptions struct { } type Source struct { - log *zap.Logger coll metrics.Collector injectedCP Checkpointer + sourceErr error + log *zap.Logger cancel context.CancelFunc done chan struct{} subs map[uint64]*sourceSubscription @@ -78,22 +79,21 @@ type Source struct { tables []string standbyInterval time.Duration statusInterval time.Duration - mu sync.Mutex - subMu sync.RWMutex nextSubID uint64 snapshotFetchSize int + maxTransactionChanges int + maxTransactionBytes int64 + subMu sync.RWMutex + mu sync.Mutex + dropMu sync.Mutex + dropSlot atomic.Bool + dropDone atomic.Bool temporary bool snapshot bool streaming bool failover bool - maxTransactionChanges int - maxTransactionBytes int64 permanentlyClosed bool - sourceErr error state sourceState - dropSlot atomic.Bool - dropDone atomic.Bool - dropMu sync.Mutex } type sourceState uint8 @@ -206,9 +206,10 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { cancel() s.mu.Lock() if s.done == done { - if s.state == sourceStopping { + switch s.state { + case sourceStopping: s.state = sourceStopped - } else if s.state == sourceStarting { + case sourceStarting: s.state = sourceFailed s.sourceErr = startErr } @@ -383,9 +384,10 @@ func (s *Source) run( defer func() { s.mu.Lock() if s.done == done { - if s.state == sourceStopping { + switch s.state { + case sourceStopping: s.state = sourceStopped - } else if s.state == sourceRunning || s.state == sourceStarting { + case sourceRunning, sourceStarting: s.state = sourceFailed } s.cancel = nil diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 065609306..2bf2ea697 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -23,6 +23,7 @@ const ( var errSubscriberOverflow = errors.New("postgres cdc subscriber backlog overflow") type sourceSubscription struct { + err error source *Source in chan config.Change out chan config.Change @@ -33,7 +34,6 @@ type sourceSubscription struct { once sync.Once closed atomic.Bool errMu sync.RWMutex - err error } func (s *Source) Subscribe(opts config.StreamOptions) config.Stream { From db5cb26a44d6c111ed015941dc750b457193ab85 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:05:20 -0400 Subject: [PATCH 20/47] chore(sql): satisfy observer lint contracts --- api/service/sql/observer.go | 8 +- service/sql/conn.go | 26 ++-- service/sql/engine.go | 3 +- service/sql/engine/sqlite/observer.go | 83 +++++++------ service/sql/engine/sqlite/observer_test.go | 132 ++++++++++----------- 5 files changed, 123 insertions(+), 129 deletions(-) diff --git a/api/service/sql/observer.go b/api/service/sql/observer.go index ba25b72b5..5881f44fc 100644 --- a/api/service/sql/observer.go +++ b/api/service/sql/observer.go @@ -62,8 +62,8 @@ type SnapshotStream interface { // belongs to one database transaction; an empty batch is not emitted. type MutationBatch struct { Transaction string - Snapshot bool Changes []Mutation + Snapshot bool } // Mutation is a driver-neutral row mutation. Values retain the database/sql @@ -72,14 +72,14 @@ type MutationBatch struct { type Mutation struct { Schema string Table string + Op string Columns []string + Before []any + After []any // OldRowID is the row identifier before the change. It is zero for an // insert; RowID is the identifier after the change and is zero for a // delete. Drivers that cannot provide a stable row identifier must fail // closed rather than emit an ambiguous mutation. OldRowID int64 RowID int64 - Before []any - After []any - Op string } diff --git a/service/sql/conn.go b/service/sql/conn.go index 2237186fe..34c3a39f9 100644 --- a/service/sql/conn.go +++ b/service/sql/conn.go @@ -8,8 +8,6 @@ import ( "sync" "sync/atomic" - config "github.com/wippyai/runtime/api/service/sql" - "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/resource" sqlapi "github.com/wippyai/runtime/api/service/sql" @@ -18,30 +16,30 @@ import ( // ConnPool represents a database connection pool that acts both as a service // and a resource provider type ConnPool struct { - db *sql.DB + driver Driver + stopErr error + stopDone chan struct{} current *dbGeneration status chan any config atomic.Pointer[any] + db *sql.DB kind registry.Kind - driver Driver - mu sync.RWMutex wg sync.WaitGroup - closed atomic.Bool + mu sync.RWMutex stopMu sync.Mutex - stopDone chan struct{} - stopErr error + closed atomic.Bool stopStarted bool } type dbGeneration struct { + closeErr error + observer sqlapi.CommittedMutationSource db *sql.DB closed chan struct{} - closeErr error - closeMu sync.Mutex once sync.Once + closeMu sync.Mutex refs atomic.Int32 closing atomic.Bool - observer sqlapi.CommittedMutationSource } func newDBGeneration(db *sql.DB, observers ...sqlapi.CommittedMutationSource) *dbGeneration { @@ -213,7 +211,7 @@ func (p *ConnPool) UpdateConfig(cfg any) error { return ErrPoolClosed } - ec, ok := cfg.(config.EngineConfig) + ec, ok := cfg.(sqlapi.EngineConfig) if !ok { return NewUnsupportedConfigTypeError(p.kind) } @@ -225,7 +223,7 @@ func (p *ConnPool) UpdateConfig(cfg any) error { return p.updateConfig(context.Background(), p.driver, ec) } -func (p *ConnPool) updateConfig(ctx context.Context, driver Driver, ec config.EngineConfig) error { +func (p *ConnPool) updateConfig(ctx context.Context, driver Driver, ec sqlapi.EngineConfig) error { if p.closed.Load() { return ErrPoolClosed } @@ -323,9 +321,9 @@ type DBConn struct { // DBResource contains both the database connection and its type type DBResource struct { + Observer sqlapi.CommittedMutationSource DB *sql.DB // The database connection Type registry.Kind // The database type (postgres, mysql, sqlite, etc.) - Observer sqlapi.CommittedMutationSource } // newDBConn creates a new database resource diff --git a/service/sql/engine.go b/service/sql/engine.go index 21c806ef4..46de55cf1 100644 --- a/service/sql/engine.go +++ b/service/sql/engine.go @@ -11,7 +11,6 @@ import ( "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/registry" config "github.com/wippyai/runtime/api/service/sql" - sqlapi "github.com/wippyai/runtime/api/service/sql" "go.uber.org/zap" ) @@ -56,7 +55,7 @@ type DBOpener interface { // cannot be accidentally shared between unrelated pool generations. type OpenedDB struct { DB *sql.DB - Observer sqlapi.CommittedMutationSource + Observer config.CommittedMutationSource } // createPool runs the generic create lifecycle for a known engine. diff --git a/service/sql/engine/sqlite/observer.go b/service/sql/engine/sqlite/observer.go index d6cd53300..be6c913d4 100644 --- a/service/sql/engine/sqlite/observer.go +++ b/service/sql/engine/sqlite/observer.go @@ -40,9 +40,9 @@ const ( // one SQL pool. It intentionally uses sql.OpenDB instead of sql.Register, so // no process-global driver name or file-path registry is involved. type sqliteConnector struct { - dsn string driver *sqlite3.SQLiteDriver backend *sqliteBackend + dsn string } func (c *sqliteConnector) Connect(context.Context) (driver.Conn, error) { @@ -90,12 +90,12 @@ func openSQLite(_ context.Context, dsn string, limits ...int) (*sql.DB, sqlapi.C type sqliteBackend struct { db *sql.DB streams map[*mutationStream]struct{} - mu sync.Mutex fence chan struct{} maxChanges int maxBytes int - closed bool sequence atomic.Uint64 + mu sync.Mutex + closed bool } func newSQLiteBackend(maxChanges, maxBytes int) *sqliteBackend { @@ -222,7 +222,7 @@ func tablesForValidation(conn *sqlite3.SQLiteConn, requested []string) ([]snapsh values := make([]driver.Value, len(rows.Columns())) for { err := rows.Next(values) - if err == io.EOF { + if errors.Is(err, io.EOF) { break } if err != nil { @@ -411,7 +411,7 @@ func validateSnapshotTablesTx(ctx context.Context, tx *sql.Tx, requested []strin return errors.New("sqlite snapshot does not support TEMP tables") } var definition sql.NullString - query := fmt.Sprintf("SELECT sql FROM %s.sqlite_master WHERE type = 'table' AND name = ?", quoteIdentifier(table.schema)) + query := sqliteMasterQuery(table.schema) if err := tx.QueryRowContext(ctx, query, table.name).Scan(&definition); err != nil { return fmt.Errorf("inspect sqlite snapshot table %s.%s: %w", table.schema, table.name, err) } @@ -477,9 +477,7 @@ func scanSnapshotTable(ctx context.Context, tx *sql.Tx, stream *mutationStream, func (b *sqliteBackend) remove(stream *mutationStream, err error) { b.mu.Lock() - if _, ok := b.streams[stream]; ok { - delete(b.streams, stream) - } + delete(b.streams, stream) b.mu.Unlock() stream.closeWithError(err) @@ -554,28 +552,28 @@ func (b *sqliteBackend) Close() error { // driver wrappers after Exec/Commit/Rows completion, when statement rollback // and savepoint effects are known. type sqliteConnectionState struct { - backend *sqliteBackend + unsupported error + failed error sqlite *sqlite3.SQLiteConn - pending []sqlapi.Mutation - pendingBytes int - maxChanges int - maxBytes int - savepoints []savepoint - statementMark int + backend *sqliteBackend statementSavepointVerb string statementSavepointName string - rollbackSeen bool - rollbackUnconfirmed bool - commitPending bool + savepoints []savepoint + pending []sqlapi.Mutation commitEnds []int + prepareMeta statementMeta + statementMark int + maxBytes int + maxChanges int + pendingBytes int confirmedEnds int - fenceHeld bool + rollbackSeen bool ddlInTxn bool dmlInTxn bool - failed error + fenceHeld bool statementDDL bool - unsupported error - prepareMeta statementMeta + commitPending bool + rollbackUnconfirmed bool } // statementMeta is collected by SQLite's authorizer while a statement is @@ -584,11 +582,11 @@ type sqliteConnectionState struct { // to its later execution; direct Exec/Query paths merge it after the driver's // native prepare loop returns. type statementMeta struct { - ddl bool unsupported error savepointVerb string savepointName string savepointCount int + ddl bool } type savepoint struct { @@ -604,13 +602,6 @@ func (s *sqliteConnectionState) bind(conn *sqlite3.SQLiteConn) { s.sqlite = conn } -func (s *sqliteConnectionState) clear(conn *sqlite3.SQLiteConn) { - conn.RegisterPreUpdateHook(nil) - conn.RegisterCommitHook(nil) - conn.RegisterRollbackHook(nil) - conn.RegisterAuthorizer(nil) -} - func (s *sqliteConnectionState) authorizer(action int, arg1, arg2, _ string) int { // Reaching authorizer for another prepared statement proves that any // earlier commit-hook boundary belongs to a completed statement. This is @@ -1182,9 +1173,9 @@ type mutationKey struct { } type netMutation struct { - mutation sqlapi.Mutation first string last string + mutation sqlapi.Mutation } // netChanges retains the earliest before-image for each row and the latest @@ -1288,7 +1279,7 @@ func (s *sqliteConnectionState) validateTable(schema, table string) error { if strings.EqualFold(schema, "temp") { return errors.New("sqlite mutation observer does not support TEMP tables") } - query := fmt.Sprintf("SELECT sql FROM %s.sqlite_master WHERE type = 'table' AND name = ?", quoteIdentifier(schema)) + query := sqliteMasterQuery(schema) rows, err := s.sqlite.Query(query, []driver.Value{table}) if err != nil { return fmt.Errorf("inspect sqlite table %s.%s: %w", schema, table, err) @@ -1296,7 +1287,7 @@ func (s *sqliteConnectionState) validateTable(schema, table string) error { defer rows.Close() values := make([]driver.Value, len(rows.Columns())) if err := rows.Next(values); err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { return fmt.Errorf("sqlite table %s.%s disappeared", schema, table) } return err @@ -1332,6 +1323,10 @@ func quoteIdentifier(value string) string { return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` } +func sqliteMasterQuery(schema string) string { + return "SELECT sql FROM " + quoteIdentifier(schema) + ".sqlite_master WHERE type = 'table' AND name = ?" +} + func scanSQLiteRow(data *sqlite3.SQLitePreUpdateData, count int, isNew bool) ([]any, error) { if count <= 0 { return nil, nil @@ -1365,8 +1360,6 @@ func (c *observedConn) bindIfActive() { c.state.bind(c.sqlite) } -func (c *observedConn) clearHooks() { c.state.clear(c.sqlite) } - func (c *observedConn) Prepare(query string) (driver.Stmt, error) { c.state.prepareMeta = statementMeta{} stmt, err := c.raw.Prepare(query) @@ -1440,6 +1433,7 @@ func (c *observedConn) Close() error { } func (c *observedConn) Begin() (driver.Tx, error) { + //nolint:staticcheck // driver.Conn requires Begin for legacy driver compatibility. tx, err := c.raw.Begin() if err != nil { return nil, err @@ -1514,6 +1508,7 @@ func (s *observedStmt) QueryContext(ctx context.Context, args []driver.NamedValu } func (s *observedStmt) ColumnConverter(index int) driver.ValueConverter { + //nolint:staticcheck // Preserve the optional legacy converter exposed by the wrapped driver. if converter, ok := s.raw.(driver.ColumnConverter); ok { return converter.ColumnConverter(index) } @@ -1522,6 +1517,7 @@ func (s *observedStmt) ColumnConverter(index int) driver.ValueConverter { func (s *observedStmt) Exec(args []driver.Value) (driver.Result, error) { s.conn.state.statementBeginWithMeta(s.meta) + //nolint:staticcheck // driver.Stmt requires Exec for legacy driver compatibility. result, err := s.raw.Exec(args) s.conn.state.statementEndWithMeta(err, s.meta) return result, err @@ -1529,6 +1525,7 @@ func (s *observedStmt) Exec(args []driver.Value) (driver.Result, error) { func (s *observedStmt) Query(args []driver.Value) (driver.Rows, error) { s.conn.state.statementBeginWithMeta(s.meta) + //nolint:staticcheck // driver.Stmt requires Query for legacy driver compatibility. rows, err := s.raw.Query(args) if err != nil { s.conn.state.statementEndWithMeta(err, s.meta) @@ -1550,7 +1547,7 @@ func (r *observedRows) Columns() []string { return r.raw.Columns() } func (r *observedRows) Next(dest []driver.Value) error { err := r.raw.Next(dest) - if err == io.EOF || err != nil { + if errors.Is(err, io.EOF) || err != nil { r.finish(err) } return err @@ -1609,23 +1606,23 @@ func (r *observedRows) finish(err error) { } type mutationStream struct { - backend *sqliteBackend - opts sqlapi.MutationOptions + err error changes chan sqlapi.MutationBatch notify chan struct{} done chan struct{} - mu sync.Mutex - err error - closed bool - snapshotting bool + cancel context.CancelFunc + backend *sqliteBackend watermark string queue []sqlapi.MutationBatch pending []sqlapi.MutationBatch + opts sqlapi.MutationOptions queuedChanges int queuedBytes int - cancel context.CancelFunc maxChanges int maxBytes int + mu sync.Mutex + snapshotting bool + closed bool } func newMutationStream(backend *sqliteBackend, opts sqlapi.MutationOptions) *mutationStream { diff --git a/service/sql/engine/sqlite/observer_test.go b/service/sql/engine/sqlite/observer_test.go index 4f1ef81b4..ffd42ab5d 100644 --- a/service/sql/engine/sqlite/observer_test.go +++ b/service/sql/engine/sqlite/observer_test.go @@ -51,7 +51,7 @@ func TestPerPoolObserverCapturesOwnDatabase(t *testing.T) { defer second.Close() for _, db := range []*openedDBForTest{first, second} { - _, err := db.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err := db.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) } @@ -62,9 +62,9 @@ func TestPerPoolObserverCapturesOwnDatabase(t *testing.T) { require.NoError(t, err) defer func() { _ = secondStream.Close() }() - _, err = first.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1, 'first')`) + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'first')`) require.NoError(t, err) - _, err = second.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (2, 'second')`) + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'second')`) require.NoError(t, err) firstBatch := receiveBatch(t, firstStream) @@ -86,7 +86,7 @@ func TestObserverRebindsAfterConnectionExpiry(t *testing.T) { require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) @@ -94,7 +94,7 @@ func TestObserverRebindsAfterConnectionExpiry(t *testing.T) { observed.opened.DB.SetConnMaxLifetime(time.Nanosecond) time.Sleep(2 * time.Millisecond) - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1, 'rebound')`) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'rebound')`) require.NoError(t, err) batch := receiveBatch(t, stream) @@ -124,10 +124,10 @@ func TestObserverCloseCancelsSnapshotRead(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-close.db")) require.NoError(t, err) defer observed.opened.DB.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) for i := 1; i <= 64; i++ { - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (?, ?)`, i, "value") + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?, ?)`, i, "value") require.NoError(t, err) } snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ @@ -149,19 +149,19 @@ func TestObserverPublishesNetTransactionAfterCommit(t *testing.T) { require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'first')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'first')`) require.NoError(t, err) - _, err = tx.Exec(`UPDATE items SET value = 'second' WHERE id = 1`) + _, err = tx.ExecContext(context.Background(), `UPDATE items SET value = 'second' WHERE id = 1`) require.NoError(t, err) - _, err = tx.Exec(`UPDATE items SET value = 'final' WHERE id = 1`) + _, err = tx.ExecContext(context.Background(), `UPDATE items SET value = 'final' WHERE id = 1`) require.NoError(t, err) require.NoError(t, tx.Commit()) @@ -172,11 +172,11 @@ func TestObserverPublishesNetTransactionAfterCommit(t *testing.T) { assert.Nil(t, change.Before) assert.Equal(t, []byte("final"), change.After[1]) - tx, err = observed.opened.DB.Begin() + tx, err = observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (2, 'transient')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'transient')`) require.NoError(t, err) - _, err = tx.Exec(`DELETE FROM items WHERE id = 2`) + _, err = tx.ExecContext(context.Background(), `DELETE FROM items WHERE id = 2`) require.NoError(t, err) require.NoError(t, tx.Commit()) select { @@ -191,23 +191,23 @@ func TestObserverSavepointRollbackDoesNotPublishRolledBackRows(t *testing.T) { require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'kept')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'kept')`) require.NoError(t, err) - _, err = tx.Exec(`SAVEPOINT /* comment */ nested`) + _, err = tx.ExecContext(context.Background(), `SAVEPOINT /* comment */ nested`) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (2, 'rolled-back')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'rolled-back')`) require.NoError(t, err) - _, err = tx.Exec(`ROLLBACK /* comment */ TO SAVEPOINT nested`) + _, err = tx.ExecContext(context.Background(), `ROLLBACK /* comment */ TO SAVEPOINT nested`) require.NoError(t, err) - _, err = tx.Exec(`RELEASE /* comment */ SAVEPOINT nested`) + _, err = tx.ExecContext(context.Background(), `RELEASE /* comment */ SAVEPOINT nested`) require.NoError(t, err) require.NoError(t, tx.Commit()) @@ -227,15 +227,15 @@ func TestObserverFailsClosedForAmbiguousPartialStatement(t *testing.T) { require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT OR FAIL INTO items (id, value) VALUES (1, 'first'), (2, 'first')`) + _, err = tx.ExecContext(context.Background(), `INSERT OR FAIL INTO items (id, value) VALUES (1, 'first'), (2, 'first')`) require.Error(t, err) require.NoError(t, tx.Commit()) @@ -253,10 +253,10 @@ func TestObserverSnapshotFencesLiveWrites(t *testing.T) { require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) for i := 1; i <= 3; i++ { - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (?, ?)`, i, "old") + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?, ?)`, i, "old") require.NoError(t, err) } @@ -268,7 +268,7 @@ func TestObserverSnapshotFencesLiveWrites(t *testing.T) { writeDone := make(chan error, 1) go func() { - _, writeErr := observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (99, 'live')`) + _, writeErr := observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (99, 'live')`) writeDone <- writeErr }() @@ -299,11 +299,11 @@ func TestObserverSnapshotHandoffIncludesInFlightWriterAsLive(t *testing.T) { require.NoError(t, err) defer observed.Close() observed.opened.DB.SetMaxOpenConns(2) - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (10, 'in-flight')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (10, 'in-flight')`) require.NoError(t, err) snapshot, err := observed.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{Tables: []string{"items"}, BatchSize: 8}) @@ -327,15 +327,15 @@ func TestObserverAbortedStatementDoesNotPublishPartialRows(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "abort.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'same'), (2, 'same')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'same'), (2, 'same')`) require.Error(t, err) require.NoError(t, tx.Commit()) select { @@ -351,15 +351,15 @@ func TestObserverFailsClosedWithoutSQLConflictInference(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "conflict-lexing.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT /* OR FAIL */ INTO items (id, value) VALUES (1, 'literal'), (2, 'literal')`) + _, err = tx.ExecContext(context.Background(), `INSERT /* OR FAIL */ INTO items (id, value) VALUES (1, 'literal'), (2, 'literal')`) require.Error(t, err) require.NoError(t, tx.Commit()) select { @@ -375,16 +375,16 @@ func TestObserverFailedSavepointCommandDoesNotCorruptCapture(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "failed-savepoint.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'kept')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'kept')`) require.NoError(t, err) - _, err = tx.Exec(`ROLLBACK TO SAVEPOINT missing`) + _, err = tx.ExecContext(context.Background(), `ROLLBACK TO SAVEPOINT missing`) require.Error(t, err) require.NoError(t, tx.Commit()) batch := receiveBatch(t, stream) @@ -396,11 +396,11 @@ func TestObserverFailsClosedForMultipleSavepointsInOneExec(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "multi-savepoint.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) - _, err = observed.opened.DB.Exec(`SAVEPOINT first; SAVEPOINT second`) + _, err = observed.opened.DB.ExecContext(context.Background(), `SAVEPOINT first; SAVEPOINT second`) require.NoError(t, err) select { case _, ok := <-stream.Changes(): @@ -415,12 +415,12 @@ func TestObserverReturningRowsFinalizeOnClose(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "returning.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - rows, err := observed.opened.DB.Query(`INSERT INTO items (id, value) VALUES (1, 'returned') RETURNING id`) + rows, err := observed.opened.DB.QueryContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'returned') RETURNING id`) require.NoError(t, err) var id int64 require.True(t, rows.Next()) @@ -436,14 +436,14 @@ func TestObserverFiltersLiveMutations(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "filters.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT); CREATE TABLE ignored (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT); CREATE TABLE ignored (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"items"}, Operations: []string{"insert"}}) require.NoError(t, err) defer func() { _ = stream.Close() }() - _, err = observed.opened.DB.Exec(`INSERT INTO ignored (id, value) VALUES (1, 'no')`) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO ignored (id, value) VALUES (1, 'no')`) require.NoError(t, err) - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (2, 'yes')`) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'yes')`) require.NoError(t, err) batch := receiveBatch(t, stream) require.Len(t, batch.Changes, 1) @@ -459,11 +459,11 @@ func TestObserverOverflowClosesWithoutBlockingCommit(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "overflow.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1, MaxBytes: 1 << 20}) require.NoError(t, err) - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1, 'one'), (2, 'two')`) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'one'), (2, 'two')`) require.NoError(t, err) select { case _, ok := <-stream.Changes(): @@ -478,12 +478,12 @@ func TestObserverKeepsMultiStatementCommitBoundaries(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "boundaries.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (?,'one'); INSERT INTO items (id, value) VALUES (?,'two')`, 1, 2) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?,'one'); INSERT INTO items (id, value) VALUES (?,'two')`, 1, 2) require.NoError(t, err) first := receiveBatch(t, stream) second := receiveBatch(t, stream) @@ -497,12 +497,12 @@ func TestObserverKeepsCommittedPrefixBeforeParameterizedLaterError(t *testing.T) observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "parameterized-boundary-error.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - _, err = observed.opened.DB.Exec( + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?,'one'); INSERT INTO items (id, value) VALUES (?,'one')`, 1, 2, ) @@ -520,12 +520,12 @@ func TestObserverKeepsCommittedPrefixBeforeParameterizedSyntaxTail(t *testing.T) observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "parameterized-syntax-tail.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - _, err = observed.opened.DB.Exec( + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (?, 'one'); INSER INTO items (id, value) VALUES (?, 'two')`, 1, 2, ) @@ -539,14 +539,14 @@ func TestObserverPublishesEarlierAutocommitBeforeLaterError(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "boundary-error.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT UNIQUE)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) defer func() { _ = stream.Close() }() - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (1,'one')`) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1,'one')`) require.NoError(t, err) - _, err = observed.opened.DB.Exec(`INSERT INTO items (id, value) VALUES (2,'one')`) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2,'one')`) require.Error(t, err) batch := receiveBatch(t, stream) require.Len(t, batch.Changes, 1) @@ -562,7 +562,7 @@ func TestObserverRejectsUnsupportedVirtualTable(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "virtual.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE VIRTUAL TABLE docs USING fts5(content)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE VIRTUAL TABLE docs USING fts5(content)`) if err != nil { t.Skipf("sqlite build has no fts5 virtual table: %v", err) } @@ -575,11 +575,11 @@ func TestObserverFailsClosedWhenVirtualTableIsCreatedDynamically(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "virtual-dynamic.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{Tables: []string{"items"}}) require.NoError(t, err) - _, err = observed.opened.DB.Exec(`CREATE VIRTUAL TABLE docs USING fts5(content)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE VIRTUAL TABLE docs USING fts5(content)`) if err != nil { t.Skipf("sqlite build has no fts5 virtual table: %v", err) } @@ -596,15 +596,15 @@ func TestObserverDetectsDDLThroughAuthorizerWithLeadingComment(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "ddl-comment.db")) require.NoError(t, err) defer observed.Close() - _, err = observed.opened.DB.Exec(`CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) stream, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) require.NoError(t, err) - tx, err := observed.opened.DB.Begin() + tx, err := observed.opened.DB.BeginTx(context.Background(), nil) require.NoError(t, err) - _, err = tx.Exec(`/* CREATE TABLE */ CREATE TABLE other (id INTEGER PRIMARY KEY, value TEXT)`) + _, err = tx.ExecContext(context.Background(), `/* CREATE TABLE */ CREATE TABLE other (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) - _, err = tx.Exec(`INSERT INTO items (id, value) VALUES (1, 'value')`) + _, err = tx.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'value')`) require.NoError(t, err) require.NoError(t, tx.Commit()) select { From 1aa488bc3fa5bc09e545bdddc723ca220000b241 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:13:55 -0400 Subject: [PATCH 21/47] fix(cdc/sqlite): own snapshot acquisition lifecycle --- service/cdc/sqlite/source.go | 77 ++++++++++++++++++++++++-- service/cdc/sqlite/source_test.go | 62 +++++++++++++++++++-- service/cdc/sqlite/subscribers.go | 31 ++++++++--- service/cdc/sqlite/subscribers_test.go | 30 ++++++++++ 4 files changed, 180 insertions(+), 20 deletions(-) diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index f7fb80b75..a2cff90b4 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -39,7 +39,7 @@ type Source struct { res resource.Registry observerSource sqlapi.CommittedMutationSource observer sqlapi.MutationStream - runDone chan struct{} + snapshotAcq map[uint64]*snapshotAcquisition runCancel context.CancelFunc status chan any startCancel context.CancelFunc @@ -47,14 +47,16 @@ type Source struct { log *zap.Logger startDone chan struct{} snapshotSubs map[*subscription]sqlapi.MutationStream - id registry.ID + runDone chan struct{} dbResID registry.ID + id registry.ID name string generation string state config.SourceState tables []string lifecycle configLifecycle snapshotWG sync.WaitGroup + nextSnapshotID uint64 statusTick time.Duration mu sync.RWMutex snapshot bool @@ -66,6 +68,10 @@ type Source struct { // not expose configuration implementation details through its API. type configLifecycle = supervisor.LifecycleConfig +type snapshotAcquisition struct { + cancel context.CancelFunc +} + func buildSource(opts sourceOptions) (managedSource, error) { log := opts.log if log == nil { @@ -101,6 +107,7 @@ func buildSource(opts sourceOptions) (managedSource, error) { snapshot: opts.snapshot, subs: newSubscribers(), snapshotSubs: make(map[*subscription]sqlapi.MutationStream), + snapshotAcq: make(map[uint64]*snapshotAcquisition), state: config.SourceStateUnknown, }, nil } @@ -430,6 +437,7 @@ func (s *Source) fail(err error) { s.observerSource = nil snapshotSubscriptions := make([]*subscription, 0, len(s.snapshotSubs)) snapshotSubs := make([]sqlapi.MutationStream, 0, len(s.snapshotSubs)) + snapshotCancels := s.snapshotAcquisitionCancelsLocked() for sub, snapshotStream := range s.snapshotSubs { snapshotSubscriptions = append(snapshotSubscriptions, sub) snapshotSubs = append(snapshotSubs, snapshotStream) @@ -439,6 +447,9 @@ func (s *Source) fail(err error) { s.mu.Unlock() s.subs.closeWithError(err) + for _, cancel := range snapshotCancels { + cancel() + } for _, sub := range snapshotSubscriptions { sub.closeWithError(err) } @@ -478,6 +489,7 @@ func (s *Source) Stop(ctx context.Context) error { stream := s.observer snapshotStreams := make([]sqlapi.MutationStream, 0, len(s.snapshotSubs)) snapshotSubscriptions := make([]*subscription, 0, len(s.snapshotSubs)) + snapshotCancels := s.snapshotAcquisitionCancelsLocked() for sub, snapshotStream := range s.snapshotSubs { snapshotSubscriptions = append(snapshotSubscriptions, sub) snapshotStreams = append(snapshotStreams, snapshotStream) @@ -494,6 +506,9 @@ func (s *Source) Stop(ctx context.Context) error { if stream != nil { _ = stream.Close() } + for _, cancel := range snapshotCancels { + cancel() + } for _, sub := range snapshotSubscriptions { sub.closeWithError(nil) } @@ -546,6 +561,44 @@ func waitDone(ctx context.Context, done <-chan struct{}) error { } } +func (s *Source) beginSnapshotAcquisition(ctx context.Context, observer sqlapi.CommittedMutationSource) (context.Context, uint64, error) { + acquisitionCtx, cancel := context.WithCancel(ctx) + s.mu.Lock() + if s.state != config.SourceStateRunning || s.stopping || s.observerSource != observer { + s.mu.Unlock() + cancel() + return nil, 0, config.ErrSourceNotReady + } + s.nextSnapshotID++ + id := s.nextSnapshotID + s.snapshotAcq[id] = &snapshotAcquisition{cancel: cancel} + s.snapshotWG.Add(1) + s.mu.Unlock() + return acquisitionCtx, id, nil +} + +func (s *Source) finishSnapshotAcquisition(id uint64) { + s.mu.Lock() + acquisition, ok := s.snapshotAcq[id] + if ok { + delete(s.snapshotAcq, id) + } + s.mu.Unlock() + if !ok { + return + } + acquisition.cancel() + s.snapshotWG.Done() +} + +func (s *Source) snapshotAcquisitionCancelsLocked() []context.CancelFunc { + cancels := make([]context.CancelFunc, 0, len(s.snapshotAcq)) + for _, acquisition := range s.snapshotAcq { + cancels = append(cancels, acquisition.cancel) + } + return cancels +} + // Subscribe exposes committed changes. Cursor resume remains unsupported // because this process-local generation has no durable checkpoint; snapshots // use the SQL-owned atomic fence and handoff stream per subscriber. @@ -599,13 +652,19 @@ func (s *Source) subscribeSnapshot(ctx context.Context, observer sqlapi.Committe return sub, nil } - stream, err := observer.Snapshot(ctx, sqlapi.SnapshotOptions{ + acquisitionCtx, acquisitionID, err := s.beginSnapshotAcquisition(ctx, observer) + if err != nil { + return nil, err + } + stream, err := observer.Snapshot(acquisitionCtx, sqlapi.SnapshotOptions{ Tables: tables, }) if err != nil { + s.finishSnapshotAcquisition(acquisitionID) return nil, err } if stream == nil { + s.finishSnapshotAcquisition(acquisitionID) return nil, errors.New("sqlite snapshot observer returned a nil stream") } @@ -615,23 +674,31 @@ func (s *Source) subscribeSnapshot(ctx context.Context, observer sqlapi.Committe if s.state != config.SourceStateRunning || s.stopping || s.observerSource != observer { s.mu.Unlock() _ = stream.Close() + s.finishSnapshotAcquisition(acquisitionID) + return nil, config.ErrSourceNotReady + } + if _, ok := s.snapshotAcq[acquisitionID]; !ok { + s.mu.Unlock() + _ = stream.Close() + s.finishSnapshotAcquisition(acquisitionID) return nil, config.ErrSourceNotReady } s.snapshotSubs[sub] = stream s.snapshotWG.Add(1) s.mu.Unlock() - go s.runSnapshot(ctx, stream, sub) + go s.runSnapshot(acquisitionCtx, stream, sub, acquisitionID) return sub, nil } -func (s *Source) runSnapshot(ctx context.Context, stream sqlapi.SnapshotStream, sub *subscription) { +func (s *Source) runSnapshot(ctx context.Context, stream sqlapi.SnapshotStream, sub *subscription, acquisitionID uint64) { defer s.snapshotWG.Done() defer func() { s.mu.Lock() delete(s.snapshotSubs, sub) s.mu.Unlock() }() + defer s.finishSnapshotAcquisition(acquisitionID) // The SQL observer owns the snapshot read transaction and scan worker. A // subscriber can end for any reason (upstream error, downstream overflow, // cancellation, or normal close), so every return path must release that diff --git a/service/cdc/sqlite/source_test.go b/service/cdc/sqlite/source_test.go index 27e898fc5..faffd9a4f 100644 --- a/service/cdc/sqlite/source_test.go +++ b/service/cdc/sqlite/source_test.go @@ -49,12 +49,14 @@ func (r *testDBResource) Release() { } type testObserver struct { - stream *testMutationStream - snapshot *testSnapshotStream - subOpts sqlapi.MutationOptions - mu sync.Mutex - closeN atomic.Int32 - closed bool + stream *testMutationStream + snapshot *testSnapshotStream + snapshotStarted chan struct{} + subOpts sqlapi.MutationOptions + snapshotCancelDelay time.Duration + mu sync.Mutex + closeN atomic.Int32 + closed bool } func (o *testObserver) Subscribe(ctx context.Context, opts sqlapi.MutationOptions) (sqlapi.MutationStream, error) { @@ -75,6 +77,17 @@ func (o *testObserver) Snapshot(ctx context.Context, _ sqlapi.SnapshotOptions) ( if err := ctx.Err(); err != nil { return nil, err } + if o.snapshotStarted != nil { + select { + case o.snapshotStarted <- struct{}{}: + default: + } + <-ctx.Done() + if o.snapshotCancelDelay > 0 { + time.Sleep(o.snapshotCancelDelay) + } + return nil, ctx.Err() + } o.mu.Lock() defer o.mu.Unlock() if o.closed { @@ -412,6 +425,43 @@ func TestSourceSnapshotOverflowClosesUpstream(t *testing.T) { stream.Close() } +func TestSourceStopWaitsForBlockedSnapshotAcquisition(t *testing.T) { + observer := &testObserver{ + snapshotStarted: make(chan struct{}, 1), + snapshotCancelDelay: 50 * time.Millisecond, + } + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + + subscribeDone := make(chan error, 1) + go func() { + _, subscribeErr := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + subscribeDone <- subscribeErr + }() + select { + case <-observer.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("snapshot acquisition did not start") + } + + stopDone := make(chan error, 1) + go func() { stopDone <- source.Stop(context.Background()) }() + select { + case err := <-stopDone: + t.Fatalf("Stop returned before the blocked snapshot acquisition ended: %v", err) + case <-time.After(20 * time.Millisecond): + } + require.NoError(t, <-stopDone) + assert.Error(t, <-subscribeDone) + + source.mu.RLock() + assert.Empty(t, source.snapshotAcq) + assert.Empty(t, source.snapshotSubs) + assert.Equal(t, cdcapi.SourceStateStopped, source.state) + source.mu.RUnlock() +} + func TestSourceStopsWithoutClosingSQLGeneration(t *testing.T) { observer := &testObserver{} resources := &testResourceRegistry{observer: observer} diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go index bc9470f0f..b62f97635 100644 --- a/service/cdc/sqlite/subscribers.go +++ b/service/cdc/sqlite/subscribers.go @@ -127,27 +127,40 @@ func (s *subscription) send(change config.Change) { return } s.mu.Lock() - defer s.mu.Unlock() - if s.closed { - return + var parent *subscribers + var id uint64 + if !s.closed { + select { + case s.changes <- change: + default: + s.closeLocked(errSubscriberOverflow) + parent = s.parent + id = s.id + } } - select { - case s.changes <- change: - default: - s.closeLocked(errSubscriberOverflow) + s.mu.Unlock() + // Detach after releasing the subscription lock. Taking the parent lock + // while holding s.mu would invert the order used by closeWithError and + // make concurrent publish/close able to deadlock. + if parent != nil { + parent.remove(id) } } func (s *subscription) closeWithError(err error) { s.mu.Lock() + var parent *subscribers + var id uint64 if s.closed { s.mu.Unlock() return } s.closeLocked(err) + parent = s.parent + id = s.id s.mu.Unlock() - if s.parent != nil { - s.parent.remove(s.id) + if parent != nil { + parent.remove(id) } } diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go index be0476f54..d24b6a3bc 100644 --- a/service/cdc/sqlite/subscribers_test.go +++ b/service/cdc/sqlite/subscribers_test.go @@ -125,6 +125,36 @@ func TestPublishNeverBlocksAndClosesLaggard(t *testing.T) { } } +func TestOverflowedSubscriberDetachesImmediately(t *testing.T) { + subs := newSubscribers() + stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) + change := config.Change{Op: "insert", Table: "users"} + + stream.send(change) + stream.send(change) + + assert.ErrorIs(t, stream.Err(), errSubscriberOverflow) + subs.mu.RLock() + remaining := len(subs.m) + subs.mu.RUnlock() + assert.Zero(t, remaining) +} + +func TestOverflowedSubscriberChurnDoesNotRetainParentEntries(t *testing.T) { + subs := newSubscribers() + change := config.Change{Op: "insert", Table: "users"} + for i := 0; i < 1000; i++ { + stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) + stream.send(change) + stream.send(change) + } + + subs.mu.RLock() + remaining := len(subs.m) + subs.mu.RUnlock() + assert.Zero(t, remaining) +} + func TestSubscribersFilterByOp(t *testing.T) { subs := newSubscribers() stream := subs.subscribe("s", config.StreamOptions{Ops: []string{"delete"}}) From f4ca3b1899f536179118325f546290b038c2b344 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:18:23 -0400 Subject: [PATCH 22/47] fix(cdc): make dispatcher admission and relay delivery cancellable --- api/relay/relay.go | 9 ++ service/cdc/dispatcher.go | 133 +++++++++++++++++++++++++---- service/cdc/dispatcher_test.go | 152 ++++++++++++++++++++++++++++++++- system/relay/mailbox.go | 20 +++++ system/relay/mailbox_test.go | 16 ++++ system/relay/node.go | 20 +++++ 6 files changed, 328 insertions(+), 22 deletions(-) diff --git a/api/relay/relay.go b/api/relay/relay.go index 67d5e2594..bcdd5b255 100644 --- a/api/relay/relay.go +++ b/api/relay/relay.go @@ -62,6 +62,15 @@ type ( Send(*Package) error } + // ContextSender is the cancellable delivery capability. Implementations + // must stop waiting for delivery when ctx is canceled. It is optional so + // existing receivers keep the original Send contract; lifecycle-sensitive + // dispatchers can require this capability instead of detaching a blocked + // Send goroutine. + ContextSender interface { + SendContext(context.Context, *Package) error + } + // AttachableReceiver extends Receiver with channel-based message delivery. AttachableReceiver interface { Receiver diff --git a/service/cdc/dispatcher.go b/service/cdc/dispatcher.go index 3d379bba0..3957e097a 100644 --- a/service/cdc/dispatcher.go +++ b/service/cdc/dispatcher.go @@ -36,6 +36,10 @@ var ( ErrNilSource = errors.New("cdc registry returned a nil source") // ErrNoRelayNode indicates that the process has no relay transport. ErrNoRelayNode = errors.New("cdc relay node not available") + // ErrRelayNotCancellable indicates that a relay node cannot bind delivery + // to the subscription lifecycle. Detaching an unowned Send goroutine would + // leak it, so CDC refuses that delivery path instead. + ErrRelayNotCancellable = errors.New("cdc relay node does not support cancellable delivery") ) type dispatcherState uint8 @@ -59,10 +63,16 @@ type Dispatcher struct { stopDone chan struct{} workersWG sync.WaitGroup relaysWG sync.WaitGroup - workers int - nextID uint64 - mu sync.Mutex - state dispatcherState + // admissionsDone is a per-run barrier. Stop cancels workers first, then + // waits for handles that already passed the state check before workers + // drain the queue. This closes the Handle/Stop admission race without + // holding mu across a potentially blocking queue send. + admissionsDone chan struct{} + admissions int + workers int + nextID uint64 + mu sync.Mutex + state dispatcherState } type dispatchJob struct { @@ -131,16 +141,21 @@ func (d *Dispatcher) Start(ctx context.Context) error { d.jobs = make(chan dispatchJob, d.workers*2) d.sessions = make(map[uint64]*relaySession) d.stopDone = make(chan struct{}) + d.admissionsDone = make(chan struct{}) + d.admissions = 0 d.state = stateRunning for i := 0; i < d.workers; i++ { d.workersWG.Add(1) } runCtx := d.ctx + stopDone := d.stopDone + admissionsDone := d.admissionsDone d.mu.Unlock() for i := 0; i < d.workers; i++ { - go d.worker(runCtx) + go d.worker(runCtx, admissionsDone) } + go d.monitorContext(runCtx, stopDone) return nil } @@ -165,6 +180,7 @@ func (d *Dispatcher) Stop(ctx context.Context) error { d.state = stateStopping done := d.stopDone cancel := d.cancel + d.closeAdmissionsLocked() sessions := make([]*relaySession, 0, len(d.sessions)) for _, session := range d.sessions { sessions = append(sessions, session) @@ -185,6 +201,20 @@ func (d *Dispatcher) Stop(ctx context.Context) error { } } +// monitorContext turns cancellation of the context supplied to Start into a +// normal dispatcher stop. The stopDone identity prevents a stale monitor from +// stopping a later run after the dispatcher has been restarted. +func (d *Dispatcher) monitorContext(runCtx context.Context, stopDone chan struct{}) { + <-runCtx.Done() + + d.mu.Lock() + valid := d.state == stateRunning && d.stopDone == stopDone + d.mu.Unlock() + if valid { + _ = d.Stop(context.Background()) + } +} + func (d *Dispatcher) finishStop(done chan struct{}) { d.workersWG.Wait() d.relaysWG.Wait() @@ -209,7 +239,7 @@ func waitForStop(ctx context.Context, done <-chan struct{}) error { } } -func (d *Dispatcher) worker(ctx context.Context) { +func (d *Dispatcher) worker(ctx context.Context, admissionsDone <-chan struct{}) { defer d.workersWG.Done() for { @@ -221,12 +251,40 @@ func (d *Dispatcher) worker(ctx context.Context) { } d.execute(ctx, job) case <-ctx.Done(): + // A Handle may have passed the running-state check but not yet + // enqueued its job. Wait for those admissions before draining; + // otherwise a send racing cancellation could land in a queue with + // no worker left to complete it. + <-admissionsDone d.drainJobs() return } } } +func (d *Dispatcher) closeAdmissionsLocked() { + if d.admissions == 0 && d.admissionsDone != nil { + select { + case <-d.admissionsDone: + default: + close(d.admissionsDone) + } + } +} + +func (d *Dispatcher) endAdmission(done chan struct{}) { + d.mu.Lock() + d.admissions-- + if d.admissions == 0 && d.state != stateRunning { + select { + case <-done: + default: + close(done) + } + } + d.mu.Unlock() +} + // drainJobs completes commands accepted before cancellation. Jobs are never // dropped silently, which prevents a process yield from remaining pending // when the dispatcher is stopped. @@ -310,13 +368,21 @@ func (d *Dispatcher) executeSubscribe(dispatchCtx, requestCtx context.Context, c func (d *Dispatcher) openStream(ctx context.Context, cmd cdcapi.SubscribeCmd) (changeStream, error) { if reg := cdcapi.GetRegistry(ctx); reg != nil { source, ok := reg.Get(registry.ParseID(cmd.Source)) - if !ok { - return nil, fmt.Errorf("%w: %s", cdcapi.ErrSourceNotFound, cmd.Source) + if ok { + if source == nil { + return nil, fmt.Errorf("%w: %s", ErrNilSource, cmd.Source) + } + return source.Subscribe(ctx, cmd.Options) } - if source == nil { - return nil, fmt.Errorf("%w: %s", ErrNilSource, cmd.Source) + + // A registry miss can still be served by a pre-registry caller's + // streamer. A present registry entry, including a corrupt nil entry, + // remains authoritative so legacy aliases cannot shadow canonical IDs. + if streamer := cdcapi.GetSourceStreamer(ctx); streamer != nil { + stream, _, err := streamer.Stream(ctx, cmd.Source, cmd.Options) + return stream, err } - return source.Subscribe(ctx, cmd.Options) + return nil, fmt.Errorf("%w: %s", cdcapi.ErrSourceNotFound, cmd.Source) } streamer := cdcapi.GetSourceStreamer(ctx) @@ -374,15 +440,15 @@ func (d *Dispatcher) relay(ctx context.Context, session *relaySession, changes < case change, ok := <-changes: if !ok { if err := streamError(stream); err != nil { - d.sendTerminal(node, target, topic, err) + d.sendTerminal(ctx, node, target, topic, err) } else { - d.sendTerminal(node, target, topic, nil) + d.sendTerminal(ctx, node, target, topic, nil) } return } pkg := relay.NewPackage(pid.Zero(), target, topic, payload.New(change)) - if err := node.Send(pkg); err != nil { + if err := sendRelay(ctx, node, pkg); err != nil { d.log.Debug("failed to relay cdc change", zap.String("source", source), zap.Error(err)) @@ -403,20 +469,38 @@ func (d *Dispatcher) relayDone(id uint64) { d.mu.Unlock() } -func (d *Dispatcher) sendTerminal(node relay.Node, target pid.PID, topic string, err error) { +func (d *Dispatcher) sendTerminal(ctx context.Context, node relay.Node, target pid.PID, topic string, err error) { var terminal payload.Payloads if err != nil { terminal = append(terminal, payload.NewError(err)) } terminal = append(terminal, payload.NewTerminal()) pkg := relay.NewPackage(pid.Zero(), target, topic, terminal...) - if sendErr := node.Send(pkg); sendErr != nil { + if sendErr := sendRelay(ctx, node, pkg); sendErr != nil { d.log.Debug("failed to send cdc terminal", zap.String("topic", topic), zap.Error(sendErr)) } } +// sendRelay uses the relay-owned cancellation contract. Starting an +// untracked goroutine around Receiver.Send would make Stop return while that +// goroutine retained the stream, node, and process context indefinitely; a +// node without the capability therefore fails the delivery explicitly. +func sendRelay(ctx context.Context, node relay.Node, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + sender, ok := node.(relay.ContextSender) + if !ok { + return ErrRelayNotCancellable + } + return sender.SendContext(ctx, pkg) +} + // streamError is an optional extension implemented by streams that can // report a typed terminal error after their change channel closes. Keeping it // optional preserves compatibility with the original stream interface while @@ -475,15 +559,28 @@ func (d *Dispatcher) Handle(ctx context.Context, cmd dispatcher.Command, tag uin } jobs := d.jobs runCtx := d.ctx + d.admissions++ + admissionsDone := d.admissionsDone d.mu.Unlock() job := dispatchJob{ctx: ctx, cmd: cmd, tag: tag, receiver: receiver} + enqueued := false select { case jobs <- job: + enqueued = true case <-runCtx.Done(): - complete(receiver, tag, nil, ErrDispatcherStopping) + // The admission barrier keeps workers from draining until this + // in-flight send has resolved, even if both cases are ready. case <-ctx.Done(): - complete(receiver, tag, nil, ctx.Err()) + // Complete below after releasing the admission barrier. + } + d.endAdmission(admissionsDone) + if !enqueued { + err := ErrDispatcherStopping + if runCtx.Err() == nil && ctx.Err() != nil { + err = ctx.Err() + } + complete(receiver, tag, nil, err) } return nil } diff --git a/service/cdc/dispatcher_test.go b/service/cdc/dispatcher_test.go index 18ed33174..234dbc6fa 100644 --- a/service/cdc/dispatcher_test.go +++ b/service/cdc/dispatcher_test.go @@ -39,16 +39,28 @@ func (s *dispatcherTestStream) Close() { func (s *dispatcherTestStream) Err() error { return s.err } type dispatcherTestSource struct { - stream *dispatcherTestStream - info cdcapi.SourceInfo + stream *dispatcherTestStream + info cdcapi.SourceInfo + subscribeN atomic.Int32 } func (s *dispatcherTestSource) Info() cdcapi.SourceInfo { return s.info } func (s *dispatcherTestSource) Subscribe(context.Context, cdcapi.StreamOptions) (cdcapi.Stream, error) { + s.subscribeN.Add(1) return s.stream, nil } +type dispatcherLegacyStreamer struct { + stream *dispatcherTestStream + calls atomic.Int32 +} + +func (s *dispatcherLegacyStreamer) Stream(context.Context, string, cdcapi.StreamOptions) (cdcapi.ChangeStream, cdcapi.SourceInfo, error) { + s.calls.Add(1) + return s.stream, cdcapi.SourceInfo{Name: "legacy"}, nil +} + type blockingSubscribeSource struct { started chan struct{} once sync.Once @@ -71,8 +83,9 @@ func (nilSourceRegistry) List() []cdcapi.SourceInfo { return nil } func (nilSourceRegistry) Get(registry.ID) (cdcapi.Source, bool) { return nil, true } type dispatcherTestNode struct { - packages chan *relay.Package - send func(*relay.Package) error + packages chan *relay.Package + send func(*relay.Package) error + sendContext func(context.Context, *relay.Package) error } func (n *dispatcherTestNode) ID() pid.NodeID { return "cdc-dispatcher-test" } @@ -85,6 +98,13 @@ func (n *dispatcherTestNode) Send(pkg *relay.Package) error { return nil } +func (n *dispatcherTestNode) SendContext(ctx context.Context, pkg *relay.Package) error { + if n.sendContext != nil { + return n.sendContext(ctx, pkg) + } + return n.Send(pkg) +} + func (n *dispatcherTestNode) RegisterHost(pid.HostID, relay.Receiver) error { return nil } func (n *dispatcherTestNode) UnregisterHost(pid.HostID) {} func (n *dispatcherTestNode) GetHost(pid.HostID) (relay.Receiver, bool) { return nil, false } @@ -174,6 +194,55 @@ func TestDispatcherUsesSystemRegistryAndRelaysChanges(t *testing.T) { assert.Eventually(t, stream.closed.Load, time.Second, 10*time.Millisecond) } +func TestDispatcherRegistryPrecedesLegacyStreamer(t *testing.T) { + id := registry.NewID("test", "canonical") + canonical := &dispatcherTestSource{stream: &dispatcherTestStream{changes: make(chan cdcapi.Change)}} + legacy := &dispatcherLegacyStreamer{stream: &dispatcherTestStream{changes: make(chan cdcapi.Change)}} + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContext(t, canonical, id, node) + ctx = cdcapi.WithSourceStreamer(ctx, legacy) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + assert.Equal(t, int32(1), canonical.subscribeN.Load()) + assert.Zero(t, legacy.calls.Load()) + + sub, ok := receiver.data.(cdcapi.Subscription) + require.True(t, ok) + sub.Stop() +} + +func TestDispatcherRegistryMissFallsBackToLegacyStreamer(t *testing.T) { + id := registry.NewID("legacy", "source") + legacyStream := &dispatcherTestStream{changes: make(chan cdcapi.Change)} + legacy := &dispatcherLegacyStreamer{stream: legacyStream} + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + reg := cdcsystem.NewRegistry(nil) + ctx := dispatcherTestContextWithRegistry(reg, node) + ctx = cdcapi.WithSourceStreamer(ctx, legacy) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + defer func() { require.NoError(t, d.Stop(context.Background())) }() + + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + cmd := dispatcherTestCommand(id) + require.NoError(t, d.Handle(ctx, cmd, 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + assert.Equal(t, int32(1), legacy.calls.Load()) + + sub, ok := receiver.data.(cdcapi.Subscription) + require.True(t, ok) + sub.Stop() +} + func TestDispatcherRelaysTypedStreamErrorBeforeTerminal(t *testing.T) { streamErr := errors.New("capture gap") stream := &dispatcherTestStream{changes: make(chan cdcapi.Change), err: streamErr} @@ -257,6 +326,8 @@ func TestDispatcherRejectsNilRegistrySource(t *testing.T) { id := registry.NewID("test", "nil") node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} ctx := dispatcherTestContextWithRegistry(nilSourceRegistry{}, node) + legacy := &dispatcherLegacyStreamer{stream: &dispatcherTestStream{changes: make(chan cdcapi.Change)}} + ctx = cdcapi.WithSourceStreamer(ctx, legacy) d := NewDispatcher(WithWorkers(1)) require.NoError(t, d.Start(ctx)) @@ -266,6 +337,41 @@ func TestDispatcherRejectsNilRegistrySource(t *testing.T) { require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) waitResult(t, receiver) assert.ErrorIs(t, receiver.err, ErrNilSource) + assert.Zero(t, legacy.calls.Load()) +} + +func TestDispatcherStopCancelsBlockedRelayDelivery(t *testing.T) { + started := make(chan struct{}) + var startOnce sync.Once + node := &dispatcherTestNode{ + sendContext: func(ctx context.Context, _ *relay.Package) error { + startOnce.Do(func() { close(started) }) + <-ctx.Done() + return ctx.Err() + }, + } + stream := &dispatcherTestStream{changes: make(chan cdcapi.Change, 1)} + id := registry.NewID("test", "blocked-relay") + ctx := dispatcherTestContext(t, &dispatcherTestSource{stream: stream}, id, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + require.NoError(t, d.Handle(ctx, dispatcherTestCommand(id), 1, receiver)) + waitResult(t, receiver) + require.NoError(t, receiver.err) + + stream.changes <- cdcapi.Change{Source: id.String(), Op: "insert"} + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for blocked relay delivery") + } + + stopCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, d.Stop(stopCtx)) + assert.True(t, stream.closed.Load()) } func TestDispatcherStopCancelsBlockingSubscribe(t *testing.T) { @@ -321,3 +427,41 @@ func TestDispatcherHandleAndStopAreSafeConcurrently(t *testing.T) { waitResult(t, receiver) } } + +func TestDispatcherAdmissionStopCompletesEveryJob(t *testing.T) { + for round := 0; round < 100; round++ { + source := &blockingSubscribeSource{started: make(chan struct{})} + id := registry.NewID("test", "admission") + reg := cdcsystem.NewRegistry(nil) + require.NoError(t, reg.Register(id, source, cdcapi.SQLite)) + node := &dispatcherTestNode{packages: make(chan *relay.Package, 1)} + ctx := dispatcherTestContextWithRegistry(reg, node) + + d := NewDispatcher(WithWorkers(1)) + require.NoError(t, d.Start(ctx)) + + const jobs = 16 + receivers := make([]*dispatcherTestReceiver, jobs) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range receivers { + receiver := &dispatcherTestReceiver{done: make(chan struct{})} + receivers[i] = receiver + wg.Add(1) + go func(tag uint64, receiver *dispatcherTestReceiver) { + defer wg.Done() + <-start + _ = d.Handle(ctx, dispatcherTestCommand(id), tag, receiver) + }(uint64(i), receiver) + } + + close(start) + stopDone := make(chan error, 1) + go func() { stopDone <- d.Stop(context.Background()) }() + wg.Wait() + require.NoError(t, <-stopDone) + for _, receiver := range receivers { + waitResult(t, receiver) + } + } +} diff --git a/system/relay/mailbox.go b/system/relay/mailbox.go index 17c759692..94d44e5a8 100644 --- a/system/relay/mailbox.go +++ b/system/relay/mailbox.go @@ -54,6 +54,10 @@ type Mailbox struct { // NewMailbox creates a new Mailbox instance with the provided options. // The supplied context will cancel all workers when done. func NewMailbox(ctx context.Context, opts ...MailboxOption) *Mailbox { + if ctx == nil { + ctx = context.Background() + } + config := mailboxConfig{ workerCount: 1, logger: zap.NewNop(), @@ -122,9 +126,23 @@ func (m *Mailbox) Detach(p pid.PID) { // Send enqueues a package for delivery. Messages from the same source // are routed to the same worker to preserve per-sender FIFO ordering. func (m *Mailbox) Send(pkg *api.Package) error { + return m.SendContext(context.Background(), pkg) +} + +// SendContext enqueues a package until either the mailbox or caller context +// is canceled. The caller context is owned by the delivery operation; the +// mailbox context remains the lifecycle boundary for its workers. +func (m *Mailbox) SendContext(ctx context.Context, pkg *api.Package) error { if pkg == nil { return NewNilPackageError() } + if ctx == nil { + ctx = context.Background() + } + + if err := ctx.Err(); err != nil { + return err + } // Check context before attempting to send to avoid sending to closed channels if err := m.ctx.Err(); err != nil { @@ -138,6 +156,8 @@ func (m *Mailbox) Send(pkg *api.Package) error { select { case m.jobQueues[workerIndex] <- pkg: return nil + case <-ctx.Done(): + return ctx.Err() case <-m.ctx.Done(): m.config.logger.Warn("send after mailbox shutdown", zap.String("pid", pkg.Target.String())) return m.ctx.Err() diff --git a/system/relay/mailbox_test.go b/system/relay/mailbox_test.go index c679e093a..ae2a80dfc 100644 --- a/system/relay/mailbox_test.go +++ b/system/relay/mailbox_test.go @@ -133,6 +133,22 @@ func TestMailbox_SendCancelledContext(t *testing.T) { assert.NoError(t, err) } +func TestMailbox_SendContextHonorsCallerCancellation(t *testing.T) { + mailbox := NewMailbox(context.Background(), + WithBufferSize(1), + WithWorkerCount(0), // keep the queue full for the cancellation check + ) + + target := pidapi.PID{Host: "host1", UniqID: "uniq1"} + pkg := &relay.Package{Target: target} + require.NoError(t, mailbox.Send(pkg)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := mailbox.SendContext(ctx, pkg) + assert.ErrorIs(t, err, context.Canceled) +} + func TestMailbox_NoReceiver(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() diff --git a/system/relay/node.go b/system/relay/node.go index 6940221f1..56733fa07 100644 --- a/system/relay/node.go +++ b/system/relay/node.go @@ -64,9 +64,26 @@ func (n *Node) GetHost(hostID pid.HostID) (api.Receiver, bool) { // Send delivers a package to its destination. The destination must be a host // registered within this node. func (n *Node) Send(pkg *api.Package) error { + return n.send(context.Background(), pkg) +} + +// SendContext delivers a package while honoring cancellation in a +// context-aware host. Hosts that do not expose ContextSender retain the +// historical synchronous receiver contract. +func (n *Node) SendContext(ctx context.Context, pkg *api.Package) error { + if ctx == nil { + ctx = context.Background() + } + return n.send(ctx, pkg) +} + +func (n *Node) send(ctx context.Context, pkg *api.Package) error { if pkg == nil { return NewNilPackageError() } + if err := ctx.Err(); err != nil { + return err + } if pkg.Target.Node != "" && pkg.Target.Node != n.nodeID { return NewExternalNodeError(pkg.Target.Node) @@ -82,6 +99,9 @@ func (n *Node) Send(pkg *api.Package) error { return NewInvalidHostTypeError(pkg.Target.Host, n.nodeID) } + if sender, ok := receiver.(api.ContextSender); ok { + return sender.SendContext(ctx, pkg) + } return receiver.Send(pkg) } From 676ff3d5b05ee545f44e2c5e8bcf4f1c06e84118 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:20:23 -0400 Subject: [PATCH 23/47] fix(relay): reject uncancellable context delivery --- system/relay/errors.go | 15 ++++++++++++-- system/relay/node.go | 3 +++ system/relay/node_test.go | 42 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/system/relay/errors.go b/system/relay/errors.go index ef5aa80e6..15040a86c 100644 --- a/system/relay/errors.go +++ b/system/relay/errors.go @@ -9,10 +9,21 @@ import ( ) var ( - ErrNilPackage = apierror.New(apierror.Invalid, "cannot send nil package").WithRetryable(apierror.False) - ErrAlreadyAttached = apierror.New(apierror.AlreadyExists, "receiver already attached").WithRetryable(apierror.False) + ErrNilPackage = apierror.New(apierror.Invalid, "cannot send nil package").WithRetryable(apierror.False) + ErrAlreadyAttached = apierror.New(apierror.AlreadyExists, "receiver already attached").WithRetryable(apierror.False) + ErrContextUnsupported = apierror.New(apierror.Unavailable, "receiver does not support cancellable delivery").WithRetryable(apierror.False) ) +// NewContextUnsupportedError identifies a host that cannot bind delivery to +// a caller context. SendContext must fail rather than invoke a legacy Send +// method that may block past the caller's lifecycle. +func NewContextUnsupportedError(hostID pid.HostID, nodeID pid.NodeID) apierror.Error { + return apierror.New(apierror.Unavailable, "receiver does not support cancellable delivery"). + WithRetryable(apierror.False). + WithDetails(attrs.NewBagFrom(map[string]any{"host_id": hostID, "node_id": nodeID})). + WithCause(ErrContextUnsupported) +} + // NewInvalidHostTypeError creates an error when host has invalid type. func NewInvalidHostTypeError(hostID pid.HostID, nodeID pid.NodeID) apierror.Error { return apierror.New(apierror.Internal, "invalid host type"). diff --git a/system/relay/node.go b/system/relay/node.go index 56733fa07..749fcb3b0 100644 --- a/system/relay/node.go +++ b/system/relay/node.go @@ -102,6 +102,9 @@ func (n *Node) send(ctx context.Context, pkg *api.Package) error { if sender, ok := receiver.(api.ContextSender); ok { return sender.SendContext(ctx, pkg) } + if ctx.Done() != nil { + return NewContextUnsupportedError(pkg.Target.Host, n.nodeID) + } return receiver.Send(pkg) } diff --git a/system/relay/node_test.go b/system/relay/node_test.go index 96bd0c436..948542426 100644 --- a/system/relay/node_test.go +++ b/system/relay/node_test.go @@ -5,8 +5,10 @@ package relay import ( "context" "errors" + "sync" "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,6 +38,24 @@ func (d *dummyHost) Detach(_ pidapi.PID) { // No-op for testing } +type blockingHost struct { + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (h *blockingHost) Send(_ *relay.Package) error { + h.once.Do(func() { close(h.entered) }) + <-h.release + return nil +} + +func (h *blockingHost) Attach(_ pidapi.PID, _ chan *relay.Package) (context.CancelFunc, error) { + return func() {}, nil +} + +func (h *blockingHost) Detach(_ pidapi.PID) {} + func TestNodeSendLocal(t *testing.T) { // Create a dummy host and register it with the node. dhost := &dummyHost{} @@ -71,6 +91,28 @@ func TestNodeSendLocal(t *testing.T) { assert.Equal(t, int32(2), dhost.sendCalled) } +func TestNodeSendContextRejectsBlockingLegacyReceiver(t *testing.T) { + host := &blockingHost{entered: make(chan struct{}), release: make(chan struct{})} + node := NewNode("node1") + require.NoError(t, node.RegisterHost("host1", host)) + pkg := &relay.Package{Target: pidapi.PID{Host: "host1", UniqID: "process"}} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + result := make(chan error, 1) + go func() { result <- node.SendContext(ctx, pkg) }() + select { + case err := <-result: + require.ErrorIs(t, err, ErrContextUnsupported) + case <-host.entered: + close(host.release) + t.Fatal("SendContext called a blocking legacy receiver") + case <-time.After(time.Second): + close(host.release) + t.Fatal("SendContext did not fail fast for legacy receiver") + } +} + func TestNodeSendHostNotFound(t *testing.T) { node := NewNode("node1") pid := pidapi.PID{ From 38317fce0dd156ae7b9891d6af6eff52d9df27fa Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:21:20 -0400 Subject: [PATCH 24/47] fix(sqlite): bound observer relay and pool usage --- api/service/sql/config.go | 10 +- api/service/sql/observer.go | 10 +- service/sql/engine/sqlite/observer.go | 201 ++++++++++++++++----- service/sql/engine/sqlite/observer_test.go | 91 +++++++++- service/sql/engine/sqlite/sqlite.go | 13 +- service/sql/engine/sqlite/sqlite_test.go | 14 +- 6 files changed, 284 insertions(+), 55 deletions(-) diff --git a/api/service/sql/config.go b/api/service/sql/config.go index 2f40fef88..6a7195197 100644 --- a/api/service/sql/config.go +++ b/api/service/sql/config.go @@ -38,8 +38,10 @@ const ( // DefaultMaxMutationChanges bounds the in-memory candidate row count held // by the SQLite observer for one transaction. DefaultMaxMutationChanges = 100000 - // DefaultMaxMutationBytes bounds the in-memory candidate value bytes held - // by the SQLite observer for one transaction. + // DefaultMaxMutationBytes is the conservative retained logical-byte bound + // for one SQLite transaction. SQLite delivers a complete native row to the + // pre-update hook, so one row can transiently materialize before the bound + // rejects the candidate. DefaultMaxMutationBytes = 64 * 1024 * 1024 ) @@ -78,7 +80,9 @@ type ( Lifecycle supervisor.LifecycleConfig `json:"lifecycle"` Pool PoolConfig `json:"pool"` MaxMutationChanges int `json:"max_mutation_changes,omitempty"` - MaxMutationBytes int `json:"max_mutation_bytes,omitempty"` + // MaxMutationBytes is a conservative retained logical-byte bound. A + // native SQLite row is materialized before the observer can reject it. + MaxMutationBytes int `json:"max_mutation_bytes,omitempty"` } ) diff --git a/api/service/sql/observer.go b/api/service/sql/observer.go index 5881f44fc..4d75ecb87 100644 --- a/api/service/sql/observer.go +++ b/api/service/sql/observer.go @@ -29,7 +29,10 @@ type MutationOptions struct { Tables []string Operations []string MaxChanges int - MaxBytes int + // MaxBytes bounds retained logical mutation bytes. A native SQLite row is + // materialized before the observer can reject it, so one row may + // transiently exceed this conservative bound. + MaxBytes int } // SnapshotOptions selects tables and the maximum number of rows in one @@ -38,7 +41,10 @@ type SnapshotOptions struct { Tables []string BatchSize int MaxChanges int - MaxBytes int + // MaxBytes bounds retained logical snapshot/live bytes. A native SQLite + // row is materialized before the observer can reject it, so one row may + // transiently exceed this conservative bound. + MaxBytes int } // MutationStream delivers committed mutation batches in commit order. diff --git a/service/sql/engine/sqlite/observer.go b/service/sql/engine/sqlite/observer.go index be6c913d4..9154fd4fa 100644 --- a/service/sql/engine/sqlite/observer.go +++ b/service/sql/engine/sqlite/observer.go @@ -63,6 +63,7 @@ func (c *sqliteConnector) Connect(context.Context) (driver.Conn, error) { state: &sqliteConnectionState{ backend: c.backend, sqlite: sqliteConn, maxChanges: c.backend.maxChanges, maxBytes: c.backend.maxBytes, + maxCommitEnds: c.backend.maxChanges, }, } // Install hooks for every physical connection when it is created. This @@ -88,20 +89,43 @@ func openSQLite(_ context.Context, dsn string, limits ...int) (*sql.DB, sqlapi.C } type sqliteBackend struct { - db *sql.DB - streams map[*mutationStream]struct{} - fence chan struct{} - maxChanges int - maxBytes int - sequence atomic.Uint64 - mu sync.Mutex - closed bool + relayWake chan struct{} + streams map[*mutationStream]struct{} + fence chan struct{} + db *sql.DB + relayDone chan struct{} + relayQueue []*backendBatch + maxChanges int + sequence atomic.Uint64 + maxBytes int + relayChanges int + relayBytes int + mu sync.Mutex + closed bool +} + +type backendBatch struct { + streams []*mutationStream + batch sqlapi.MutationBatch + bytes int } func newSQLiteBackend(maxChanges, maxBytes int) *sqliteBackend { fence := make(chan struct{}, 1) fence <- struct{}{} - return &sqliteBackend{streams: make(map[*mutationStream]struct{}), fence: fence, maxChanges: maxChanges, maxBytes: maxBytes} + backend := &sqliteBackend{ + streams: make(map[*mutationStream]struct{}), + fence: fence, + maxChanges: maxChanges, + maxBytes: maxBytes, + relayWake: make(chan struct{}, 1), + relayDone: make(chan struct{}), + } + go func() { + defer close(backend.relayDone) + backend.relay() + }() + return backend } func observerLimits(limits []int) (int, int) { @@ -134,19 +158,7 @@ func (b *sqliteBackend) releaseFence() { func (b *sqliteBackend) hasObservers() bool { b.mu.Lock() - active := !b.closed - if active { - active = false - for stream := range b.streams { - stream.mu.Lock() - closed := stream.closed - stream.mu.Unlock() - if !closed { - active = true - break - } - } - } + active := !b.closed && len(b.streams) > 0 b.mu.Unlock() return active } @@ -164,7 +176,7 @@ func (b *sqliteBackend) Subscribe(ctx context.Context, opts sqlapi.MutationOptio if err := b.validateTables(ctx, opts.Tables); err != nil { return nil, err } - stream := newMutationStream(b, opts) + stream := newMutationStream(ctx, b, opts) b.mu.Lock() if b.closed { b.mu.Unlock() @@ -309,7 +321,7 @@ func (b *sqliteBackend) Snapshot(ctx context.Context, opts sqlapi.SnapshotOption if opts.MaxBytes <= 0 { opts.MaxBytes = b.maxBytes } - stream := newSnapshotStream(b, opts, watermark, cancel) + stream := newSnapshotStream(scanCtx, b, opts, watermark, cancel) b.mu.Lock() if b.closed { b.mu.Unlock() @@ -492,20 +504,38 @@ func (b *sqliteBackend) publish(changes []sqlapi.Mutation) { b.mu.Unlock() return } + if len(b.streams) == 0 { + b.mu.Unlock() + return + } streams := make([]*mutationStream, 0, len(b.streams)) for stream := range b.streams { streams = append(streams, stream) } + changes = append([]sqlapi.Mutation(nil), changes...) sequence := b.sequence.Add(1) - b.mu.Unlock() - batch := sqlapi.MutationBatch{ Transaction: strconv.FormatUint(sequence, 10), Changes: changes, } - for _, stream := range streams { - stream.push(batch) + batchBytes := mutationBatchBytes(batch) + if (b.maxChanges > 0 && (len(changes) > b.maxChanges || b.relayChanges > b.maxChanges-len(changes))) || + (b.maxBytes > 0 && (batchBytes > b.maxBytes || b.relayBytes > b.maxBytes-batchBytes)) { + b.closed = true + b.streams = make(map[*mutationStream]struct{}) + b.relayQueue = nil + b.relayChanges = 0 + b.relayBytes = 0 + b.mu.Unlock() + b.closeStreams(streams, errObserverOverflow) + b.signalRelay() + return } + b.relayQueue = append(b.relayQueue, &backendBatch{batch: batch, streams: streams, bytes: batchBytes}) + b.relayChanges = saturatingAdd(b.relayChanges, len(changes)) + b.relayBytes = saturatingAdd(b.relayBytes, batchBytes) + b.mu.Unlock() + b.signalRelay() } func (b *sqliteBackend) fail(err error) { @@ -520,17 +550,20 @@ func (b *sqliteBackend) fail(err error) { streams = append(streams, stream) } b.streams = make(map[*mutationStream]struct{}) + b.relayQueue = nil + b.relayChanges = 0 + b.relayBytes = 0 b.mu.Unlock() - for _, stream := range streams { - stream.closeWithError(err) - } + b.closeStreams(streams, err) + b.signalRelay() } func (b *sqliteBackend) Close() error { b.mu.Lock() if b.closed { b.mu.Unlock() + <-b.relayDone return nil } b.closed = true @@ -539,12 +572,57 @@ func (b *sqliteBackend) Close() error { streams = append(streams, stream) } b.streams = make(map[*mutationStream]struct{}) + b.relayQueue = nil + b.relayChanges = 0 + b.relayBytes = 0 b.mu.Unlock() + b.closeStreams(streams, errObserverClosed) + b.signalRelay() + <-b.relayDone + return nil +} + +func (b *sqliteBackend) closeStreams(streams []*mutationStream, err error) { for _, stream := range streams { - stream.closeWithError(errObserverClosed) + stream.closeWithError(err) + } +} + +func (b *sqliteBackend) signalRelay() { + select { + case b.relayWake <- struct{}{}: + default: + } +} + +func (b *sqliteBackend) relay() { + for { + b.mu.Lock() + if len(b.relayQueue) == 0 { + closed := b.closed + b.mu.Unlock() + if closed { + return + } + <-b.relayWake + continue + } + item := b.relayQueue[0] + b.mu.Unlock() + + for _, stream := range item.streams { + stream.push(item.batch) + } + + b.mu.Lock() + if len(b.relayQueue) > 0 && b.relayQueue[0] == item { + b.relayQueue = b.relayQueue[1:] + b.relayChanges -= len(item.batch.Changes) + b.relayBytes -= item.bytes + } + b.mu.Unlock() } - return nil } // sqliteConnectionState is attached to one physical SQLite connection. The @@ -567,6 +645,7 @@ type sqliteConnectionState struct { maxChanges int pendingBytes int confirmedEnds int + maxCommitEnds int rollbackSeen bool ddlInTxn bool dmlInTxn bool @@ -726,6 +805,10 @@ func (s *sqliteConnectionState) commit() int { s.fenceHeld = true } s.commitPending = true + if s.maxCommitEnds > 0 && len(s.commitEnds) >= s.maxCommitEnds { + s.failed = errObserverOverflow + return 0 + } s.commitEnds = append(s.commitEnds, len(s.pending)) return 0 } @@ -1607,6 +1690,7 @@ func (r *observedRows) finish(err error) { type mutationStream struct { err error + ctx context.Context changes chan sqlapi.MutationBatch notify chan struct{} done chan struct{} @@ -1625,8 +1709,12 @@ type mutationStream struct { closed bool } -func newMutationStream(backend *sqliteBackend, opts sqlapi.MutationOptions) *mutationStream { +func newMutationStream(ctx context.Context, backend *sqliteBackend, opts sqlapi.MutationOptions) *mutationStream { + if ctx == nil { + ctx = context.Background() + } stream := &mutationStream{ + ctx: ctx, backend: backend, opts: opts, changes: make(chan sqlapi.MutationBatch), @@ -1639,8 +1727,8 @@ func newMutationStream(backend *sqliteBackend, opts sqlapi.MutationOptions) *mut return stream } -func newSnapshotStream(backend *sqliteBackend, opts sqlapi.SnapshotOptions, watermark string, cancel context.CancelFunc) *mutationStream { - stream := newMutationStream(backend, sqlapi.MutationOptions{ +func newSnapshotStream(ctx context.Context, backend *sqliteBackend, opts sqlapi.SnapshotOptions, watermark string, cancel context.CancelFunc) *mutationStream { + stream := newMutationStream(ctx, backend, sqlapi.MutationOptions{ Tables: opts.Tables, MaxChanges: opts.MaxChanges, MaxBytes: opts.MaxBytes, }) stream.snapshotting = true @@ -1675,19 +1763,23 @@ func (s *mutationStream) push(batch sqlapi.MutationBatch) { return } s.mu.Lock() - defer s.mu.Unlock() if s.closed { + s.mu.Unlock() return } + overflow := false if s.snapshotting { if !s.enqueuePendingLocked(batch) { s.closeLocked(errObserverOverflow) - return + overflow = true } - return - } - if !s.enqueueLocked(batch) { + } else if !s.enqueueLocked(batch) { s.closeLocked(errObserverOverflow) + overflow = true + } + s.mu.Unlock() + if overflow { + s.backend.remove(s, errObserverOverflow) } } @@ -1729,34 +1821,41 @@ func matchesValue(value string, filters []string) bool { func (s *mutationStream) pushSnapshot(batch sqlapi.MutationBatch) error { s.mu.Lock() - defer s.mu.Unlock() if s.closed { - if s.err != nil { - return s.err + err := s.err + s.mu.Unlock() + if err != nil { + return err } return errObserverClosed } if !s.enqueueLocked(batch) { s.closeLocked(errObserverOverflow) + s.mu.Unlock() + s.backend.remove(s, errObserverOverflow) return errObserverOverflow } + s.mu.Unlock() return nil } func (s *mutationStream) finishSnapshot(err error) { s.mu.Lock() - defer s.mu.Unlock() if s.closed { + s.mu.Unlock() return } if err != nil { s.closeLocked(err) + s.mu.Unlock() + s.backend.remove(s, err) return } s.snapshotting = false s.queue = append(s.queue, s.pending...) s.pending = nil s.signalLocked() + s.mu.Unlock() } func (s *mutationStream) Watermark() string { return s.watermark } @@ -1810,7 +1909,13 @@ func (s *mutationStream) relay() { close(s.changes) return } - <-s.notify + select { + case <-s.notify: + case <-s.ctx.Done(): + s.backend.remove(s, s.ctx.Err()) + close(s.changes) + return + } continue } batch := s.queue[0] @@ -1821,6 +1926,10 @@ func (s *mutationStream) relay() { case <-s.done: close(s.changes) return + case <-s.ctx.Done(): + s.backend.remove(s, s.ctx.Err()) + close(s.changes) + return } s.mu.Lock() diff --git a/service/sql/engine/sqlite/observer_test.go b/service/sql/engine/sqlite/observer_test.go index ffd42ab5d..6fee1c723 100644 --- a/service/sql/engine/sqlite/observer_test.go +++ b/service/sql/engine/sqlite/observer_test.go @@ -298,7 +298,7 @@ func TestObserverSnapshotHandoffIncludesInFlightWriterAsLive(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-inflight.db")) require.NoError(t, err) defer observed.Close() - observed.opened.DB.SetMaxOpenConns(2) + assert.Equal(t, 0, observed.opened.DB.Stats().MaxOpenConnections, "file-backed SQLite should retain the default unlimited pool") _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) require.NoError(t, err) tx, err := observed.opened.DB.BeginTx(context.Background(), nil) @@ -323,6 +323,77 @@ func TestObserverSnapshotHandoffIncludesInFlightWriterAsLive(t *testing.T) { } } +func TestObserverRemovesCancelledAndOverflowedStreams(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "stream-churn.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + backend := observed.opened.Observer.(*sqliteBackend) + + ctx, cancel := context.WithCancel(context.Background()) + cancelled, err := observed.opened.Observer.Subscribe(ctx, config.MutationOptions{}) + require.NoError(t, err) + cancel() + waitForStreamCount(t, backend, 0) + select { + case _, ok := <-cancelled.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("cancelled stream did not close") + } + + overflowed, err := observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1}) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'one')`) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'two')`) + require.NoError(t, err) + waitForStreamCount(t, backend, 0) + select { + case _, ok := <-overflowed.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("overflowed stream did not close") + } + assert.ErrorIs(t, overflowed.Err(), errObserverOverflow) +} + +func TestObserverRelayHandlesManyStreamsWithoutBlockingCommit(t *testing.T) { + observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "stream-scale.db")) + require.NoError(t, err) + defer observed.Close() + _, err = observed.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + backend := observed.opened.Observer.(*sqliteBackend) + const streamCount = 256 + for range streamCount { + _, err = observed.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1}) + require.NoError(t, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err = observed.opened.DB.ExecContext(ctx, `INSERT INTO items (id, value) VALUES (1, 'one')`) + require.NoError(t, err) + _, err = observed.opened.DB.ExecContext(ctx, `INSERT INTO items (id, value) VALUES (2, 'two')`) + require.NoError(t, err) + waitForStreamCount(t, backend, 0) +} + +func TestObserverBoundsCommitMarkers(t *testing.T) { + backend := newSQLiteBackend(2, config.DefaultMaxMutationBytes) + defer func() { _ = backend.Close() }() + state := &sqliteConnectionState{backend: backend, maxCommitEnds: 2} + + assert.Equal(t, 0, state.commit()) + assert.Equal(t, 0, state.commit()) + assert.Equal(t, 0, state.commit()) + assert.Len(t, state.commitEnds, 2) + assert.ErrorIs(t, state.failed, errObserverOverflow) + state.finalize() +} + func TestObserverAbortedStatementDoesNotPublishPartialRows(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "abort.db")) require.NoError(t, err) @@ -628,3 +699,21 @@ func receiveBatch(t *testing.T, stream interface { return config.MutationBatch{} } } + +func waitForStreamCount(t *testing.T, backend *sqliteBackend, want int) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + backend.mu.Lock() + count := len(backend.streams) + backend.mu.Unlock() + if count == want { + return + } + time.Sleep(time.Millisecond) + } + backend.mu.Lock() + count := len(backend.streams) + backend.mu.Unlock() + t.Fatalf("stream count = %d, want %d", count, want) +} diff --git a/service/sql/engine/sqlite/sqlite.go b/service/sql/engine/sqlite/sqlite.go index 68aadcdcb..754b9af31 100644 --- a/service/sql/engine/sqlite/sqlite.go +++ b/service/sql/engine/sqlite/sqlite.go @@ -96,8 +96,17 @@ func (engine) Tune(db *sql.DB, ec config.EngineConfig) { return } - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) + // A private in-memory database is scoped to one physical connection, so + // sharing it across a pool would create multiple unrelated databases. File + // databases, however, need the configured pool width so a snapshot read + // transaction does not consume the only writer connection. + if cfg.File == ":memory:" { + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + } else { + db.SetMaxOpenConns(cfg.Pool.MaxOpen) + db.SetMaxIdleConns(cfg.Pool.MaxIdle) + } db.SetConnMaxLifetime(cfg.Pool.MaxLifetime) } diff --git a/service/sql/engine/sqlite/sqlite_test.go b/service/sql/engine/sqlite/sqlite_test.go index 0cf2a616c..ad7222f50 100644 --- a/service/sql/engine/sqlite/sqlite_test.go +++ b/service/sql/engine/sqlite/sqlite_test.go @@ -66,10 +66,22 @@ func TestTuneSingleWriter(t *testing.T) { require.NoError(t, err) defer func() { _ = db.Close() }() - engine{}.Tune(db, &config.SQLiteConfig{Pool: config.PoolConfig{MaxLifetime: time.Hour}}) + engine{}.Tune(db, &config.SQLiteConfig{File: ":memory:", Pool: config.PoolConfig{MaxLifetime: time.Hour, MaxOpen: 4, MaxIdle: 4}}) assert.Equal(t, 1, db.Stats().MaxOpenConnections) } +func TestTuneHonorsFilePoolWidth(t *testing.T) { + db, err := sql.Open("sqlite3", "file:test-tune-pool?mode=memory&cache=shared") + require.NoError(t, err) + defer func() { _ = db.Close() }() + + engine{}.Tune(db, &config.SQLiteConfig{ + File: filepath.Join(t.TempDir(), "tune.db"), + Pool: config.PoolConfig{MaxOpen: 4, MaxIdle: 3, MaxLifetime: time.Hour}, + }) + assert.Equal(t, 4, db.Stats().MaxOpenConnections) +} + func TestValidateConfigType(t *testing.T) { require.NoError(t, engine{}.ValidateConfigType(&config.SQLiteConfig{File: ":memory:"})) err := engine{}.ValidateConfigType(&config.DBConfig{}) From 7e5e817d4b61604ffbd96769f1fc659305944ab2 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:22:07 -0400 Subject: [PATCH 25/47] fix(cdc/postgres): bound interleaved transactions and subscriptions --- api/service/cdc/config.go | 24 +++++ api/service/cdc/config_test.go | 14 +++ api/service/cdc/errors.go | 2 + service/cdc/postgres/config_decode_test.go | 4 + service/cdc/postgres/decoder.go | 69 +++++++++++---- service/cdc/postgres/decoder_test.go | 93 ++++++++++++++++++++ service/cdc/postgres/driver.go | 12 +-- service/cdc/postgres/limits.go | 20 ++++- service/cdc/postgres/manager.go | 10 ++- service/cdc/postgres/service.go | 45 ++++++++-- service/cdc/postgres/service_metrics_test.go | 10 +++ service/cdc/postgres/service_test.go | 18 +++- service/cdc/postgres/stream.go | 29 ++++++ service/cdc/postgres/stream_test.go | 27 ++++++ 14 files changed, 335 insertions(+), 42 deletions(-) diff --git a/api/service/cdc/config.go b/api/service/cdc/config.go index 6efa24490..342f4b819 100644 --- a/api/service/cdc/config.go +++ b/api/service/cdc/config.go @@ -24,6 +24,8 @@ const ( // are omitted. Zero in Config means "use this default", never unlimited. DefaultPostgresMaxTransactionChanges = 1_000_000 DefaultPostgresMaxTransactionBytes = 256 << 20 + DefaultPostgresMaxInflightChanges = 1_000_000 + DefaultPostgresMaxInflightBytes = 256 << 20 ) type Config struct { @@ -42,6 +44,8 @@ type Config struct { SnapshotFetchSize int `json:"snapshot_fetch_size,omitempty"` MaxTransactionChanges int `json:"max_transaction_changes,omitempty"` MaxTransactionBytes int64 `json:"max_transaction_bytes,omitempty"` + MaxInflightChanges int `json:"max_inflight_changes,omitempty"` + MaxInflightBytes int64 `json:"max_inflight_bytes,omitempty"` Temporary bool `json:"temporary,omitempty"` Snapshot bool `json:"snapshot,omitempty"` Streaming bool `json:"streaming,omitempty"` @@ -89,6 +93,12 @@ func (c *Config) Validate() error { if c.MaxTransactionBytes < 0 { return ErrInvalidMaxTransactionBytes } + if c.MaxInflightChanges < 0 { + return ErrInvalidMaxInflightChanges + } + if c.MaxInflightBytes < 0 { + return ErrInvalidMaxInflightBytes + } if _, err := c.StandbyDuration(); err != nil { return err } @@ -112,6 +122,20 @@ func (c *Config) EffectiveMaxTransactionBytes() int64 { return DefaultPostgresMaxTransactionBytes } +func (c *Config) EffectiveMaxInflightChanges() int { + if c.MaxInflightChanges > 0 { + return c.MaxInflightChanges + } + return DefaultPostgresMaxInflightChanges +} + +func (c *Config) EffectiveMaxInflightBytes() int64 { + if c.MaxInflightBytes > 0 { + return c.MaxInflightBytes + } + return DefaultPostgresMaxInflightBytes +} + func (c *Config) StandbyDuration() (time.Duration, error) { return parseInterval(c.StandbyInterval) } diff --git a/api/service/cdc/config_test.go b/api/service/cdc/config_test.go index 455122230..a266ce839 100644 --- a/api/service/cdc/config_test.go +++ b/api/service/cdc/config_test.go @@ -126,11 +126,17 @@ func TestConfigTransactionLimitsUseFiniteDefaults(t *testing.T) { c := validConfig() assert.Equal(t, DefaultPostgresMaxTransactionChanges, c.EffectiveMaxTransactionChanges()) assert.Equal(t, int64(DefaultPostgresMaxTransactionBytes), c.EffectiveMaxTransactionBytes()) + assert.Equal(t, DefaultPostgresMaxInflightChanges, c.EffectiveMaxInflightChanges()) + assert.Equal(t, int64(DefaultPostgresMaxInflightBytes), c.EffectiveMaxInflightBytes()) c.MaxTransactionChanges = 123 c.MaxTransactionBytes = 456 + c.MaxInflightChanges = 789 + c.MaxInflightBytes = 101112 assert.Equal(t, 123, c.EffectiveMaxTransactionChanges()) assert.Equal(t, int64(456), c.EffectiveMaxTransactionBytes()) + assert.Equal(t, 789, c.EffectiveMaxInflightChanges()) + assert.Equal(t, int64(101112), c.EffectiveMaxInflightBytes()) require.NoError(t, c.Validate()) } @@ -142,6 +148,14 @@ func TestConfigTransactionLimitsRejectNegative(t *testing.T) { c = validConfig() c.MaxTransactionBytes = -1 require.ErrorIs(t, c.Validate(), ErrInvalidMaxTransactionBytes) + + c = validConfig() + c.MaxInflightChanges = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxInflightChanges) + + c = validConfig() + c.MaxInflightBytes = -1 + require.ErrorIs(t, c.Validate(), ErrInvalidMaxInflightBytes) } func TestConfigFailoverRequiresPersistentSlot(t *testing.T) { diff --git a/api/service/cdc/errors.go b/api/service/cdc/errors.go index 725b82d58..098135a1e 100644 --- a/api/service/cdc/errors.go +++ b/api/service/cdc/errors.go @@ -17,6 +17,8 @@ var ( ErrInvalidSnapshotFetchSize = apierror.New(apierror.Invalid, "snapshot_fetch_size must be non-negative").WithRetryable(apierror.False) ErrInvalidMaxTransactionChanges = apierror.New(apierror.Invalid, "max_transaction_changes must be non-negative").WithRetryable(apierror.False) ErrInvalidMaxTransactionBytes = apierror.New(apierror.Invalid, "max_transaction_bytes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxInflightChanges = apierror.New(apierror.Invalid, "max_inflight_changes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxInflightBytes = apierror.New(apierror.Invalid, "max_inflight_bytes must be non-negative").WithRetryable(apierror.False) ErrDBResourceRequired = apierror.New(apierror.Invalid, "db_resource is required").WithRetryable(apierror.False) ErrSourceNotFound = apierror.New(apierror.NotFound, "cdc source not found").WithRetryable(apierror.False) ErrUnsupported = apierror.New(apierror.Invalid, "cdc operation is not supported by this source").WithRetryable(apierror.False) diff --git a/service/cdc/postgres/config_decode_test.go b/service/cdc/postgres/config_decode_test.go index 6d17353bd..7200cebc3 100644 --- a/service/cdc/postgres/config_decode_test.go +++ b/service/cdc/postgres/config_decode_test.go @@ -37,6 +37,8 @@ func TestConfigWireFormatMapsAndBuildsDSN(t *testing.T) { "snapshot": true, "max_transaction_changes": 1234, "max_transaction_bytes": 65536, + "max_inflight_changes": 2345, + "max_inflight_bytes": 131072, "standby_interval": "5s", "status_interval": "1m", "tables": []any{"public.accounts", "public.orders"}, @@ -52,6 +54,8 @@ func TestConfigWireFormatMapsAndBuildsDSN(t *testing.T) { assert.True(t, cfg.Snapshot) assert.Equal(t, 1234, cfg.MaxTransactionChanges) assert.Equal(t, int64(65536), cfg.MaxTransactionBytes) + assert.Equal(t, 2345, cfg.MaxInflightChanges) + assert.Equal(t, int64(131072), cfg.MaxInflightBytes) assert.Equal(t, "5s", cfg.StandbyInterval) assert.Equal(t, []string{"public.accounts", "public.orders"}, cfg.Tables) diff --git a/service/cdc/postgres/decoder.go b/service/cdc/postgres/decoder.go index 71765fc1d..37cb7e5ce 100644 --- a/service/cdc/postgres/decoder.go +++ b/service/cdc/postgres/decoder.go @@ -24,16 +24,18 @@ type decodeResult struct { } type decoder struct { - rels *relationCache - buffer map[uint32][]bufferedChange - usage map[uint32]int64 - limits decoderLimits - commitLSN pglogrepl.LSN - xid uint32 - curTopXid uint32 - streaming bool - inStream bool - txActive bool + rels *relationCache + buffer map[uint32][]bufferedChange + usage map[uint32]int64 + limits decoderLimits + commitLSN pglogrepl.LSN + inflightChanges int + inflightBytes int64 + xid uint32 + curTopXid uint32 + streaming bool + inStream bool + txActive bool } func newDecoder(limits ...decoderLimits) *decoder { @@ -229,8 +231,7 @@ func (d *decoder) flushTransaction(commitLSN pglogrepl.LSN) []RowChange { commitLSN = d.commitLSN } buffered := d.buffer[0] - delete(d.buffer, 0) - delete(d.usage, 0) + d.releaseBuffer(0) changes := make([]RowChange, 0, len(buffered)) for i := range buffered { buffered[i].rc.CommitLSN = commitLSN.String() @@ -252,6 +253,7 @@ func (d *decoder) one(op Op, relID uint32, oldT, newT *pglogrepl.TupleData, walS } rc, err := d.changeFor(op, relID, oldT, newT, walStart) if err != nil { + d.releaseReservation(0, 1, bytes) return decodeResult{}, err } rc.XID = d.xid @@ -306,6 +308,7 @@ func (d *decoder) bufferOne(subxid uint32, op Op, relID uint32, oldT, newT *pglo } rc, err := d.changeFor(op, relID, oldT, newT, walStart) if err != nil { + d.releaseReservation(d.curTopXid, 1, bytes) return err } rc.XID = d.curTopXid @@ -336,8 +339,7 @@ func (d *decoder) bufferTruncate(m *pglogrepl.TruncateMessageV2, walStart pglogr func (d *decoder) flushStream(topXid uint32, commitLSN pglogrepl.LSN) []RowChange { buffered := d.buffer[topXid] - delete(d.buffer, topXid) - delete(d.usage, topXid) + d.releaseBuffer(topXid) d.inStream = false d.curTopXid = 0 @@ -353,8 +355,7 @@ func (d *decoder) abortStream(topXid, subXid uint32) { d.inStream = false d.curTopXid = 0 if topXid == subXid { - delete(d.buffer, topXid) - delete(d.usage, topXid) + d.releaseBuffer(topXid) return } @@ -378,6 +379,10 @@ func (d *decoder) abortStream(topXid, subXid uint32) { // is nothing to truncate in that case. return } + oldChanges := len(src) + oldBytes := d.usage[topXid] + d.inflightChanges -= oldChanges + d.inflightBytes -= oldBytes // Clear the discarded tail before reslicing so decoded row maps and their // byte slices are no longer retained by the backing array. clear(src[cut:]) @@ -393,6 +398,28 @@ func (d *decoder) abortStream(topXid, subXid uint32) { bytes += bc.bytes } d.usage[topXid] = bytes + d.inflightChanges += len(src) + d.inflightBytes += bytes +} + +func (d *decoder) releaseBuffer(key uint32) { + buffered, exists := d.buffer[key] + if !exists { + return + } + d.inflightChanges -= len(buffered) + d.inflightBytes -= d.usage[key] + delete(d.buffer, key) + delete(d.usage, key) +} + +func (d *decoder) releaseReservation(key uint32, changes int, bytes int64) { + d.usage[key] -= bytes + d.inflightChanges -= changes + d.inflightBytes -= bytes + if d.usage[key] == 0 { + delete(d.usage, key) + } } func (d *decoder) reserveRow(key, relID uint32, oldT, newT *pglogrepl.TupleData) (int64, error) { @@ -438,7 +465,17 @@ func (d *decoder) reserve(key uint32, changes int, bytes int64) error { if bytes > d.limits.maxBytes-currentBytes { return fmt.Errorf("%w: bytes=%d limit=%d", ErrTransactionLimit, currentBytes+bytes, d.limits.maxBytes) } + if changes > d.limits.maxInflightChanges-d.inflightChanges { + return fmt.Errorf("%w: inflight_changes=%d limit=%d", ErrTransactionLimit, + d.inflightChanges+changes, d.limits.maxInflightChanges) + } + if bytes > d.limits.maxInflightBytes-d.inflightBytes { + return fmt.Errorf("%w: inflight_bytes=%d limit=%d", ErrTransactionLimit, + d.inflightBytes+bytes, d.limits.maxInflightBytes) + } d.usage[key] = currentBytes + bytes + d.inflightChanges += changes + d.inflightBytes += bytes return nil } diff --git a/service/cdc/postgres/decoder_test.go b/service/cdc/postgres/decoder_test.go index 0f3cd9e2f..24f3b8e92 100644 --- a/service/cdc/postgres/decoder_test.go +++ b/service/cdc/postgres/decoder_test.go @@ -264,6 +264,99 @@ func TestDecoderEnforcesTransactionByteLimitForStreamedSegments(t *testing.T) { require.ErrorIs(t, err, ErrTransactionLimit) } +func TestStreamingDecoderEnforcesAggregateChangeLimitAcrossXIDs(t *testing.T) { + d := newStreamingDecoder(decoderLimits{ + maxChanges: 100, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: 1, + maxInflightBytes: defaultMaxInflightBytes, + }) + _, err := d.apply(relV2(), 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "first@w.ai"), 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 200, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(200, "2", "second@w.ai"), 0x30) + require.ErrorIs(t, err, ErrTransactionLimit) + assert.Equal(t, 1, d.inflightChanges) + + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.Zero(t, d.inflightChanges) +} + +func TestStreamingDecoderEnforcesAggregateByteLimitAcrossXIDs(t *testing.T) { + rel := relV2() + rowBytes := estimateChangeBytes(&rel.RelationMessage, textTuple("1", "first@w.ai"), nil) + d := newStreamingDecoder(decoderLimits{ + maxChanges: 100, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: 100, + maxInflightBytes: rowBytes, + }) + _, err := d.apply(rel, 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "first@w.ai"), 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 200, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(200, "2", "second@w.ai"), 0x30) + require.ErrorIs(t, err, ErrTransactionLimit) + assert.Equal(t, rowBytes, d.inflightBytes) +} + +func TestStreamingDecoderEnforcesAggregateLimitAcrossOrdinaryAndStreamedTransactions(t *testing.T) { + d := newStreamingDecoder(decoderLimits{ + maxChanges: 100, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: 2, + maxInflightBytes: defaultMaxInflightBytes, + }) + _, err := d.apply(relV2(), 0) + require.NoError(t, err) + + _, err = d.apply(&pglogrepl.StreamStartMessageV2{Xid: 100, FirstSegment: 1}, 0) + require.NoError(t, err) + _, err = d.apply(insertV2(100, "1", "stream@w.ai"), 0x20) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.StreamStopMessageV2{}, 0) + require.NoError(t, err) + + // Protocol v2 may interleave a regular transaction while a streamed + // transaction remains buffered. Both must count against the same bound. + _, err = d.apply(&pglogrepl.BeginMessage{FinalLSN: 0x30, Xid: 7}, 0) + require.NoError(t, err) + _, err = d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("2", "ordinary@w.ai")}, 0x31) + require.NoError(t, err) + assert.Equal(t, 2, d.inflightChanges) + + _, err = d.apply(&pglogrepl.InsertMessage{RelationID: 42, Tuple: textTuple("3", "over-limit@w.ai")}, 0x32) + require.ErrorIs(t, err, ErrTransactionLimit) + assert.Equal(t, 2, d.inflightChanges) + + _, err = d.apply(&pglogrepl.CommitMessage{CommitLSN: 0x40}, 0) + require.NoError(t, err) + assert.Equal(t, 1, d.inflightChanges) + _, err = d.apply(&pglogrepl.StreamCommitMessageV2{Xid: 100, CommitLSN: 0x50}, 0) + require.NoError(t, err) + assert.Zero(t, d.inflightChanges) +} + func TestTupleToMapNullAndToast(t *testing.T) { rel := &pglogrepl.RelationMessage{Columns: []*pglogrepl.RelationMessageColumn{ {Name: "a"}, diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index d81dcfc8d..f61a4006c 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -67,6 +67,8 @@ func (Driver) Create(ctx context.Context, entry registry.Entry, deps cdcservice. SnapshotFetchSize: cfg.SnapshotFetchSize, MaxTransactionChanges: cfg.MaxTransactionChanges, MaxTransactionBytes: cfg.MaxTransactionBytes, + MaxInflightChanges: cfg.MaxInflightChanges, + MaxInflightBytes: cfg.MaxInflightBytes, Log: log.With(zap.String("id", entry.ID.String())), } return &sourceAdapter{ @@ -146,16 +148,8 @@ func (s *sourceAdapter) Subscribe(ctx context.Context, opts config.StreamOptions s.mu.RLock() source := s.source - source.mu.Lock() - state := source.state - source.mu.Unlock() - if state != sourceRunning { - s.mu.RUnlock() - return nil, config.ErrSourceNotReady - } - stream := source.Subscribe(opts) s.mu.RUnlock() - return stream, nil + return source.subscribe(ctx, opts) } func (s *sourceAdapter) Start(ctx context.Context) (<-chan any, error) { diff --git a/service/cdc/postgres/limits.go b/service/cdc/postgres/limits.go index b8bf98cab..38a6f97c8 100644 --- a/service/cdc/postgres/limits.go +++ b/service/cdc/postgres/limits.go @@ -11,17 +11,23 @@ const ( // while making the finite defaults owned by the public CDC configuration. defaultMaxTransactionChanges = config.DefaultPostgresMaxTransactionChanges defaultMaxTransactionBytes = config.DefaultPostgresMaxTransactionBytes + defaultMaxInflightChanges = config.DefaultPostgresMaxInflightChanges + defaultMaxInflightBytes = config.DefaultPostgresMaxInflightBytes ) type decoderLimits struct { - maxChanges int - maxBytes int64 + maxChanges int + maxBytes int64 + maxInflightChanges int + maxInflightBytes int64 } func defaultDecoderLimits() decoderLimits { return decoderLimits{ - maxChanges: defaultMaxTransactionChanges, - maxBytes: defaultMaxTransactionBytes, + maxChanges: defaultMaxTransactionChanges, + maxBytes: defaultMaxTransactionBytes, + maxInflightChanges: defaultMaxInflightChanges, + maxInflightBytes: defaultMaxInflightBytes, } } @@ -33,5 +39,11 @@ func normalizeDecoderLimits(limits decoderLimits) decoderLimits { if limits.maxBytes <= 0 { limits.maxBytes = defaults.maxBytes } + if limits.maxInflightChanges <= 0 { + limits.maxInflightChanges = defaults.maxInflightChanges + } + if limits.maxInflightBytes <= 0 { + limits.maxInflightBytes = defaults.maxInflightBytes + } return limits } diff --git a/service/cdc/postgres/manager.go b/service/cdc/postgres/manager.go index ce6b75cd3..d76796c1d 100644 --- a/service/cdc/postgres/manager.go +++ b/service/cdc/postgres/manager.go @@ -96,6 +96,8 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { SnapshotFetchSize: cfg.SnapshotFetchSize, MaxTransactionChanges: cfg.MaxTransactionChanges, MaxTransactionBytes: cfg.MaxTransactionBytes, + MaxInflightChanges: cfg.MaxInflightChanges, + MaxInflightBytes: cfg.MaxInflightBytes, Log: m.log.With(zap.String("id", entry.ID.String())), }) @@ -156,6 +158,8 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { SnapshotFetchSize: cfg.SnapshotFetchSize, MaxTransactionChanges: cfg.MaxTransactionChanges, MaxTransactionBytes: cfg.MaxTransactionBytes, + MaxInflightChanges: cfg.MaxInflightChanges, + MaxInflightBytes: cfg.MaxInflightBytes, Log: m.log.With(zap.String("id", entry.ID.String())), }) m.sources[entry.ID] = src @@ -228,7 +232,11 @@ func (m *Manager) Stream(_ context.Context, name string, opts config.StreamOptio if !ok { return nil, config.SourceInfo{}, NewServiceNotFoundError(registry.ParseID(name)) } - return src.Subscribe(opts), info, nil + stream := src.Subscribe(opts) + if stream == nil { + return nil, info, config.ErrSourceNotReady + } + return stream, info, nil } func (m *Manager) lookupSourceLocked(name string) (*Source, config.SourceInfo, bool) { diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 3c0cb04d2..841abe163 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -23,9 +23,10 @@ import ( ) const ( - retainedWALGauge = "wippy_cdc_retained_wal_bytes" - changesCounter = "wippy_cdc_changes_total" - errorsCounter = "wippy_cdc_errors_total" + retainedWALGauge = "wippy_cdc_retained_wal_bytes" + changesCounter = "wippy_cdc_changes_total" + errorsCounter = "wippy_cdc_errors_total" + transactionLimitCounter = "wippy_cdc_transaction_limit_total" ) const ( @@ -61,6 +62,12 @@ type SourceOptions struct { // MaxTransactionBytes bounds the estimated memory retained for one // ordinary or streamed transaction. Zero uses the safe default. MaxTransactionBytes int64 + // MaxInflightChanges bounds all uncommitted row changes across interleaved + // ordinary and streamed transactions. Zero uses the safe default. + MaxInflightChanges int + // MaxInflightBytes bounds estimated memory retained by all uncommitted + // ordinary and streamed transactions. Zero uses the safe default. + MaxInflightBytes int64 } type Source struct { @@ -83,6 +90,8 @@ type Source struct { snapshotFetchSize int maxTransactionChanges int maxTransactionBytes int64 + maxInflightChanges int + maxInflightBytes int64 subMu sync.RWMutex mu sync.Mutex dropMu sync.Mutex @@ -144,8 +153,10 @@ func NewSource(opts SourceOptions) *Source { fetch = defaultSnapshotFetchSize } limits := normalizeDecoderLimits(decoderLimits{ - maxChanges: opts.MaxTransactionChanges, - maxBytes: opts.MaxTransactionBytes, + maxChanges: opts.MaxTransactionChanges, + maxBytes: opts.MaxTransactionBytes, + maxInflightChanges: opts.MaxInflightChanges, + maxInflightBytes: opts.MaxInflightBytes, }) return &Source{ log: log, @@ -166,6 +177,8 @@ func NewSource(opts SourceOptions) *Source { snapshotFetchSize: fetch, maxTransactionChanges: limits.maxChanges, maxTransactionBytes: limits.maxBytes, + maxInflightChanges: limits.maxInflightChanges, + maxInflightBytes: limits.maxInflightBytes, } } @@ -383,7 +396,8 @@ func (s *Source) run( ) { defer func() { s.mu.Lock() - if s.done == done { + current := s.done == done + if current { switch s.state { case sourceStopping: s.state = sourceStopped @@ -393,10 +407,15 @@ func (s *Source) run( s.cancel = nil } s.mu.Unlock() + // A failed generation can finish after a supervisor has already + // started its replacement. Only the active generation owns the + // subscription set; an old run must never prune new subscribers. + if current { + s.closeSubscriptions() + } close(done) }() defer close(status) - defer s.closeSubscriptions() defer func() { _ = adminDB.Close() }() defer func() { _ = conn.Close(context.Background()) }() @@ -439,8 +458,10 @@ func (s *Source) run( } limits := decoderLimits{ - maxChanges: s.maxTransactionChanges, - maxBytes: s.maxTransactionBytes, + maxChanges: s.maxTransactionChanges, + maxBytes: s.maxTransactionBytes, + maxInflightChanges: s.maxInflightChanges, + maxInflightBytes: s.maxInflightBytes, } dec := newDecoder(limits) if s.streaming { @@ -624,10 +645,16 @@ func (s *Source) fail(_ context.Context, status chan any, err error) { } s.mu.Lock() s.sourceErr = err + if s.state == sourceRunning || s.state == sourceStarting { + s.state = sourceFailed + } s.mu.Unlock() s.log.Error("cdc stream error", zap.String("slot", s.slot), zap.Error(err)) if s.coll != nil { s.coll.CounterInc(errorsCounter, metrics.Labels{"source": s.name}) + if errors.Is(err, ErrTransactionLimit) { + s.coll.CounterInc(transactionLimitCounter, metrics.Labels{"source": s.name}) + } } s.closeSubscriptionsWithError(err) select { diff --git a/service/cdc/postgres/service_metrics_test.go b/service/cdc/postgres/service_metrics_test.go index af1ab0384..447604978 100644 --- a/service/cdc/postgres/service_metrics_test.go +++ b/service/cdc/postgres/service_metrics_test.go @@ -28,3 +28,13 @@ func TestSource_FailNilCollector(t *testing.T) { status := make(chan any, 1) s.fail(context.Background(), status, errors.New("boom")) } + +func TestSource_FailEmitsTransactionLimitCounter(t *testing.T) { + rec := telemetrytest.NewRecorder() + s := &Source{log: zap.NewNop(), name: "test:source", coll: rec} + + status := make(chan any, 1) + s.fail(context.Background(), status, ErrTransactionLimit) + + assert.Equal(t, 1.0, rec.CounterValue(transactionLimitCounter, metrics.Labels{"source": "test:source"})) +} diff --git a/service/cdc/postgres/service_test.go b/service/cdc/postgres/service_test.go index de4ac9e10..c333e4354 100644 --- a/service/cdc/postgres/service_test.go +++ b/service/cdc/postgres/service_test.go @@ -16,18 +16,30 @@ func TestNewSourceDefaults(t *testing.T) { assert.Equal(t, defaultStandbyInterval, s.standbyInterval) assert.Equal(t, defaultStatusInterval, s.statusInterval) assert.Equal(t, defaultSnapshotFetchSize, s.snapshotFetchSize) + assert.Equal(t, defaultMaxTransactionChanges, s.maxTransactionChanges) + assert.Equal(t, int64(defaultMaxTransactionBytes), s.maxTransactionBytes) + assert.Equal(t, defaultMaxInflightChanges, s.maxInflightChanges) + assert.Equal(t, int64(defaultMaxInflightBytes), s.maxInflightBytes) assert.NotNil(t, s.log) } func TestNewSourceHonorsOverrides(t *testing.T) { s := NewSource(SourceOptions{ - StandbyInterval: 1 * time.Second, - StatusInterval: 2 * time.Second, - SnapshotFetchSize: 4096, + StandbyInterval: 1 * time.Second, + StatusInterval: 2 * time.Second, + SnapshotFetchSize: 4096, + MaxTransactionChanges: 123, + MaxTransactionBytes: 456, + MaxInflightChanges: 789, + MaxInflightBytes: 101112, }) assert.Equal(t, 1*time.Second, s.standbyInterval) assert.Equal(t, 2*time.Second, s.statusInterval) assert.Equal(t, 4096, s.snapshotFetchSize) + assert.Equal(t, 123, s.maxTransactionChanges) + assert.Equal(t, int64(456), s.maxTransactionBytes) + assert.Equal(t, 789, s.maxInflightChanges) + assert.Equal(t, int64(101112), s.maxInflightBytes) } func TestStopBeforeStartIsSafe(t *testing.T) { diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 2bf2ea697..ae372b335 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -37,6 +37,35 @@ type sourceSubscription struct { } func (s *Source) Subscribe(opts config.StreamOptions) config.Stream { + s.mu.Lock() + defer s.mu.Unlock() + if s.state != sourceNew && s.state != sourceRunning { + return nil + } + return s.newSubscription(opts) +} + +// subscribe is the driver-facing subscription path. It holds the source +// lifecycle lock while registering the child, so Stop/fault cannot transition +// the source and close its current subscriptions between the state check and +// registration. +func (s *Source) subscribe(ctx context.Context, opts config.StreamOptions) (config.Stream, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + if s.state != sourceRunning || s.permanentlyClosed || s.sourceErr != nil { + return nil, config.ErrSourceNotReady + } + return s.newSubscription(opts), nil +} + +func (s *Source) newSubscription(opts config.StreamOptions) config.Stream { buffer := opts.Buffer if buffer <= 0 { buffer = defaultStreamBuffer diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index ff876388d..a93985c90 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -97,6 +97,33 @@ func TestSourceSubscriptionRetainsTerminalError(t *testing.T) { assert.ErrorIs(t, stream.(interface{ Err() error }).Err(), err) } +func TestSourceSubscriptionIsPrunedWhenStopWins(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + src.mu.Lock() + src.state = sourceRunning + src.mu.Unlock() + + stream, err := src.subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + require.NotNil(t, stream) + + require.NoError(t, src.Stop(context.Background())) + assert.Eventually(t, func() bool { + src.subMu.RLock() + defer src.subMu.RUnlock() + return len(src.subs) == 0 + }, time.Second, time.Millisecond) + + select { + case _, ok := <-stream.Changes(): + assert.False(t, ok) + case <-time.After(time.Second): + t.Fatal("stopped source left a live subscription") + } + _, err = src.subscribe(context.Background(), cdcapi.StreamOptions{}) + assert.ErrorIs(t, err, cdcapi.ErrSourceNotReady) +} + func TestSourceSubscriptionOverflowIsBoundedAndLocal(t *testing.T) { src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) laggard := src.Subscribe(cdcapi.StreamOptions{Buffer: 1}) From 6be8d0ca46e118002018bbfc76ce06f71a584c50 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:24:23 -0400 Subject: [PATCH 26/47] fix(cdc/postgres): bound subscriber queues to stream capacity --- service/cdc/postgres/stream.go | 38 +++++++++++++++------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index ae372b335..b1cf95efe 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -25,13 +25,13 @@ var errSubscriberOverflow = errors.New("postgres cdc subscriber backlog overflow type sourceSubscription struct { err error source *Source - in chan config.Change out chan config.Change done chan struct{} tables map[string]struct{} ops map[string]struct{} id uint64 once sync.Once + sendMu sync.Mutex closed atomic.Bool errMu sync.RWMutex } @@ -79,7 +79,8 @@ func (s *Source) newSubscription(opts config.StreamOptions) config.Stream { sub := &sourceSubscription{ source: s, id: s.nextSubID, - in: make(chan config.Change, buffer), + // out is the only event queue. sendMu serializes producers with + // terminal close so a slow consumer cannot retain a second buffer. out: make(chan config.Change, buffer), done: make(chan struct{}), tables: filterSet(opts.Tables), @@ -147,7 +148,12 @@ func (s *sourceSubscription) Err() error { func (s *sourceSubscription) closeWithError(err error) { s.once.Do(func() { + // Serialize the terminal transition with send. The run goroutine closes + // out under the same lock after done, so no producer can send to a + // closed channel. + s.sendMu.Lock() s.closed.Store(true) + s.sendMu.Unlock() if err != nil { s.errMu.Lock() s.err = err @@ -161,33 +167,23 @@ func (s *sourceSubscription) closeWithError(err error) { } func (s *sourceSubscription) run() { - defer close(s.out) - for { - select { - case <-s.done: - return - default: - } - select { - case change := <-s.in: - select { - case <-s.done: - return - case s.out <- change: - } - case <-s.done: - return - } - } + <-s.done + s.sendMu.Lock() + close(s.out) + s.sendMu.Unlock() } func (s *sourceSubscription) send(_ context.Context, change config.Change) { + s.sendMu.Lock() if s.closed.Load() { + s.sendMu.Unlock() return } select { - case s.in <- change: + case s.out <- change: + s.sendMu.Unlock() default: + s.sendMu.Unlock() // Never wait for a slow consumer from the replication goroutine. The // subscription gets a terminal error and is removed; other consumers // continue receiving the transaction. From 94cce6d018d9a729cb39283ceedb8d4192855b64 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:26:26 -0400 Subject: [PATCH 27/47] fix(cdc/postgres): close subscriber streams synchronously --- service/cdc/postgres/stream.go | 33 ++++++++++++----------------- service/cdc/postgres/stream_test.go | 26 +++++++++++++++++++---- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index b1cf95efe..b2ea38e5d 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -7,7 +7,6 @@ import ( "errors" "strings" "sync" - "sync/atomic" config "github.com/wippyai/runtime/api/service/cdc" ) @@ -26,13 +25,12 @@ type sourceSubscription struct { err error source *Source out chan config.Change - done chan struct{} tables map[string]struct{} ops map[string]struct{} id uint64 once sync.Once sendMu sync.Mutex - closed atomic.Bool + closed bool errMu sync.RWMutex } @@ -82,14 +80,12 @@ func (s *Source) newSubscription(opts config.StreamOptions) config.Stream { // out is the only event queue. sendMu serializes producers with // terminal close so a slow consumer cannot retain a second buffer. out: make(chan config.Change, buffer), - done: make(chan struct{}), tables: filterSet(opts.Tables), ops: filterSet(opts.Ops), } s.subs[sub.id] = sub s.subMu.Unlock() - go sub.run() return sub } @@ -148,34 +144,31 @@ func (s *sourceSubscription) Err() error { func (s *sourceSubscription) closeWithError(err error) { s.once.Do(func() { - // Serialize the terminal transition with send. The run goroutine closes - // out under the same lock after done, so no producer can send to a - // closed channel. - s.sendMu.Lock() - s.closed.Store(true) - s.sendMu.Unlock() + // Publish the terminal error before closing Changes. Err is therefore + // immediately observable when the caller receives the closed channel. if err != nil { s.errMu.Lock() s.err = err s.errMu.Unlock() } + // Serialize the terminal transition with send and close the event + // queue synchronously. No forwarding goroutine is needed, and no + // producer can send to a closed channel. + s.sendMu.Lock() + s.closed = true + close(s.out) + s.sendMu.Unlock() + // Detach outside sendMu: source removal takes subMu and must never + // participate in the producer/close critical section. if s.source != nil { s.source.removeSubscription(s.id) } - close(s.done) }) } -func (s *sourceSubscription) run() { - <-s.done - s.sendMu.Lock() - close(s.out) - s.sendMu.Unlock() -} - func (s *sourceSubscription) send(_ context.Context, change config.Change) { s.sendMu.Lock() - if s.closed.Load() { + if s.closed { s.sendMu.Unlock() return } diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index a93985c90..81bfa4eef 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -77,9 +77,10 @@ func TestSourceSubscriptionCloseReleasesChannel(t *testing.T) { select { case _, ok := <-stream.Changes(): assert.False(t, ok) - case <-time.After(time.Second): - t.Fatal("timed out waiting for closed cdc stream") + default: + t.Fatal("stream channel was not closed synchronously") } + assert.NoError(t, stream.Err()) } func TestSourceSubscriptionRetainsTerminalError(t *testing.T) { @@ -91,12 +92,29 @@ func TestSourceSubscriptionRetainsTerminalError(t *testing.T) { select { case _, ok := <-stream.Changes(): assert.False(t, ok) - case <-time.After(time.Second): - t.Fatal("stream did not close") + default: + t.Fatal("stream channel was not closed synchronously") } assert.ErrorIs(t, stream.(interface{ Err() error }).Err(), err) } +func TestSourceSubscriptionChurnDetachesImmediately(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + for i := 0; i < 1000; i++ { + stream := src.Subscribe(cdcapi.StreamOptions{Buffer: 1}) + stream.Close() + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + default: + t.Fatal("churned stream channel was not closed") + } + } + src.subMu.RLock() + defer src.subMu.RUnlock() + assert.Empty(t, src.subs) +} + func TestSourceSubscriptionIsPrunedWhenStopWins(t *testing.T) { src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) src.mu.Lock() From c1bfdfd6ddaf69672740c4c98581edc449a5eed4 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:40:22 -0400 Subject: [PATCH 28/47] fix(cdc): make source replacement lifecycle transactional --- service/cdc/dispatcher.go | 16 +- service/cdc/manager.go | 128 +++++++--- service/cdc/manager_test.go | 82 +++++-- service/cdc/slot.go | 330 ++++++++++++++++---------- system/supervisor/controller.go | 8 + system/supervisor/supervisor.go | 109 ++++++--- system/supervisor/supervisor_test.go | 168 +++++++++++++ system/supervisor/transaction.go | 43 +++- system/supervisor/transaction_test.go | 24 ++ 9 files changed, 687 insertions(+), 221 deletions(-) diff --git a/service/cdc/dispatcher.go b/service/cdc/dispatcher.go index 3957e097a..f56b12212 100644 --- a/service/cdc/dispatcher.go +++ b/service/cdc/dispatcher.go @@ -55,19 +55,19 @@ const ( // configured source manager. The dispatcher owns subscription relays; a // driver owns the source and its stream implementation. type Dispatcher struct { - ctx context.Context - log *zap.Logger - cancel context.CancelFunc - jobs chan dispatchJob - sessions map[uint64]*relaySession - stopDone chan struct{} - workersWG sync.WaitGroup - relaysWG sync.WaitGroup + ctx context.Context + log *zap.Logger + cancel context.CancelFunc + jobs chan dispatchJob + sessions map[uint64]*relaySession + stopDone chan struct{} // admissionsDone is a per-run barrier. Stop cancels workers first, then // waits for handles that already passed the state check before workers // drain the queue. This closes the Handle/Stop admission race without // holding mu across a potentially blocking queue send. admissionsDone chan struct{} + workersWG sync.WaitGroup + relaysWG sync.WaitGroup admissions int workers int nextID uint64 diff --git a/service/cdc/manager.go b/service/cdc/manager.go index c80630001..c0ba37e43 100644 --- a/service/cdc/manager.go +++ b/service/cdc/manager.go @@ -82,6 +82,12 @@ type ExclusiveResource interface { // the manager is built; package initialization must not mutate global routing. type Driver interface { Kind() registry.Kind + // Create validates the entry and returns an idle source. It must not start + // network, filesystem, or other durable resources; Start owns activation. + // Once a source is returned, the manager owns cleanup on every later + // registration/update failure. A driver that allocates while constructing + // must clean those allocations before returning an error because no source + // exists for the manager to reclaim. Create(context.Context, registry.Entry, Dependencies) (ManagedSource, error) } @@ -92,6 +98,8 @@ type Option func(*Manager) // the same kind intentionally replaces an earlier test or extension driver. func WithDriver(drivers ...Driver) Option { return func(m *Manager) { + m.mu.Lock() + defer m.mu.Unlock() for _, driver := range drivers { if driver == nil { continue @@ -110,8 +118,17 @@ type Manager struct { log *zap.Logger drivers map[registry.Kind]Driver leases map[string]resourceLease + ops map[registry.ID]*sourceOperation + // mu protects the injected driver map. Drivers are normally immutable after + // construction; keeping this lock makes test and extension replacement + // safe without holding it across driver calls. + mu sync.RWMutex + // leaseMu is deliberately separate from source operation locks. Lease + // release is synchronous at the resource cleanup commit point and never + // waits behind another source's network or filesystem work. + leaseMu sync.Mutex leaseSeq uint64 - mu sync.Mutex + opsMu sync.Mutex } type resourceLease struct { @@ -119,6 +136,11 @@ type resourceLease struct { token uint64 } +type sourceOperation struct { + mu sync.Mutex + refs int +} + func NewManager( reg Registry, dtt payload.Transcoder, @@ -147,6 +169,7 @@ func NewManager( log: log, drivers: make(map[registry.Kind]Driver), leases: make(map[string]resourceLease), + ops: make(map[registry.ID]*sourceOperation), } for _, opt := range opts { if opt != nil { @@ -157,14 +180,14 @@ func NewManager( } func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() + id := canonicalID(entry.ID) + release := m.lockSource(id) + defer release() - driver, ok := m.drivers[entry.Kind] + driver, ok := m.driver(entry.Kind) if !ok { return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) } - id := canonicalID(entry.ID) if _, exists := m.registry.Get(id); exists { return fmt.Errorf("%w: %s", ErrSourceExists, id.String()) } @@ -177,18 +200,18 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { return ErrDriverRequired } key := exclusiveResourceKey(source) - leaseToken, err := m.reserveLeaseLocked(key, id) + leaseToken, err := m.reserveLease(key, id) if err != nil { - _ = stopSource(ctx, source) + _ = stopUnstartedSource(ctx, source) return err } slot := newSourceSlot(id, entry.Kind, source, m.log.With(zap.String("id", id.String()))) slot.setRetiredCleanupHook(func(retiredKey string, retiredToken uint64) { - go m.releaseLease(id, retiredKey, retiredToken) + m.releaseLease(id, retiredKey, retiredToken) }) if err := m.registry.Register(id, slot, entry.Kind); err != nil { - m.releaseLeaseLocked(id, key, leaseToken) - _ = source.Stop(ctx) + m.releaseLease(id, key, leaseToken) + _ = stopUnstartedSource(ctx, source) return err } m.registerSupervisor(ctx, id, slot) @@ -197,10 +220,10 @@ func (m *Manager) Add(ctx context.Context, entry registry.Entry) error { } func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - id := canonicalID(entry.ID) + release := m.lockSource(id) + defer release() + existing, exists := m.registry.Get(id) if exists { existingKind, knownKind := sourceKind(existing) @@ -211,7 +234,7 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { return fmt.Errorf("%w: %s -> %s", ErrSourceKindChange, existingKind, entry.Kind) } } - driver, ok := m.drivers[entry.Kind] + driver, ok := m.driver(entry.Kind) if !ok { return fmt.Errorf("%w: %s", ErrUnsupportedKind, entry.Kind) } @@ -229,22 +252,22 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { } managedSlot, ok := existing.(*sourceSlot) if !ok { - _ = replacement.Stop(ctx) + _ = stopUnstartedSource(ctx, replacement) return errors.New("cdc manager: source is not managed by a stable slot") } oldKey := exclusiveResourceKey(managedSlot.currentSource()) newKey := exclusiveResourceKey(replacement) - oldToken := m.leaseTokenLocked(oldKey, id) - reservedNew := oldKey != newKey + oldToken := m.leaseToken(oldKey, id) + reservedNew := oldKey != newKey || (newKey != "" && oldToken == 0) newToken := oldToken if reservedNew { if managedSlot.hasRetiredKey(newKey) { - _ = stopSource(ctx, replacement) + _ = stopUnstartedSource(ctx, replacement) return ErrSourceBusy } - newToken, err = m.reserveLeaseLocked(newKey, id) + newToken, err = m.reserveLease(newKey, id) if err != nil { - _ = stopSource(ctx, replacement) + _ = stopUnstartedSource(ctx, replacement) return err } } @@ -256,7 +279,7 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { // healthy or deleted. committed := managedSlot.currentSource() == replacement if reservedNew && !committed { - m.releaseLeaseLocked(id, newKey, newToken) + m.releaseLease(id, newKey, newToken) } if committed { m.reconfigureSupervisorIfChanged(ctx, id, managedSlot, oldLifecycle) @@ -264,7 +287,7 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { return replaceErr } if oldKey != newKey && !managedSlot.hasRetiredKey(oldKey) { - m.releaseLeaseLocked(id, oldKey, oldToken) + m.releaseLease(id, oldKey, oldToken) } m.reconfigureSupervisorIfChanged(ctx, id, managedSlot, oldLifecycle) m.log.Info("updated cdc source", zap.String("id", id.String()), zap.String("kind", entry.Kind)) @@ -276,19 +299,19 @@ func (m *Manager) reconfigureSupervisorIfChanged(ctx context.Context, id registr if reflect.DeepEqual(old, newLifecycle) { return } - // ServiceUpdate is emitted by the supervisor for status changes and is - // not a reconfiguration primitive. Re-register the same stable slot in - // order, so its controller rebuilds security/dependency/autostart state - // without exposing a second source identity. + // ServiceUpdate is emitted by the supervisor for status changes and is not + // a reconfiguration primitive. Keep the canonical remove/register pair; + // the supervisor transaction retains both operations for a same-ID + // replacement and rebuilds the controller through its normal sequencer. m.unregisterSupervisor(ctx, id) m.registerSupervisorWithConfig(ctx, id, source, newLifecycle) } func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { - m.mu.Lock() - defer m.mu.Unlock() - id := canonicalID(entry.ID) + release := m.lockSource(id) + defer release() + source, ok := m.registry.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrSourceNotFound, id.String()) @@ -315,7 +338,7 @@ func (m *Manager) Delete(ctx context.Context, entry registry.Entry) error { } if slot, ok := source.(*sourceSlot); ok { for _, key := range slot.resourceKeys() { - m.releaseLeaseLocked(id, key, m.leaseTokenLocked(key, id)) + m.releaseLease(id, key, m.leaseToken(key, id)) } } m.unregisterSupervisor(ctx, id) @@ -331,6 +354,35 @@ func (m *Manager) Get(id registry.ID) (api.Source, bool) { return m.registry.Get(id) } +func (m *Manager) driver(kind registry.Kind) (Driver, bool) { + m.mu.RLock() + driver, ok := m.drivers[kind] + m.mu.RUnlock() + return driver, ok +} + +func (m *Manager) lockSource(id registry.ID) func() { + m.opsMu.Lock() + op := m.ops[id] + if op == nil { + op = &sourceOperation{} + m.ops[id] = op + } + op.refs++ + m.opsMu.Unlock() + + op.mu.Lock() + return func() { + m.opsMu.Lock() + op.refs-- + if op.refs == 0 { + delete(m.ops, id) + } + op.mu.Unlock() + m.opsMu.Unlock() + } +} + func (m *Manager) registerSupervisor(ctx context.Context, id registry.ID, source ManagedSource) { cfg := supervisor.LifecycleConfig{} if configured, ok := source.(interface { @@ -361,6 +413,12 @@ func (m *Manager) unregisterSupervisor(ctx context.Context, id registry.ID) { }) } +func (m *Manager) reserveLease(key string, id registry.ID) (uint64, error) { + m.leaseMu.Lock() + defer m.leaseMu.Unlock() + return m.reserveLeaseLocked(key, id) +} + func (m *Manager) reserveLeaseLocked(key string, id registry.ID) (uint64, error) { if key == "" { return 0, nil @@ -378,9 +436,9 @@ func (m *Manager) reserveLeaseLocked(key string, id registry.ID) (uint64, error) } func (m *Manager) releaseLease(id registry.ID, key string, token uint64) { - m.mu.Lock() + m.leaseMu.Lock() m.releaseLeaseLocked(id, key, token) - m.mu.Unlock() + m.leaseMu.Unlock() } func (m *Manager) releaseLeaseLocked(id registry.ID, key string, token uint64) { @@ -402,6 +460,12 @@ func (m *Manager) leaseTokenLocked(key string, id registry.ID) uint64 { return 0 } +func (m *Manager) leaseToken(key string, id registry.ID) uint64 { + m.leaseMu.Lock() + defer m.leaseMu.Unlock() + return m.leaseTokenLocked(key, id) +} + func stopSource(ctx context.Context, source api.Source) error { if isNilSource(source) { return nil diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go index 2482decc4..0d3c1a6ac 100644 --- a/service/cdc/manager_test.go +++ b/service/cdc/manager_test.go @@ -448,7 +448,10 @@ func TestManagerUpdateRejectsExclusiveResourceOwnedByAnotherID(t *testing.T) { second := registry.NewID("app", "second") old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-old"} other := &managedTestSource{info: api.SourceInfo{Name: "other"}, exclusive: "slot-new"} - candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-new"} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }} next := map[string]int{} driver := testDriver{ kind: kind, @@ -470,6 +473,7 @@ func TestManagerUpdateRejectsExclusiveResourceOwnedByAnotherID(t *testing.T) { require.ErrorIs(t, m.Update(context.Background(), registry.Entry{ID: first, Kind: kind}), ErrExclusiveOwned) require.Same(t, old, mustSlot(t, m, first).currentSource()) require.EqualValues(t, 1, candidate.stopCount.Load(), "a lease conflict must stop the uncommitted candidate") + require.EqualValues(t, 0, candidate.disposeCount.Load(), "an unstarted candidate must never destructively dispose a shared resource") } func TestManagerDeleteChecksEntryKind(t *testing.T) { @@ -567,24 +571,21 @@ func TestManagerUpdateRetriesFailedRetiredDisposalBeforeRestart(t *testing.T) { require.NoError(t, m.Add(context.Background(), entry)) require.EqualError(t, m.Update(context.Background(), entry), "retired cleanup failed") slot := mustSlot(t, m, id) - require.Same(t, candidate, slot.currentSource()) + require.Same(t, old, slot.currentSource(), "the candidate must not become visible before old disposal succeeds") require.Equal(t, slotFaulted, slot.state) require.EqualValues(t, 1, candidate.stopCount.Load()) require.Contains(t, m.leases, "slot-old") require.NoError(t, slot.Stop(context.Background())) - require.EqualValues(t, 2, old.disposeCount.Load()) - require.EqualValues(t, 2, candidate.stopCount.Load()) - _, err := slot.Start(context.Background()) - require.NoError(t, err) - require.EqualValues(t, 2, candidate.startCount.Load()) + require.EqualValues(t, 2, old.disposeCount.Load(), "shutdown Stop must retry the failed destructive cleanup") + require.EqualValues(t, 1, candidate.stopCount.Load()) require.Eventually(t, func() bool { - m.mu.Lock() - defer m.mu.Unlock() + m.leaseMu.Lock() + defer m.leaseMu.Unlock() _, ok := m.leases["slot-old"] return !ok }, time.Second, time.Millisecond) - require.NoError(t, slot.Stop(context.Background())) + require.Equal(t, slotStopped, slot.state) } func TestManagerDeleteRetriesRetiredDisposalBeforeUnregister(t *testing.T) { @@ -619,7 +620,7 @@ func TestManagerDeleteRetriesRetiredDisposalBeforeUnregister(t *testing.T) { func TestManagerRetiredLeaseTokenCannotReleaseReplacement(t *testing.T) { m, _ := newManagerTest(t) id := registry.NewID("app", "events") - m.mu.Lock() + m.leaseMu.Lock() m.leaseSeq = 1 m.leases["slot"] = resourceLease{id: id, token: 1} m.releaseLeaseLocked(id, "slot", 1) @@ -628,7 +629,7 @@ func TestManagerRetiredLeaseTokenCannotReleaseReplacement(t *testing.T) { require.NotEqual(t, uint64(1), newToken) m.releaseLeaseLocked(id, "slot", 1) owner, ok := m.leases["slot"] - m.mu.Unlock() + m.leaseMu.Unlock() require.True(t, ok) require.Equal(t, newToken, owner.token) } @@ -662,12 +663,17 @@ func TestManagerUpdateFailedStartRetainsRunningGeneration(t *testing.T) { require.Same(t, old, slot.current) require.Equal(t, slotRunning, slot.state) slot.mu.RUnlock() - require.EqualValues(t, 0, old.stopCount.Load()) + require.EqualValues(t, 1, old.stopCount.Load(), "the old running generation must be stopped before a candidate can start") + require.Equal(t, "2", slot.Info().Generation) } func TestManagerUpdateSameExclusiveKeyStopsAndRestores(t *testing.T) { old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-1"} - candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-1", startErr: errors.New("candidate start failed")} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-1", + startErr: errors.New("candidate start failed"), + }} next := 0 driver := testDriver{ kind: "db.cdc.test", @@ -695,9 +701,43 @@ func TestManagerUpdateSameExclusiveKeyStopsAndRestores(t *testing.T) { require.EqualValues(t, 1, old.stopCount.Load()) require.EqualValues(t, 2, old.startCount.Load(), "old generation must be restored after its initial start") require.EqualValues(t, 1, old.maxActive.Load(), "exclusive generations must never overlap") + require.EqualValues(t, 1, candidate.stopCount.Load(), "failed same-key candidate must be stopped") + require.EqualValues(t, 0, candidate.disposeCount.Load(), "failed same-key candidate must not dispose the old resource") require.Equal(t, "2", slot.Info().Generation, "restoring old ownership creates a new stream generation") } +func TestManagerUpdateDifferentResourceCleansSpeculativeCandidateDestructively(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-old", stopErr: errors.New("old stop failed")} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + _, err := mustSlot(t, m, id).Start(context.Background()) + require.NoError(t, err) + + require.EqualError(t, m.Update(context.Background(), entry), "old stop failed") + require.Same(t, old, mustSlot(t, m, id).currentSource()) + require.EqualValues(t, 1, candidate.startCount.Load()) + require.EqualValues(t, 1, candidate.disposeCount.Load(), "a started different-key candidate may be destructively cleaned") + require.EqualValues(t, 1, old.stopCount.Load()) +} + func TestManagerUpdateSameExclusiveKeyStopFailureFaultsSlot(t *testing.T) { old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-1", stopErr: errors.New("old stop failed")} candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-1"} @@ -752,6 +792,20 @@ func TestSourceSlotStampsCanonicalIdentityAndGeneration(t *testing.T) { require.NoError(t, slot.Stop(context.Background())) } +func TestSourceSlotRejectsSubscribeDuringReplacement(t *testing.T) { + id := registry.NewID("app", "events") + source := &managedTestSource{stream: &testStream{changes: make(chan api.Change, 1)}} + slot := newSourceSlot(id, "db.cdc.test", source) + slot.mu.Lock() + slot.state = slotRunning + slot.replacing = true + slot.mu.Unlock() + + stream, err := slot.Subscribe(context.Background(), api.StreamOptions{}) + require.Nil(t, stream) + require.ErrorIs(t, err, api.ErrSourceNotReady) +} + func TestSourceSlotInfoNormalizesLegacyAliases(t *testing.T) { id := registry.NewID("app", "events") source := &managedTestSource{info: api.SourceInfo{ diff --git a/service/cdc/slot.go b/service/cdc/slot.go index 6162a80cf..014056405 100644 --- a/service/cdc/slot.go +++ b/service/cdc/slot.go @@ -107,7 +107,7 @@ func (s *sourceSlot) Info() api.SourceInfo { func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api.Stream, error) { s.mu.RLock() - if s.state != slotRunning || isNilSource(s.current) || s.disposing { + if s.state != slotRunning || isNilSource(s.current) || s.disposing || s.replacing { s.mu.RUnlock() return nil, api.ErrSourceNotReady } @@ -123,7 +123,7 @@ func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api } s.mu.RLock() - stillCurrent := s.state == slotRunning && s.current == current && s.generation == generation + stillCurrent := s.state == slotRunning && s.current == current && s.generation == generation && !s.replacing s.mu.RUnlock() if !stillCurrent { stream.Close() @@ -147,6 +147,10 @@ func (s *sourceSlot) Start(ctx context.Context) (<-chan any, error) { s.mu.Unlock() return nil, ErrSourceBusy } + if len(s.retired) > 0 { + s.mu.Unlock() + return nil, ErrSourceBusy + } if s.state == slotRunning { status := s.status s.mu.Unlock() @@ -214,29 +218,30 @@ func (s *sourceSlot) Stop(ctx context.Context) error { defer s.opMu.Unlock() s.mu.Lock() - if s.disposing { - if (s.state == slotStopped || s.state == slotFaulted) && len(s.retired) == 0 { - s.mu.Unlock() - return nil - } - if s.state != slotStopped && s.state != slotFaulted { - s.mu.Unlock() - return ErrSourceBusy - } - } - if s.state == slotStopped && len(s.retired) == 0 { + if !s.disposing && s.state == slotStopped && len(s.retired) == 0 { s.mu.Unlock() return nil } s.state = slotStopping current := s.current cancel := s.runCancel + pendingCurrent := s.isRetiredLocked(current) + disposing := s.disposing s.mu.Unlock() if cancel != nil { cancel() } - err := stopSource(ctx, current) + var err error + if disposing && !pendingCurrent { + if disposable, ok := current.(Disposable); ok { + err = disposable.Dispose(ctx) + } else { + err = stopSource(ctx, current) + } + } else { + err = stopSource(ctx, current) + } if err == nil { err = s.retryRetired(ctx) } @@ -270,6 +275,7 @@ func (s *sourceSlot) Dispose(ctx context.Context) error { s.state = slotStopping current := s.current cancel := s.runCancel + pendingCurrent := s.isRetiredLocked(current) s.mu.Unlock() if cancel != nil { @@ -278,10 +284,14 @@ func (s *sourceSlot) Dispose(ctx context.Context) error { var err error if isNilSource(current) { err = ErrSourceClosed - } else if disposable, ok := current.(Disposable); ok { - err = disposable.Dispose(ctx) + } else if !pendingCurrent { + if disposable, ok := current.(Disposable); ok { + err = disposable.Dispose(ctx) + } else { + err = stopSource(ctx, current) + } } else { - err = stopSource(ctx, current) + err = nil } if err == nil { err = s.retryRetired(ctx) @@ -314,127 +324,159 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir s.opMu.Lock() defer s.opMu.Unlock() - s.mu.RLock() - old := s.current - state := s.state - runCtx := s.runCtx - runCancel := s.runCancel - disposing := s.disposing - hasRetired := len(s.retired) > 0 - s.mu.RUnlock() - if disposing { + s.mu.Lock() + if s.disposing || len(s.retired) > 0 { + s.mu.Unlock() + _ = stopUnstartedSource(ctx, candidate) return ErrSourceBusy } - if hasRetired { + old := s.current + oldState := s.state + oldRunCancel := s.runCancel + if oldState == slotStopping { + s.mu.Unlock() + _ = stopUnstartedSource(ctx, candidate) return ErrSourceBusy } + s.replacing = true + s.mu.Unlock() + retiredToken := uint64(0) if len(retiredTokens) > 0 { retiredToken = retiredTokens[0] } - - startCandidate := state == slotRunning || lifecycleAutoStart(candidate) oldKey := exclusiveResourceKey(old) candidateKey := exclusiveResourceKey(candidate) - sameExclusive := oldKey != "" && oldKey == candidateKey - var underlying <-chan any - var err error - oldStopped := false - createdRunContext := false - keepRunContext := false - defer func() { - if createdRunContext && !keepRunContext && runCancel != nil { + differentResource := oldKey != candidateKey + startCandidate := oldState == slotRunning || lifecycleAutoStart(candidate) + shouldStopOld := !isNilSource(old) && (oldState != slotStopped && oldState != slotIdle || differentResource || oldKey == "") + + var ( + underlying <-chan any + runCtx context.Context + runCancel context.CancelFunc + ) + speculative := differentResource && startCandidate + startCandidateGeneration := func() error { + runCtx, runCancel = detachedContext(ctx) + var err error + underlying, err = startSource(ctx, runCtx, candidate) + if err != nil { + _ = cleanupStartedSource(ctx, candidate, differentResource) runCancel() } - }() - if startCandidate && state == slotRunning { - s.mu.Lock() - s.replacing = true - s.mu.Unlock() + return err + } + if speculative { + // Different resource keys may be prepared in parallel. The candidate + // remains private until old Stop and Dispose both commit below. + if err := startCandidateGeneration(); err != nil { + return s.resetReplaceFailure(oldState, err) + } } - if sameExclusive && state == slotRunning { - // A source such as PostgreSQL may not start a second generation while - // the old generation owns the same slot. Stop the old generation first; - // if the candidate fails, restore the old generation before returning. - if err := stopSource(ctx, old); err != nil { + + // The old source is stopped before destructive cleanup. A speculative + // candidate may already be running, but it is not visible through the slot + // until this handoff has completed. + oldStopped := !shouldStopOld + if shouldStopOld { + if err := stopGeneration(ctx, old, oldRunCancel); err != nil { + if speculative { + runCancel() + _ = cleanupStartedSource(ctx, candidate, differentResource) + } s.mu.Lock() - s.replacing = false s.state = slotFaulted + s.replacing = false s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil s.mu.Unlock() + if !speculative { + _ = stopUnstartedSource(ctx, candidate) + } return err } oldStopped = true + s.mu.Lock() + s.state = slotStopped + s.runCtx = nil + s.runCancel = nil + s.closeStatusLocked() + s.mu.Unlock() } - if startCandidate { - if runCtx == nil { - runCtx, runCancel = detachedContext(ctx) - createdRunContext = true - } - underlying, err = startSource(ctx, runCtx, candidate) - if err != nil { - if sameExclusive { - _ = stopSource(ctx, candidate) - } else { - _ = cleanupSource(ctx, candidate) + + if differentResource && !isNilSource(old) { + if disposable, ok := old.(Disposable); ok { + if err := disposable.Dispose(ctx); err != nil { + // Keep the old source current and retain its lease. Stop retries + // this pending destructive cleanup during shutdown or delete. + s.recordRetired(old, oldKey, retiredToken) + s.mu.Lock() + s.state = slotFaulted + s.replacing = false + s.mu.Unlock() + if speculative { + runCancel() + } + _ = cleanupStartedSource(ctx, candidate, speculative && differentResource) + return err + } + if oldKey != "" { + s.mu.RLock() + hook := s.retiredHook + s.mu.RUnlock() + if hook != nil { + hook(oldKey, retiredToken) + } } - if sameExclusive { - var restartErr error - underlying, restartErr = startSource(ctx, runCtx, old) - if restartErr == nil { + } + } + + if startCandidate && !speculative { + if err := startCandidateGeneration(); err != nil { + _, oldDisposable := old.(Disposable) + restoreOld := oldState == slotRunning && (!differentResource || !oldDisposable) + if restoreOld && oldStopped { + // The old generation still owns the shared resource. Restore it + // before making the failed update visible to the caller. + restoreCtx, restoreCancel := detachedContext(ctx) + oldUpdates, restoreErr := startSource(ctx, restoreCtx, old) + if restoreErr == nil { s.mu.Lock() s.state = slotRunning s.generation++ + s.runCtx = restoreCtx + s.runCancel = restoreCancel s.replacing = false generation := s.generation + if s.status == nil || s.statusDone { + s.status = make(chan any, 8) + s.statusDone = false + } s.mu.Unlock() - s.watchStatus(old, generation, underlying) + s.watchStatus(old, generation, oldUpdates) return err } - s.mu.Lock() - s.state = slotFaulted - s.replacing = false - s.closeStatusLocked() - s.mu.Unlock() - return errors.Join(err, restartErr) + restoreCancel() + return s.finishReplaceFailure(errors.Join(err, restoreErr)) } - s.mu.Lock() - s.replacing = false - s.mu.Unlock() - return err + return s.finishReplaceFailure(err) } } - if !sameExclusive && !oldStopped && !isNilSource(old) { - // Candidate startup is deliberately speculative. The stable slot and - // registry continue to expose the old generation until its non- - // destructive Stop succeeds, so a failed handoff cannot publish an - // unowned or half-stopped replacement. - if err := stopSource(ctx, old); err != nil { - if sameExclusive { - _ = stopSource(ctx, candidate) - } else { - _ = cleanupSource(ctx, candidate) - } - s.mu.Lock() - s.state = slotFaulted - s.closeStatusLocked() - s.replacing = false - s.mu.Unlock() - return err - } - oldStopped = true - } s.mu.Lock() - if s.state == slotStopping { + if s.disposing || s.state == slotStopping { s.mu.Unlock() - _ = cleanupSource(ctx, candidate) + if runCancel != nil { + runCancel() + } + _ = cleanupStartedSource(ctx, candidate, startCandidate && differentResource) return ErrSourceBusy } s.current = candidate s.generation++ if startCandidate { - keepRunContext = true if s.status == nil || s.statusDone { s.status = make(chan any, 8) s.statusDone = false @@ -442,37 +484,62 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir s.state = slotRunning s.runCtx = runCtx s.runCancel = runCancel - s.replacing = false - generation := s.generation - s.mu.Unlock() - s.watchStatus(candidate, generation, underlying) + } else if oldState == slotIdle { + s.state = slotIdle } else { - s.mu.Unlock() + s.state = slotStopped } + s.replacing = false + generation := s.generation + s.mu.Unlock() - if old != nil && oldStopped && !sameExclusive { - if disposable, ok := old.(Disposable); ok { - if err := disposable.Dispose(ctx); err != nil { - s.recordRetired(old, oldKey, retiredToken) - if runCancel != nil { - runCancel() - } - _ = stopSource(ctx, candidate) - s.mu.Lock() - s.state = slotFaulted - s.closeStatusLocked() - s.runCtx = nil - s.runCancel = nil - s.replacing = false - s.mu.Unlock() - return err - } - } + if startCandidate { + s.watchStatus(candidate, generation, underlying) } return nil } -func cleanupSource(ctx context.Context, source api.Source) error { +func (s *sourceSlot) finishReplaceFailure(err error) error { + s.mu.Lock() + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil + s.replacing = false + s.mu.Unlock() + return err +} + +func (s *sourceSlot) resetReplaceFailure(state slotState, err error) error { + s.mu.Lock() + s.state = state + s.replacing = false + s.mu.Unlock() + return err +} + +// stopUnstartedSource abandons a source returned by Driver.Create before its +// Start method has successfully handed ownership of a durable resource to the +// manager. Drivers must make Create side-effect-free; Stop is intentionally +// the only cleanup allowed on this path so an unstarted candidate cannot drop +// a shared replication slot/checkpoint. +func stopUnstartedSource(ctx context.Context, source api.Source) error { + return stopSource(ctx, source) +} + +// cleanupStartedSource cleans a candidate after Start was attempted. A +// different exclusive resource is owned solely by that candidate and may be +// destructively disposed. Same-key candidates share the old resource contract +// and must only be stopped; disposing them could drop the old generation's +// resource. +func cleanupStartedSource(ctx context.Context, source api.Source, destructive bool) error { + if destructive { + return disposeSource(ctx, source) + } + return stopSource(ctx, source) +} + +func disposeSource(ctx context.Context, source api.Source) error { if isNilSource(source) { return nil } @@ -482,6 +549,13 @@ func cleanupSource(ctx context.Context, source api.Source) error { return stopSource(ctx, source) } +func stopGeneration(ctx context.Context, source api.Source, cancel context.CancelFunc) error { + if cancel != nil { + cancel() + } + return stopSource(ctx, source) +} + func (s *sourceSlot) recordRetired(source ManagedSource, key string, token uint64) { if isNilSource(source) { return @@ -491,6 +565,18 @@ func (s *sourceSlot) recordRetired(source ManagedSource, key string, token uint6 s.mu.Unlock() } +func (s *sourceSlot) isRetiredLocked(source ManagedSource) bool { + if isNilSource(source) { + return false + } + for _, retired := range s.retired { + if retired.source == source { + return true + } + } + return false +} + func (s *sourceSlot) retryRetired(ctx context.Context) error { for { s.mu.RLock() @@ -501,7 +587,7 @@ func (s *sourceSlot) retryRetired(ctx context.Context) error { retired := s.retired[0] s.mu.RUnlock() - if err := cleanupSource(ctx, retired.source); err != nil { + if err := disposeSource(ctx, retired.source); err != nil { return err } diff --git a/system/supervisor/controller.go b/system/supervisor/controller.go index aa3c985f5..48cd46d4b 100644 --- a/system/supervisor/controller.go +++ b/system/supervisor/controller.go @@ -120,6 +120,14 @@ func (c *Controller) cancelStart() { } } +// close releases the controller's supervision context after the service has +// been stopped or detached from the supervisor. Stop intentionally does not +// cancel this context because callers may retry a failed stop; once a +// controller is replaced or removed there is no future lifecycle work for it. +func (c *Controller) close() { + c.cancel() +} + func (c *Controller) setStartCancel(cancel context.CancelFunc) { c.startMu.Lock() c.startCancel = cancel diff --git a/system/supervisor/supervisor.go b/system/supervisor/supervisor.go index 02dd16f51..ed1a18e5c 100644 --- a/system/supervisor/supervisor.go +++ b/system/supervisor/supervisor.go @@ -242,6 +242,9 @@ func (s *Supervisor) StopContext(ctx context.Context) error { s.stopErr = ctx.Err() } } + for _, ctrl := range controllers { + ctrl.close() + } s.wg.Wait() @@ -619,8 +622,12 @@ func (s *Supervisor) resolveServiceDependencyRefs( return services, blockers, nil } -// execute processes the transaction by creating new services, -// stopping removed services, and starting auto-start services. +// execute processes a registry transaction through the normal lifecycle +// sequencer. A remove/register pair for one ID is a replacement: the old +// controller is stopped and detached before the new controller is created. +// This keeps controller configuration, dependency ordering, and desired-state +// handling in one lifecycle path instead of introducing an out-of-band +// reconfiguration operation. // // All iterations of tx.register, tx.remove, and s.controllers traverse a // pre-sorted slice of IDs. The supervisor feeds the sequencer in this order, @@ -630,9 +637,57 @@ func (s *Supervisor) resolveServiceDependencyRefs( func (s *Supervisor) execute(ctx context.Context, tx *regTx) (err error) { registerIDs := sortedRegisterIDs(tx.register) removeIDs := sortedRemoveIDs(tx.remove) + oldControllers := s.snapshotControllers() + oldStates := make(map[string]State, len(removeIDs)) + for _, id := range removeIDs { + if ctrl := oldControllers[id]; ctrl != nil { + oldStates[id] = ctrl.State() + } + } + + // Stop removed controllers first. In particular, a replacement must not + // expose a new controller while the old service still owns resources or + // dependency edges. A stop failure leaves the old registry untouched and + // the transaction can be retried by the caller. + stopOperations := make([]operation, 0, len(removeIDs)) + for _, id := range removeIDs { + if ctrl := oldControllers[id]; ctrl != nil { + deps, resolveErr := s.resolveDependencies(oldControllers, id) + if resolveErr != nil { + return NewDependencyResolveError(id, resolveErr) + } + stopOperations = append(stopOperations, operation{ + kind: opStop, + id: id, + controller: ctrl, + dependencies: deps, + }) + } + } + if err := s.runTransition(ctx, stopOperations); err != nil { + return NewTransitionError(err) + } + + // Detach successfully stopped controllers before constructing replacements. + // Closing the controller context releases its frame/supervision resources; + // Stop alone intentionally leaves a controller retryable for callers that + // need to recover a failed stop. + s.mu.Lock() + for _, id := range removeIDs { + if old := oldControllers[id]; old != nil && s.controllers[id] == old { + delete(s.controllers, id) + } + } + s.mu.Unlock() + for _, id := range removeIDs { + if old := oldControllers[id]; old != nil { + old.close() + } + } - // Mutate controller registry under lock, then run potentially long transitions - // lock-free so state readers are never blocked behind start/stop timeouts. + // Construct new controllers only after the stop phase has committed. This + // is also what makes remove/register a true generation handoff rather than + // silently retaining the old controller. created := make(map[string]*Controller, len(registerIDs)) s.mu.Lock() for _, id := range registerIDs { @@ -665,31 +720,24 @@ func (s *Supervisor) execute(ctx context.Context, tx *regTx) (err error) { }() controllers := s.snapshotControllers() - var operations []operation - - // Queue stop operations for services being removed - for _, id := range removeIDs { - if ctrl, exists := controllers[id]; exists { - deps, err := s.resolveDependencies(controllers, id) - if err != nil { - return NewDependencyResolveError(id, err) - } - operations = append(operations, operation{ - kind: opStop, - id: id, - controller: ctrl, - dependencies: deps, - }) - } - } - roots := make([]startRoot, 0, len(registerIDs)) for _, id := range registerIDs { entry := tx.register[id] - if entry.Config.AutoStart { + shouldStart := entry.Config.AutoStart + required := entry.Config.StartupRequired() + // Preserve a running/desired-running generation across a config or + // service replacement even when the replacement's AutoStart is false. + // This is an update of an active service, not an implicit user stop. + if previous, ok := oldStates[id]; ok && + (previous.Desired == supervisor.StatusRunning || + previous.Status == supervisor.StatusRunning || + previous.Status == supervisor.StatusStarting) { + shouldStart = true + } + if shouldStart { roots = append(roots, startRoot{ id: id, - required: entry.Config.StartupRequired(), + required: required, }) } } @@ -697,20 +745,13 @@ func (s *Supervisor) execute(ctx context.Context, tx *regTx) (err error) { if err != nil { return NewStartOperationsError(err) } - operations = append(operations, startOps...) - // Execute transitions in dependency order - if err := s.runTransition(ctx, operations); err != nil { + // Start additions and replacements through the same dependency-aware + // sequencer used by explicit ServiceStart actions. + if err := s.runTransition(ctx, startOps); err != nil { return NewTransitionError(err) } - // Done stopped services - s.mu.Lock() - for _, id := range removeIDs { - delete(s.controllers, id) - } - s.mu.Unlock() - return nil } diff --git a/system/supervisor/supervisor_test.go b/system/supervisor/supervisor_test.go index 4cd945197..7c95d3622 100644 --- a/system/supervisor/supervisor_test.go +++ b/system/supervisor/supervisor_test.go @@ -35,6 +35,17 @@ type testService struct { stopped bool } +type dependencyCheckingService struct { + *testService + dependency *testService + dependencyReady atomic.Bool +} + +func (s *dependencyCheckingService) Start(ctx context.Context) (<-chan any, error) { + s.dependencyReady.Store(s.dependency.IsStarted()) + return s.testService.Start(ctx) +} + type blockingStartService struct { startedCh chan struct{} releaseCh chan struct{} @@ -326,6 +337,163 @@ func TestSupervisor_BasicLifecycle(t *testing.T) { h.assertLog("supervisor stopped") } +func TestSupervisor_SameIDReplacementUsesNormalLifecycleTransaction(t *testing.T) { + h := newTestHarness(t) + h.start(context.Background()) + + h.registerServices(map[string]bool{ + "dependency": false, + "service": true, + }) + old := h.services["service"] + old.WaitForStart(t) + oldController := func() *Controller { + h.sup.mu.RLock() + defer h.sup.mu.RUnlock() + return h.sup.controllers["service"] + }() + + dependency := h.services["dependency"] + replacementBase := newTestService() + replacement := &dependencyCheckingService{ + testService: replacementBase, + dependency: dependency, + } + + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxBegin}) + h.sup.handleEvent(event.Event{System: supervisor.System, Kind: supervisor.ServiceRemove, Path: "service"}) + h.sup.handleEvent(event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRegister, + Path: "service", + Data: &supervisor.Entry{ + Service: replacement, + Config: supervisor.LifecycleConfig{ + AutoStart: false, + Requires: []string{"dependency"}, + StartTimeout: time.Second, + StopTimeout: time.Second, + }, + }, + }) + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxCommit}) + + replacement.WaitForStart(t) + dependency.WaitForStart(t) + old.WaitForStop(t) + require.True(t, replacement.dependencyReady.Load(), "replacement must start after its required dependency") + + h.sup.mu.RLock() + newController := h.sup.controllers["service"] + h.sup.mu.RUnlock() + require.NotSame(t, oldController, newController) + require.Same(t, replacement, newController.service) + require.False(t, newController.config.AutoStart) + state, err := h.sup.GetState("service") + require.NoError(t, err) + require.Equal(t, supervisor.StatusRunning, state.Status, "replacement preserves a running generation") + select { + case <-oldController.ctx.Done(): + default: + t.Fatal("replaced controller supervision context was not released") + } + + h.stop() +} + +func TestSupervisor_SameIDReplacementStopFailureRetainsOldController(t *testing.T) { + old := newTestService() + replacement := newTestService() + oldController := NewController(context.Background(), old, supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + }, nil) + defer oldController.close() + require.NoError(t, oldController.Start()) + old.stopErr = errors.New("old stop failed") + + bus := eventbus.NewBus() + defer bus.Stop() + sup := NewSupervisor(bus, zap.NewNop()) + id := "service" + sup.controllers[id] = oldController + tx := newRegTx(zap.NewNop()) + tx.open = true + tx.remove[id] = struct{}{} + tx.register[id] = &supervisor.Entry{ + Service: replacement, + Config: supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + RetryPolicy: supervisor.RetryPolicy{MaxAttempts: 1}, + }, + } + + err := sup.execute(context.Background(), tx) + require.Error(t, err) + sup.mu.RLock() + current := sup.controllers[id] + sup.mu.RUnlock() + require.Same(t, oldController, current, "failed replacement must retain the old controller") + require.False(t, replacement.IsStarted(), "candidate must not start after old stop fails") + + old.stopErr = nil + require.NoError(t, oldController.Stop()) +} + +func TestSupervisor_SameIDReplacementStartFailureKeepsNewController(t *testing.T) { + old := newTestService() + replacement := newTestService() + replacement.startErr = errors.New("replacement start failed") + oldController := NewController(context.Background(), old, supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + }, nil) + defer oldController.close() + require.NoError(t, oldController.Start()) + + bus := eventbus.NewBus() + defer bus.Stop() + sup := NewSupervisor(bus, zap.NewNop()) + sup.ctx = context.Background() + id := "service" + sup.controllers[id] = oldController + tx := newRegTx(zap.NewNop()) + tx.open = true + tx.remove[id] = struct{}{} + tx.register[id] = &supervisor.Entry{ + Service: replacement, + Config: supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + RetryPolicy: supervisor.RetryPolicy{MaxAttempts: 1}, + }, + } + + err := sup.execute(context.Background(), tx) + require.Error(t, err) + sup.mu.RLock() + current := sup.controllers[id] + sup.mu.RUnlock() + require.NotSame(t, oldController, current) + require.Same(t, replacement, current.service) + state := current.State() + require.Equal(t, supervisor.StatusFailed, state.Status) + require.False(t, replacement.IsStarted()) + select { + case <-oldController.ctx.Done(): + default: + t.Fatal("old controller supervision context was not released after commit") + } + + require.NoError(t, current.Stop()) + current.close() +} + func TestSupervisor_MultipleServices(t *testing.T) { h := newTestHarness(t) ctx := context.Background() diff --git a/system/supervisor/transaction.go b/system/supervisor/transaction.go index d29847113..f0abbdec6 100644 --- a/system/supervisor/transaction.go +++ b/system/supervisor/transaction.go @@ -10,17 +10,19 @@ import ( ) type regTx struct { - register map[string]*supervisor.Entry - remove map[string]struct{} - logger *zap.Logger - open bool + register map[string]*supervisor.Entry + remove map[string]struct{} + registeredBeforeRemove map[string]bool + logger *zap.Logger + open bool } func newRegTx(logger *zap.Logger) *regTx { return ®Tx{ - register: make(map[string]*supervisor.Entry), - remove: make(map[string]struct{}), - logger: logger, + register: make(map[string]*supervisor.Entry), + remove: make(map[string]struct{}), + registeredBeforeRemove: make(map[string]bool), + logger: logger, } } @@ -30,8 +32,7 @@ func (th *regTx) begin() { } th.open = true - th.register = make(map[string]*supervisor.Entry) - th.remove = make(map[string]struct{}) + th.resetChanges() } func (th *regTx) commit(removeFn func(string) error, registerFn func(string, *supervisor.Entry) error) error { @@ -88,7 +89,17 @@ func (th *regTx) registerService(id string, entry *supervisor.Entry) error { return supervisor.ErrOutsideTransaction } - delete(th.remove, id) + if _, removed := th.remove[id]; removed { + // A register following a remove is a replacement when the remove was + // already pending before this transaction saw a registration. Keep both + // operations so commit stops the old controller before installing the + // new one. A register/remove/register sequence is a canceled + // registration and retains the historical cancellation behavior. + if th.registeredBeforeRemove[id] { + delete(th.remove, id) + delete(th.registeredBeforeRemove, id) + } + } th.register[id] = entry // always use the latest entry return nil } @@ -98,19 +109,29 @@ func (th *regTx) removeService(id string) error { return supervisor.ErrOutsideTransaction } - // duplicate check + // A duplicate remove is idempotent, but a remove after a replacement's + // register cancels that new registration while retaining removal of the old + // controller. This preserves the final event in the transaction. if _, exists := th.remove[id]; exists { + delete(th.register, id) return nil } + _, registered := th.register[id] delete(th.register, id) th.remove[id] = struct{}{} + th.registeredBeforeRemove[id] = registered return nil } func (th *regTx) reset() { th.open = false + th.resetChanges() +} + +func (th *regTx) resetChanges() { th.register = make(map[string]*supervisor.Entry) th.remove = make(map[string]struct{}) + th.registeredBeforeRemove = make(map[string]bool) } diff --git a/system/supervisor/transaction_test.go b/system/supervisor/transaction_test.go index 1db3e44fb..74ef967fb 100644 --- a/system/supervisor/transaction_test.go +++ b/system/supervisor/transaction_test.go @@ -242,3 +242,27 @@ func TestTransactionHelper_RemoveService_NoTransaction(t *testing.T) { t.Error("removeService should return error outside of transaction") } } + +func TestTransactionHelper_SameIDRemoveThenRegisterIsReplacement(t *testing.T) { + th := newRegTx(noopLogger()) + th.begin() + + entry := &supervisor.Entry{} + assert.NoError(t, th.removeService("service1")) + assert.NoError(t, th.registerService("service1", entry)) + + var sequence []string + assert.NoError(t, th.commit( + func(id string) error { + sequence = append(sequence, "remove:"+id) + return nil + }, + func(id string, got *supervisor.Entry) error { + assert.Same(t, entry, got) + sequence = append(sequence, "register:"+id) + return nil + }, + )) + + assert.Equal(t, []string{"remove:service1", "register:service1"}, sequence) +} From 204da0ab35ebe5b5cdcfe101fe387b47e0c2ed6f Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 01:53:49 -0400 Subject: [PATCH 29/47] fix(cdc): retain failed candidate cleanup leases --- service/cdc/manager.go | 18 ++-- service/cdc/manager_test.go | 160 ++++++++++++++++++++++++++++++ service/cdc/slot.go | 192 +++++++++++++++++++++++++----------- 3 files changed, 304 insertions(+), 66 deletions(-) diff --git a/service/cdc/manager.go b/service/cdc/manager.go index c0ba37e43..9c822f9fc 100644 --- a/service/cdc/manager.go +++ b/service/cdc/manager.go @@ -272,18 +272,16 @@ func (m *Manager) Update(ctx context.Context, entry registry.Entry) error { } } oldLifecycle := normalizeLifecycleConfig(managedSlot.LifecycleConfig()) - if replaceErr := managedSlot.Replace(ctx, replacement, oldToken); replaceErr != nil { - // A failed candidate start leaves the old generation current and the - // speculative lease can be released. A retired-resource cleanup error - // leaves the candidate current but faulted; retain its lease until it is - // healthy or deleted. - committed := managedSlot.currentSource() == replacement - if reservedNew && !committed { + oldLease := leaseRef{key: oldKey, token: oldToken} + newLease := leaseRef{key: newKey, token: newToken, owned: reservedNew} + if replaceErr := managedSlot.Replace(ctx, replacement, oldLease, newLease); replaceErr != nil { + // A failed handoff never publishes the candidate. If its cleanup also + // failed, the stable slot retains it as retired work and therefore owns + // the candidate lease until Stop/Delete retries that cleanup. + retainedCandidate := managedSlot.hasRetiredSource(replacement) + if reservedNew && !retainedCandidate { m.releaseLease(id, newKey, newToken) } - if committed { - m.reconfigureSupervisorIfChanged(ctx, id, managedSlot, oldLifecycle) - } return replaceErr } if oldKey != newKey && !managedSlot.hasRetiredKey(oldKey) { diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go index 0d3c1a6ac..86dc46876 100644 --- a/service/cdc/manager_test.go +++ b/service/cdc/manager_test.go @@ -32,6 +32,7 @@ type managedTestSource struct { stopErr error stream *testStream lifecycle *supervisor.LifecycleConfig + onStart func() exclusive string info api.SourceInfo startCount atomic.Int32 @@ -66,6 +67,9 @@ func (s *managedTestSource) Subscribe(context.Context, api.StreamOptions) (api.S func (s *managedTestSource) Start(context.Context) (<-chan any, error) { s.startCount.Add(1) + if s.onStart != nil { + s.onStart() + } if s.startErr == nil { active := s.active.Add(1) for { @@ -738,6 +742,162 @@ func TestManagerUpdateDifferentResourceCleansSpeculativeCandidateDestructively(t require.EqualValues(t, 1, old.stopCount.Load()) } +func TestManagerUpdateRetainsCandidateLeaseWhenCleanupFailsAfterOldStopFailure(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + stopErr: errors.New("old stop failed"), + } + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }, disposeErr: errors.New("candidate dispose failed")} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + err = m.Update(context.Background(), entry) + require.ErrorContains(t, err, "old stop failed") + require.ErrorContains(t, err, "candidate dispose failed") + require.Same(t, old, slot.currentSource()) + require.True(t, slot.hasRetiredSource(candidate), "failed candidate cleanup must remain retryable") + require.EqualValues(t, 1, candidate.disposeCount.Load()) + m.leaseMu.Lock() + _, oldLeaseHeld := m.leases["slot-old"] + _, candidateLeaseHeld := m.leases["slot-new"] + m.leaseMu.Unlock() + require.True(t, oldLeaseHeld) + require.True(t, candidateLeaseHeld, "candidate lease must survive failed cleanup") + + old.stopErr = nil + require.NoError(t, slot.Stop(context.Background())) + require.EqualValues(t, 2, candidate.disposeCount.Load()) + require.False(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, candidateLeaseHeld = m.leases["slot-new"] + m.leaseMu.Unlock() + require.False(t, candidateLeaseHeld, "successful retry must release candidate lease") + require.NoError(t, m.Delete(context.Background(), entry)) +} + +func TestManagerUpdateRetainsBothLeasesWhenOldAndCandidateDisposeFail(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "old"}, + exclusive: "slot-old", + }, disposeErr: errors.New("old dispose failed")} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }, disposeErr: errors.New("candidate dispose failed")} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + _, err := slot.Start(context.Background()) + require.NoError(t, err) + + err = m.Update(context.Background(), entry) + require.ErrorContains(t, err, "old dispose failed") + require.ErrorContains(t, err, "candidate dispose failed") + require.Same(t, old, slot.currentSource()) + require.True(t, slot.hasRetiredSource(old)) + require.True(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, oldLeaseHeld := m.leases["slot-old"] + _, candidateLeaseHeld := m.leases["slot-new"] + m.leaseMu.Unlock() + require.True(t, oldLeaseHeld) + require.True(t, candidateLeaseHeld) + + require.NoError(t, slot.Stop(context.Background())) + require.False(t, slot.hasRetiredSource(old)) + require.False(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, oldLeaseHeld = m.leases["slot-old"] + _, candidateLeaseHeld = m.leases["slot-new"] + m.leaseMu.Unlock() + require.False(t, oldLeaseHeld) + require.False(t, candidateLeaseHeld) + require.NoError(t, m.Delete(context.Background(), entry)) +} + +func TestManagerUpdateRetainsCandidateOnLatePrecommitAbort(t *testing.T) { + kind := registry.Kind("db.cdc.test") + old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-old"} + candidate := &disposableTestSource{managedTestSource: &managedTestSource{ + info: api.SourceInfo{Name: "candidate"}, + exclusive: "slot-new", + }, disposeErr: errors.New("candidate dispose failed")} + next := 0 + driver := testDriver{ + kind: kind, + create: func(registry.Entry) (ManagedSource, error) { + next++ + if next == 1 { + return old, nil + } + return candidate, nil + }, + } + m, _ := newManagerTest(t, driver) + id := registry.NewID("app", "events") + entry := registry.Entry{ID: id, Kind: kind} + require.NoError(t, m.Add(context.Background(), entry)) + slot := mustSlot(t, m, id) + candidate.onStart = func() { + slot.mu.Lock() + slot.disposing = true + slot.mu.Unlock() + } + + err := m.Update(context.Background(), entry) + require.ErrorContains(t, err, ErrSourceBusy.Error()) + require.ErrorContains(t, err, "candidate dispose failed") + require.Same(t, old, slot.currentSource()) + require.True(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, candidateLeaseHeld := m.leases["slot-new"] + m.leaseMu.Unlock() + require.True(t, candidateLeaseHeld) + + require.NoError(t, slot.Stop(context.Background())) + require.False(t, slot.hasRetiredSource(candidate)) + m.leaseMu.Lock() + _, candidateLeaseHeld = m.leases["slot-new"] + m.leaseMu.Unlock() + require.False(t, candidateLeaseHeld) + require.NoError(t, m.Delete(context.Background(), entry)) +} + func TestManagerUpdateSameExclusiveKeyStopFailureFaultsSlot(t *testing.T) { old := &managedTestSource{info: api.SourceInfo{Name: "old"}, exclusive: "slot-1", stopErr: errors.New("old stop failed")} candidate := &managedTestSource{info: api.SourceInfo{Name: "candidate"}, exclusive: "slot-1"} diff --git a/service/cdc/slot.go b/service/cdc/slot.go index 014056405..d603789a7 100644 --- a/service/cdc/slot.go +++ b/service/cdc/slot.go @@ -53,9 +53,20 @@ type sourceSlot struct { } type retiredSource struct { - source ManagedSource - key string - token uint64 + source ManagedSource + key string + token uint64 + destructive bool +} + +// leaseRef is passed through a replacement handoff so a failed private +// candidate can retain exactly the lease it reserved. The manager owns the +// lease map; the slot only reports successful retired cleanup through its +// hook. +type leaseRef struct { + key string + token uint64 + owned bool } func newSourceSlot(id registry.ID, kind registry.Kind, source ManagedSource, logs ...*zap.Logger) *sourceSlot { @@ -242,9 +253,7 @@ func (s *sourceSlot) Stop(ctx context.Context) error { } else { err = stopSource(ctx, current) } - if err == nil { - err = s.retryRetired(ctx) - } + err = errors.Join(err, s.retryRetired(ctx)) s.mu.Lock() if err != nil { @@ -293,9 +302,7 @@ func (s *sourceSlot) Dispose(ctx context.Context) error { } else { err = nil } - if err == nil { - err = s.retryRetired(ctx) - } + err = errors.Join(err, s.retryRetired(ctx)) s.mu.Lock() if err != nil { @@ -312,9 +319,10 @@ func (s *sourceSlot) Dispose(ctx context.Context) error { } // Replace starts a candidate before changing visibility whenever the slot is -// running or the candidate is configured for auto-start. Failure leaves the -// old generation current and untouched. -func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retiredTokens ...uint64) error { +// running or the candidate is configured for auto-start. A failed handoff +// never publishes the candidate; any failed candidate cleanup is retained as +// retired work for a later Stop/Delete retry. +func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, oldLease, candidateLease leaseRef) error { if isNilSource(candidate) { return ErrDriverRequired } @@ -325,29 +333,34 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir defer s.opMu.Unlock() s.mu.Lock() - if s.disposing || len(s.retired) > 0 { - s.mu.Unlock() - _ = stopUnstartedSource(ctx, candidate) - return ErrSourceBusy - } old := s.current oldState := s.state oldRunCancel := s.runCancel - if oldState == slotStopping { - s.mu.Unlock() - _ = stopUnstartedSource(ctx, candidate) - return ErrSourceBusy - } - s.replacing = true + disposing := s.disposing + hasRetired := len(s.retired) > 0 s.mu.Unlock() - retiredToken := uint64(0) - if len(retiredTokens) > 0 { - retiredToken = retiredTokens[0] - } oldKey := exclusiveResourceKey(old) candidateKey := exclusiveResourceKey(candidate) differentResource := oldKey != candidateKey + if disposing || hasRetired || oldState == slotStopping { + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, false) + return errors.Join(ErrSourceBusy, cleanupErr) + } + + // Re-check under the slot lock after calculating resource identity. No + // other lifecycle operation can replace current while opMu is held, but a + // status watcher may still have changed the state. + s.mu.Lock() + if s.disposing || len(s.retired) > 0 || s.state == slotStopping { + s.replacing = false + s.mu.Unlock() + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, false) + return errors.Join(ErrSourceBusy, cleanupErr) + } + s.replacing = true + s.mu.Unlock() + startCandidate := oldState == slotRunning || lifecycleAutoStart(candidate) shouldStopOld := !isNilSource(old) && (oldState != slotStopped && oldState != slotIdle || differentResource || oldKey == "") @@ -362,10 +375,11 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir var err error underlying, err = startSource(ctx, runCtx, candidate) if err != nil { - _ = cleanupStartedSource(ctx, candidate, differentResource) runCancel() + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, true) + return errors.Join(err, cleanupErr) } - return err + return nil } if speculative { // Different resource keys may be prepared in parallel. The candidate @@ -381,9 +395,12 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir oldStopped := !shouldStopOld if shouldStopOld { if err := stopGeneration(ctx, old, oldRunCancel); err != nil { + var cleanupErr error if speculative { runCancel() - _ = cleanupStartedSource(ctx, candidate, differentResource) + cleanupErr = s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, true) + } else { + cleanupErr = s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, false) } s.mu.Lock() s.state = slotFaulted @@ -392,10 +409,7 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir s.runCtx = nil s.runCancel = nil s.mu.Unlock() - if !speculative { - _ = stopUnstartedSource(ctx, candidate) - } - return err + return errors.Join(err, cleanupErr) } oldStopped = true s.mu.Lock() @@ -411,23 +425,24 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir if err := disposable.Dispose(ctx); err != nil { // Keep the old source current and retain its lease. Stop retries // this pending destructive cleanup during shutdown or delete. - s.recordRetired(old, oldKey, retiredToken) + s.recordRetired(old, oldKey, oldLease.token, true) s.mu.Lock() s.state = slotFaulted s.replacing = false s.mu.Unlock() + var cleanupErr error if speculative { runCancel() } - _ = cleanupStartedSource(ctx, candidate, speculative && differentResource) - return err + cleanupErr = s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, speculative) + return errors.Join(err, cleanupErr) } if oldKey != "" { s.mu.RLock() hook := s.retiredHook s.mu.RUnlock() if hook != nil { - hook(oldKey, retiredToken) + hook(oldKey, oldLease.token) } } } @@ -467,12 +482,17 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, retir s.mu.Lock() if s.disposing || s.state == slotStopping { + s.replacing = false + s.state = slotFaulted + s.closeStatusLocked() + s.runCtx = nil + s.runCancel = nil s.mu.Unlock() if runCancel != nil { runCancel() } - _ = cleanupStartedSource(ctx, candidate, startCandidate && differentResource) - return ErrSourceBusy + cleanupErr := s.cleanupCandidate(ctx, candidate, candidateLease, differentResource, startCandidate) + return errors.Join(ErrSourceBusy, cleanupErr) } s.current = candidate s.generation++ @@ -539,6 +559,37 @@ func cleanupStartedSource(ctx context.Context, source api.Source, destructive bo return stopSource(ctx, source) } +// cleanupCandidate performs the only cleanup of a private replacement +// generation. When cleanup itself fails, the candidate is retained in the +// slot's retired queue so Stop/Delete can retry it. A different resource keeps +// its candidate lease; a same-key candidate never owns the old lease and must +// not release it when its non-destructive Stop eventually succeeds. +func (s *sourceSlot) cleanupCandidate( + ctx context.Context, + source ManagedSource, + lease leaseRef, + differentResource bool, + started bool, +) error { + if isNilSource(source) { + return nil + } + var err error + if started { + err = cleanupStartedSource(ctx, source, differentResource) + } else { + err = stopUnstartedSource(ctx, source) + } + if err == nil { + return nil + } + if !differentResource && !lease.owned { + lease = leaseRef{} + } + s.recordRetired(source, lease.key, lease.token, started && differentResource) + return err +} + func disposeSource(ctx context.Context, source api.Source) error { if isNilSource(source) { return nil @@ -556,12 +607,23 @@ func stopGeneration(ctx context.Context, source api.Source, cancel context.Cance return stopSource(ctx, source) } -func (s *sourceSlot) recordRetired(source ManagedSource, key string, token uint64) { +func (s *sourceSlot) recordRetired(source ManagedSource, key string, token uint64, destructive bool) { if isNilSource(source) { return } s.mu.Lock() - s.retired = append(s.retired, retiredSource{source: source, key: key, token: token}) + for _, existing := range s.retired { + if existing.source == source { + s.mu.Unlock() + return + } + } + s.retired = append(s.retired, retiredSource{ + source: source, + key: key, + token: token, + destructive: destructive, + }) s.mu.Unlock() } @@ -577,30 +639,48 @@ func (s *sourceSlot) isRetiredLocked(source ManagedSource) bool { return false } -func (s *sourceSlot) retryRetired(ctx context.Context) error { - for { - s.mu.RLock() - if len(s.retired) == 0 { - s.mu.RUnlock() - return nil +func (s *sourceSlot) hasRetiredSource(source ManagedSource) bool { + s.mu.RLock() + defer s.mu.RUnlock() + for _, retired := range s.retired { + if retired.source == source { + return true } - retired := s.retired[0] - s.mu.RUnlock() + } + return false +} - if err := disposeSource(ctx, retired.source); err != nil { - return err +func (s *sourceSlot) retryRetired(ctx context.Context) error { + s.mu.RLock() + retired := append([]retiredSource(nil), s.retired...) + s.mu.RUnlock() + var errs []error + for _, item := range retired { + var err error + if item.destructive { + err = disposeSource(ctx, item.source) + } else { + err = stopSource(ctx, item.source) + } + if err != nil { + errs = append(errs, err) + continue } s.mu.Lock() - if len(s.retired) > 0 && s.retired[0].source == retired.source { - s.retired = s.retired[1:] + for i, current := range s.retired { + if current.source == item.source { + s.retired = append(s.retired[:i], s.retired[i+1:]...) + break + } } hook := s.retiredHook s.mu.Unlock() - if hook != nil && retired.key != "" { - hook(retired.key, retired.token) + if hook != nil && item.key != "" { + hook(item.key, item.token) } } + return errors.Join(errs...) } func (s *sourceSlot) setRetiredCleanupHook(hook func(string, uint64)) { From 23c07b4f1b21756522b9607f42478cfe779819d9 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 02:13:20 -0400 Subject: [PATCH 30/47] fix(cdc/sqlite): make stop retries lifecycle-safe --- service/cdc/sqlite/source.go | 117 ++++++++++++++++++------- service/cdc/sqlite/source_test.go | 140 ++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 32 deletions(-) diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index a2cff90b4..5dbff54e2 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -39,25 +39,27 @@ type Source struct { res resource.Registry observerSource sqlapi.CommittedMutationSource observer sqlapi.MutationStream - snapshotAcq map[uint64]*snapshotAcquisition - runCancel context.CancelFunc status chan any + runCancel context.CancelFunc + stopGate chan struct{} startCancel context.CancelFunc subs *subscribers log *zap.Logger startDone chan struct{} snapshotSubs map[*subscription]sqlapi.MutationStream runDone chan struct{} - dbResID registry.ID + snapshotWait chan struct{} + snapshotAcq map[uint64]*snapshotAcquisition id registry.ID + dbResID registry.ID name string - generation string state config.SourceState + generation string tables []string lifecycle configLifecycle snapshotWG sync.WaitGroup - nextSnapshotID uint64 statusTick time.Duration + nextSnapshotID uint64 mu sync.RWMutex snapshot bool statusClosed bool @@ -94,6 +96,8 @@ func buildSource(opts sourceOptions) (managedSource, error) { if name == "" { name = "sqlite" } + stopGate := make(chan struct{}, 1) + stopGate <- struct{}{} return &Source{ res: opts.res, @@ -108,6 +112,7 @@ func buildSource(opts sourceOptions) (managedSource, error) { subs: newSubscribers(), snapshotSubs: make(map[*subscription]sqlapi.MutationStream), snapshotAcq: make(map[uint64]*snapshotAcquisition), + stopGate: stopGate, state: config.SourceStateUnknown, }, nil } @@ -165,6 +170,10 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { } s.mu.Lock() + if s.stopping { + s.mu.Unlock() + return nil, ErrSourceClosed + } if s.state == config.SourceStateRunning { status := s.status s.mu.Unlock() @@ -174,11 +183,6 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { s.mu.Unlock() return nil, fmt.Errorf("%w: start already in progress", config.ErrSourceNotReady) } - if s.stopping { - s.mu.Unlock() - return nil, ErrSourceClosed - } - startCtx, startCancel := context.WithCancel(ctx) startDone := make(chan struct{}) status := make(chan any, 8) @@ -196,13 +200,13 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { startCancel() s.mu.Lock() s.sourceErr = err - if s.stopping { - s.state = config.SourceStateStopped - } else { + if !s.stopping { s.state = config.SourceStateFaulted } s.startCancel = nil - s.closeStatusLocked() + if !s.stopping { + s.closeStatusLocked() + } s.mu.Unlock() return nil, err } @@ -216,13 +220,13 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { startCancel() s.mu.Lock() s.sourceErr = err - if s.stopping { - s.state = config.SourceStateStopped - } else { + if !s.stopping { s.state = config.SourceStateFaulted } s.startCancel = nil - s.closeStatusLocked() + if !s.stopping { + s.closeStatusLocked() + } s.mu.Unlock() return nil, fmt.Errorf("subscribe sqlite mutation observer: %w", err) } @@ -475,13 +479,21 @@ func (s *Source) Stop(ctx context.Context) error { if ctx == nil { ctx = context.Background() } + // Serialize cleanup attempts. A timed-out attempt leaves the source in its + // pre-stop state with stopping set, so a later caller must be able to retry + // the same cleanup without racing the first attempt or observing a false + // successful stop. + if err := s.acquireStop(ctx); err != nil { + return err + } + defer s.releaseStop() + s.mu.Lock() - if s.state == config.SourceStateStopped { + if s.state == config.SourceStateStopped && !s.stopping { s.mu.Unlock() return nil } s.stopping = true - s.state = config.SourceStateStopped startCancel := s.startCancel runCancel := s.runCancel startDone := s.startDone @@ -516,24 +528,32 @@ func (s *Source) Stop(ctx context.Context) error { _ = snapshotStream.Close() } - waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + s.mu.Lock() + snapshotDone := s.snapshotWait + if snapshotDone == nil { + snapshotDone = make(chan struct{}) + s.snapshotWait = snapshotDone + go func() { + s.snapshotWG.Wait() + close(snapshotDone) + }() + } + s.mu.Unlock() + + stopTimeout := cleanupTimeout + if s.lifecycle.StopTimeout > 0 { + stopTimeout = s.lifecycle.StopTimeout + } + waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stopTimeout) defer cancel() if err := waitDone(waitCtx, startDone); err != nil { - s.subs.closeWithError(err) - return err + return s.stopFailed(err) } if err := waitDone(waitCtx, runDone); err != nil { - s.subs.closeWithError(err) - return err + return s.stopFailed(err) } - snapshotDone := make(chan struct{}) - go func() { - s.snapshotWG.Wait() - close(snapshotDone) - }() if err := waitDone(waitCtx, snapshotDone); err != nil { - s.subs.closeWithError(err) - return err + return s.stopFailed(err) } s.subs.closeAll() @@ -544,11 +564,41 @@ func (s *Source) Stop(ctx context.Context) error { s.observerSource = nil s.startCancel = nil s.runCancel = nil + s.snapshotWait = nil s.closeStatusLocked() s.mu.Unlock() return nil } +func (s *Source) acquireStop(ctx context.Context) error { + if s.stopGate == nil { + return nil + } + select { + case <-s.stopGate: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *Source) releaseStop() { + if s.stopGate != nil { + s.stopGate <- struct{}{} + } +} + +func (s *Source) stopFailed(err error) error { + s.subs.closeWithError(err) + s.mu.Lock() + if s.state != config.SourceStateStopped { + s.state = config.SourceStateFaulted + s.sourceErr = err + } + s.mu.Unlock() + return err +} + func waitDone(ctx context.Context, done <-chan struct{}) error { if done == nil { return nil @@ -609,6 +659,9 @@ func (s *Source) Subscribe(ctx context.Context, opts config.StreamOptions) (conf if err := ctx.Err(); err != nil { return nil, err } + if err := opts.Validate(); err != nil { + return nil, err + } if opts.After != "" { return nil, config.ErrUnsupported } diff --git a/service/cdc/sqlite/source_test.go b/service/cdc/sqlite/source_test.go index faffd9a4f..629f51953 100644 --- a/service/cdc/sqlite/source_test.go +++ b/service/cdc/sqlite/source_test.go @@ -19,6 +19,7 @@ import ( "github.com/wippyai/runtime/api/resource" cdcapi "github.com/wippyai/runtime/api/service/cdc" sqlapi "github.com/wippyai/runtime/api/service/sql" + "github.com/wippyai/runtime/api/supervisor" sqlservice "github.com/wippyai/runtime/service/sql" ) @@ -462,6 +463,145 @@ func TestSourceStopWaitsForBlockedSnapshotAcquisition(t *testing.T) { source.mu.RUnlock() } +func TestSourceStopTimeoutIsRetryableAndRestartable(t *testing.T) { + observer := &testObserver{ + snapshotStarted: make(chan struct{}, 1), + snapshotCancelDelay: 50 * time.Millisecond, + } + source := newTestSource(t, observer, sourceOptions{ + lifecycle: supervisor.LifecycleConfig{StopTimeout: 10 * time.Millisecond}, + }) + _, err := source.Start(context.Background()) + require.NoError(t, err) + + subscribeDone := make(chan error, 1) + go func() { + _, subscribeErr := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + subscribeDone <- subscribeErr + }() + select { + case <-observer.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("snapshot acquisition did not start") + } + + err = source.Stop(context.Background()) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorIs(t, func() error { + _, startErr := source.Start(context.Background()) + return startErr + }(), ErrSourceClosed) + source.mu.RLock() + assert.Equal(t, cdcapi.SourceStateFaulted, source.state, "a timed-out stop reports failure without publishing Stopped") + assert.True(t, source.stopping, "a timed-out stop remains retryable") + source.mu.RUnlock() + + assert.Error(t, <-subscribeDone) + require.NoError(t, source.Stop(context.Background())) + assert.Equal(t, cdcapi.SourceStateStopped, source.Info().State) + + _, err = source.Start(context.Background()) + require.NoError(t, err) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceFaultCancelsBlockedSnapshotAcquisition(t *testing.T) { + observer := &testObserver{snapshotStarted: make(chan struct{}, 1)} + source := newTestSource(t, observer, sourceOptions{}) + _, err := source.Start(context.Background()) + require.NoError(t, err) + + subscribeDone := make(chan error, 1) + go func() { + _, subscribeErr := source.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + subscribeDone <- subscribeErr + }() + select { + case <-observer.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("snapshot acquisition did not start") + } + + observer.currentStream(t).closeWithError(errors.New("observer generation failed")) + assert.Error(t, <-subscribeDone) + assert.Eventually(t, func() bool { + return source.Info().State == cdcapi.SourceStateFaulted + }, time.Second, time.Millisecond) + require.NoError(t, source.Stop(context.Background())) +} + +func TestSourceLifecycleIsolationAcrossResources(t *testing.T) { + firstObserver := &testObserver{ + snapshotStarted: make(chan struct{}, 1), + snapshotCancelDelay: 50 * time.Millisecond, + } + first := newTestSource(t, firstObserver, sourceOptions{ + id: registry.NewID("app", "cdc-first"), + res: &testResourceRegistry{observer: firstObserver}, + lifecycle: supervisor.LifecycleConfig{StopTimeout: 10 * time.Millisecond}, + }) + secondObserver := &testObserver{} + second := newTestSource(t, secondObserver, sourceOptions{ + id: registry.NewID("app", "cdc-second"), + res: &testResourceRegistry{observer: secondObserver}, + }) + _, err := first.Start(context.Background()) + require.NoError(t, err) + _, err = second.Start(context.Background()) + require.NoError(t, err) + defer func() { + _ = first.Stop(context.Background()) + _ = second.Stop(context.Background()) + }() + + _, err = first.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + secondLive, err := second.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + + snapshotDone := make(chan error, 1) + go func() { + _, snapshotErr := first.Subscribe(context.Background(), cdcapi.StreamOptions{Snapshot: true}) + snapshotDone <- snapshotErr + }() + select { + case <-firstObserver.snapshotStarted: + case <-time.After(time.Second): + t.Fatal("first snapshot acquisition did not start") + } + + assert.ErrorIs(t, first.Stop(context.Background()), context.DeadlineExceeded) + secondObserver.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "second-while-first-stopping", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, + After: []any{int64(2)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(2), receiveChange(t, secondLive).After["id"]) + assert.Equal(t, cdcapi.SourceStateRunning, second.Info().State) + + assert.Error(t, <-snapshotDone) + require.NoError(t, first.Stop(context.Background())) + assert.Equal(t, cdcapi.SourceStateStopped, first.Info().State) + assert.Equal(t, cdcapi.SourceStateRunning, second.Info().State) + assert.Equal(t, int32(0), secondObserver.closeN.Load(), "stopping one source must not close another SQL observer") + + _, err = first.Start(context.Background()) + require.NoError(t, err) + firstLive, err := first.Subscribe(context.Background(), cdcapi.StreamOptions{}) + require.NoError(t, err) + firstObserver.currentStream(t).push(sqlapi.MutationBatch{ + Transaction: "first-after-restart", + Changes: []sqlapi.Mutation{{ + Schema: "main", Table: "t", Columns: []string{"id"}, + After: []any{int64(1)}, Op: "insert", + }}, + }) + assert.Equal(t, int64(1), receiveChange(t, firstLive).After["id"]) + assert.Equal(t, cdcapi.SourceStateRunning, second.Info().State) +} + func TestSourceStopsWithoutClosingSQLGeneration(t *testing.T) { observer := &testObserver{} resources := &testResourceRegistry{observer: observer} From e2eeada4bdbb962264c93a30f164a11f5630b7e2 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 02:13:46 -0400 Subject: [PATCH 31/47] fix(supervisor): cancel active starts with run context Propagate sequencer cancellation into in-flight controller starts while keeping the controller root alive for bounded shutdown. Treat cancellation as terminal for the start generation so retry policy cannot resurrect it. Add deterministic cancellation coverage and a concurrent CDC multi-source lifecycle regression. --- service/cdc/manager_test.go | 118 +++++++++++++++++++++++++++ system/supervisor/controller.go | 41 +++++++++- system/supervisor/sequencer.go | 12 ++- system/supervisor/supervisor_test.go | 88 +++++++++++++++++--- 4 files changed, 245 insertions(+), 14 deletions(-) diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go index 86dc46876..fc081af43 100644 --- a/service/cdc/manager_test.go +++ b/service/cdc/manager_test.go @@ -48,6 +48,31 @@ type disposableTestSource struct { failedOnce atomic.Bool } +type blockingStopSource struct { + *managedTestSource + stopEntered chan struct{} + releaseStop chan struct{} + stopOnce sync.Once +} + +func newBlockingStopSource(source *managedTestSource) *blockingStopSource { + return &blockingStopSource{ + managedTestSource: source, + stopEntered: make(chan struct{}), + releaseStop: make(chan struct{}), + } +} + +func (s *blockingStopSource) Stop(ctx context.Context) error { + s.stopOnce.Do(func() { close(s.stopEntered) }) + select { + case <-s.releaseStop: + case <-ctx.Done(): + return ctx.Err() + } + return s.managedTestSource.Stop(ctx) +} + func (s *disposableTestSource) Dispose(ctx context.Context) error { s.disposeCount.Add(1) if s.disposeErr != nil && s.failedOnce.CompareAndSwap(false, true) { @@ -446,6 +471,99 @@ func TestManagerExclusiveResourceLeaseAcrossIDs(t *testing.T) { require.NoError(t, m.Add(context.Background(), registry.Entry{ID: second, Kind: kind})) } +func TestManagerConcurrentSourcesDoNotSerializeAndRemovalPreservesOthers(t *testing.T) { + kind := registry.Kind("db.cdc.test") + aID := registry.NewID("app", "db-a") + bID := registry.NewID("app", "db-b") + aOld := newBlockingStopSource(&managedTestSource{ + info: api.SourceInfo{Name: "db-a-old"}, + exclusive: "slot-a", + }) + aNew := &managedTestSource{info: api.SourceInfo{Name: "db-a-new"}, exclusive: "slot-a-new"} + bOld := &managedTestSource{info: api.SourceInfo{Name: "db-b-old"}, exclusive: "slot-b"} + bNew := &managedTestSource{info: api.SourceInfo{Name: "db-b-new"}, exclusive: "slot-b"} + var aCreates atomic.Int32 + var bCreates atomic.Int32 + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + switch entry.ID { + case aID: + if aCreates.Add(1) == 1 { + return aOld, nil + } + return aNew, nil + case bID: + if bCreates.Add(1) == 1 { + return bOld, nil + } + return bNew, nil + default: + return nil, errors.New("unexpected source id") + } + }, + } + bus := &recordingBus{} + m, err := NewManager(cdcsystem.NewRegistry(nil), nil, bus, nil, nil, WithDriver(driver)) + require.NoError(t, err) + entryA := registry.Entry{ID: aID, Kind: kind} + entryB := registry.Entry{ID: bID, Kind: kind} + require.NoError(t, m.Add(context.Background(), entryA)) + require.NoError(t, m.Add(context.Background(), entryB)) + _, err = mustSlot(t, m, aID).Start(context.Background()) + require.NoError(t, err) + _, err = mustSlot(t, m, bID).Start(context.Background()) + require.NoError(t, err) + + aUpdateDone := make(chan error, 1) + go func() { aUpdateDone <- m.Update(context.Background(), entryA) }() + select { + case <-aOld.stopEntered: + case <-time.After(time.Second): + t.Fatal("source A update did not reach its blocking Stop") + } + + bUpdateDone := make(chan error, 1) + go func() { bUpdateDone <- m.Update(context.Background(), entryB) }() + select { + case err := <-bUpdateDone: + require.NoError(t, err, "source B update must not wait for source A") + case <-time.After(time.Second): + t.Fatal("source B update was serialized behind source A") + } + require.Same(t, bNew, mustSlot(t, m, bID).currentSource()) + require.Same(t, aOld, mustSlot(t, m, aID).currentSource()) + + close(aOld.releaseStop) + select { + case err := <-aUpdateDone: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("source A update did not finish after Stop release") + } + require.Same(t, aNew, mustSlot(t, m, aID).currentSource()) + + require.NoError(t, m.Delete(context.Background(), entryA)) + _, aExists := m.Get(aID) + require.False(t, aExists) + _, bExists := m.Get(bID) + require.True(t, bExists, "removing source A must preserve source B") + m.leaseMu.Lock() + _, aLease := m.leases["slot-a-new"] + bLease, bLeaseHeld := m.leases["slot-b"] + m.leaseMu.Unlock() + require.False(t, aLease) + require.True(t, bLeaseHeld) + require.Equal(t, bID, bLease.id) + events := bus.snapshot() + for _, event := range events { + if event.Kind == supervisor.ServiceRemove { + require.Equal(t, aID.String(), event.Path, "source B supervisor registration must remain") + } + } + require.NoError(t, m.Delete(context.Background(), entryB)) +} + func TestManagerUpdateRejectsExclusiveResourceOwnedByAnotherID(t *testing.T) { kind := registry.Kind("db.cdc.test") first := registry.NewID("app", "first") diff --git a/system/supervisor/controller.go b/system/supervisor/controller.go index 48cd46d4b..e0395a2bc 100644 --- a/system/supervisor/controller.go +++ b/system/supervisor/controller.go @@ -93,8 +93,21 @@ func NewController( // Start initiates the service and transitions it to the running state. func (c *Controller) Start() error { + return c.startContext(context.Background()) +} + +// startContext starts the service while honoring the caller's lifecycle +// context. The controller root remains independent so a canceled supervisor +// run can still perform a bounded Stop afterward. +func (c *Controller) startContext(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } c.state.setDesiredStatus(supervisor.StatusRunning) - return c.runCommand(ctrlOp{kind: ctrlStart}) + return c.runCommand(ctrlOp{kind: ctrlStart, ctx: ctx}) } // Stop gracefully stops the service and transitions it to the stopped state. @@ -263,6 +276,13 @@ func (c *Controller) supervise() { continue case ctrlStart: + if op.ctx != nil { + if startErr := op.ctx.Err(); startErr != nil { + c.state.setDesiredStatus(supervisor.StatusStopped) + err = startErr + break + } + } if c.state.getDesiredStatus() != supervisor.StatusRunning { err = context.Canceled break @@ -271,10 +291,29 @@ func (c *Controller) supervise() { break } ctx, cancel = context.WithCancel(c.ctx) + var stopStartPropagation func() bool + if op.ctx != nil { + stopStartPropagation = context.AfterFunc(op.ctx, cancel) + } c.setStartCancel(cancel) detailsCh, sErr := c.tryStart(ctx, cancel) + if stopStartPropagation != nil { + stopStartPropagation() + } c.clearStartCancel() if sErr != nil { + if op.ctx != nil && op.ctx.Err() != nil { + // The lifecycle operation was canceled by the supervisor + // run context. Do not leave Desired=Running, otherwise the + // retry policy can resurrect this generation after commit + // cancellation. The root controller context remains alive + // for an independent bounded Stop. + c.state.setDesiredStatus(supervisor.StatusStopped) + err = op.ctx.Err() + c.updateState(supervisor.StatusStopped, err) + respondAndCancel(err) + break + } if startCh == nil && op.result != nil { startCh = op.result op.result = nil diff --git a/system/supervisor/sequencer.go b/system/supervisor/sequencer.go index 2a79eabd9..eb3875041 100644 --- a/system/supervisor/sequencer.go +++ b/system/supervisor/sequencer.go @@ -149,7 +149,13 @@ func (sp *sequencer) processStartOperations(ctx context.Context, operations []op sp.logger.Info("starting service", zap.String("service_id", op.id)) - if err := op.controller.Start(); err != nil { + var err error + if contextual, ok := op.controller.(contextStartControllable); ok { + err = contextual.startContext(ctx) + } else { + err = op.controller.Start() + } + if err != nil { resultCh <- startResult{ serviceID: op.id, err: NewServiceStartError(op.id, err), @@ -227,6 +233,10 @@ type startStateChangeNotifier interface { startStateChanged() <-chan struct{} } +type contextStartControllable interface { + startContext(context.Context) error +} + func forwardStartStateChanges( ctx context.Context, done <-chan struct{}, diff --git a/system/supervisor/supervisor_test.go b/system/supervisor/supervisor_test.go index 7c95d3622..04302de50 100644 --- a/system/supervisor/supervisor_test.go +++ b/system/supervisor/supervisor_test.go @@ -54,6 +54,44 @@ type blockingStartService struct { stoppedOnce sync.Once } +type cancellationAwareStartService struct { + startEntered chan struct{} + startCanceled chan struct{} + + secondAttempt chan struct{} + startOnce sync.Once + cancelOnce sync.Once + secondOnce sync.Once + startAttempts atomic.Int32 + startCompleted atomic.Bool +} + +func newCancellationAwareStartService() *cancellationAwareStartService { + return &cancellationAwareStartService{ + startEntered: make(chan struct{}), + startCanceled: make(chan struct{}), + secondAttempt: make(chan struct{}), + } +} + +func (s *cancellationAwareStartService) Start(ctx context.Context) (<-chan any, error) { + if s.startAttempts.Add(1) > 1 { + s.secondOnce.Do(func() { close(s.secondAttempt) }) + } + s.startOnce.Do(func() { close(s.startEntered) }) + <-ctx.Done() + s.cancelOnce.Do(func() { close(s.startCanceled) }) + return nil, ctx.Err() +} + +func (s *cancellationAwareStartService) Stop(context.Context) error { + return nil +} + +func (s *cancellationAwareStartService) IsStarted() bool { + return s.startCompleted.Load() +} + func newBlockingStartService() *blockingStartService { return &blockingStartService{ startedCh: make(chan struct{}), @@ -1145,22 +1183,48 @@ func TestSupervisor_ContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) h.start(ctx) - // Register a service that takes time to start - svc := h.service("slow-service") - svc.startDelay = 2 * time.Second - - h.registerServices(map[string]bool{ - "slow-service": true, + // Register a service that cannot complete until its Start context is + // canceled. Waiting for startEntered makes the cancellation race + // deterministic: the supervisor has already admitted the start operation. + svc := newCancellationAwareStartService() + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxBegin}) + h.sup.handleEvent(event.Event{ + System: supervisor.System, + Kind: supervisor.ServiceRegister, + Path: "slow-service", + Data: &supervisor.Entry{ + Service: svc, + Config: supervisor.LifecycleConfig{ + AutoStart: true, + StartTimeout: time.Second, + StopTimeout: time.Second, + RetryPolicy: supervisor.RetryPolicy{ + InitialDelay: 10 * time.Millisecond, + MaxDelay: 10 * time.Millisecond, + }, + }, + }, }) + h.sup.handleEvent(event.Event{System: registry.System, Kind: registry.TxCommit}) - // Cancel context while service is starting + select { + case <-svc.startEntered: + case <-time.After(time.Second): + t.Fatal("timed out waiting for service Start") + } cancel() - // Wait a bit to ensure cancellation is processed - time.Sleep(100 * time.Millisecond) - - // Verify service was not started - require.False(t, svc.IsStarted(), "Service should not be started after context cancellation") + select { + case <-svc.startCanceled: + case <-time.After(time.Second): + t.Fatal("timed out waiting for in-flight Start cancellation") + } + select { + case <-svc.secondAttempt: + t.Fatal("canceled start scheduled a retry") + case <-time.After(200 * time.Millisecond): + } + require.False(t, svc.IsStarted(), "service should not be started after context cancellation") h.stop() } From 1f3031a3476b900fbdd9c47008bad2230e764209 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 02:14:09 -0400 Subject: [PATCH 32/47] fix(sqlite): bound snapshot batches before emission --- service/sql/engine/sqlite/observer.go | 80 ++++++++++-- service/sql/engine/sqlite/observer_test.go | 141 +++++++++++++++++++++ 2 files changed, 207 insertions(+), 14 deletions(-) diff --git a/service/sql/engine/sqlite/observer.go b/service/sql/engine/sqlite/observer.go index 9154fd4fa..9ffb31d9d 100644 --- a/service/sql/engine/sqlite/observer.go +++ b/service/sql/engine/sqlite/observer.go @@ -176,15 +176,15 @@ func (b *sqliteBackend) Subscribe(ctx context.Context, opts sqlapi.MutationOptio if err := b.validateTables(ctx, opts.Tables); err != nil { return nil, err } - stream := newMutationStream(ctx, b, opts) b.mu.Lock() if b.closed { b.mu.Unlock() - stream.closeWithError(errObserverClosed) return nil, errObserverClosed } + stream := newMutationStream(ctx, b, opts) b.streams[stream] = struct{}{} b.mu.Unlock() + stream.start() return stream, nil } @@ -321,7 +321,6 @@ func (b *sqliteBackend) Snapshot(ctx context.Context, opts sqlapi.SnapshotOption if opts.MaxBytes <= 0 { opts.MaxBytes = b.maxBytes } - stream := newSnapshotStream(scanCtx, b, opts, watermark, cancel) b.mu.Lock() if b.closed { b.mu.Unlock() @@ -330,6 +329,7 @@ func (b *sqliteBackend) Snapshot(ctx context.Context, opts sqlapi.SnapshotOption _ = conn.Close() return nil, errObserverClosed } + stream := newSnapshotStream(scanCtx, b, opts, watermark, cancel) b.streams[stream] = struct{}{} b.mu.Unlock() // The fence remains held until the stream is registered and its read view @@ -337,6 +337,7 @@ func (b *sqliteBackend) Snapshot(ctx context.Context, opts sqlapi.SnapshotOption // than watermark and are buffered by this stream. release = false b.releaseFence() + stream.start() go b.scanSnapshot(scanCtx, conn, tx, stream, opts) return stream, nil } @@ -452,7 +453,7 @@ func scanSnapshotTable(ctx context.Context, tx *sql.Tx, stream *mutationStream, return nil } columns = append([]string(nil), columns[1:]...) - changes := make([]sqlapi.Mutation, 0, batchSize) + batcher := newSnapshotBatcher(stream.watermark, batchSize, stream.maxChanges, stream.maxBytes) for rows.Next() { values := make([]any, len(columns)+1) dest := make([]any, len(values)) @@ -467,26 +468,74 @@ func scanSnapshotTable(ctx context.Context, tx *sql.Tx, stream *mutationStream, return fmt.Errorf("sqlite snapshot %s.%s returned invalid rowid %v", schema, table, values[0]) } after := append([]any(nil), values[1:]...) - changes = append(changes, sqlapi.Mutation{ + change := sqlapi.Mutation{ Schema: schema, Table: table, Columns: columns, RowID: rowID, After: after, Op: "snapshot", - }) - if len(changes) >= batchSize { - if err := stream.pushSnapshot(sqlapi.MutationBatch{Transaction: stream.watermark, Snapshot: true, Changes: append([]sqlapi.Mutation(nil), changes...)}); err != nil { - return err - } - changes = changes[:0] + } + if err := batcher.add(change, stream.pushSnapshot); err != nil { + return err } } if err := rows.Err(); err != nil { return err } - if len(changes) > 0 { - return stream.pushSnapshot(sqlapi.MutationBatch{Transaction: stream.watermark, Snapshot: true, Changes: append([]sqlapi.Mutation(nil), changes...)}) + return batcher.flush(stream.pushSnapshot) +} + +type snapshotBatcher struct { + transaction string + changes []sqlapi.Mutation + batchBytes int + batchSize int + maxChanges int + maxBytes int +} + +func newSnapshotBatcher(transaction string, batchSize, maxChanges, maxBytes int) *snapshotBatcher { + return &snapshotBatcher{ + transaction: transaction, + batchBytes: mutationBatchBytes(sqlapi.MutationBatch{Transaction: transaction}), + batchSize: batchSize, + maxChanges: maxChanges, + maxBytes: maxBytes, + } +} + +func (b *snapshotBatcher) add(change sqlapi.Mutation, emit func(sqlapi.MutationBatch) error) error { + changeBytes := mutationSize(change) + if len(b.changes) > 0 { + bytesExceed := b.maxBytes > 0 && (b.batchBytes > b.maxBytes || changeBytes > b.maxBytes-b.batchBytes) + changesExceed := b.maxChanges > 0 && len(b.changes) >= b.maxChanges + if bytesExceed || changesExceed { + if err := b.flush(emit); err != nil { + return err + } + } + } + b.changes = append(b.changes, change) + b.batchBytes = saturatingAdd(b.batchBytes, changeBytes) + if len(b.changes) >= b.batchSize || + (b.maxBytes > 0 && b.batchBytes >= b.maxBytes) || + (b.maxChanges > 0 && len(b.changes) >= b.maxChanges) { + return b.flush(emit) } return nil } +func (b *snapshotBatcher) flush(emit func(sqlapi.MutationBatch) error) error { + if len(b.changes) == 0 { + return nil + } + batch := sqlapi.MutationBatch{ + Transaction: b.transaction, + Snapshot: true, + Changes: append([]sqlapi.Mutation(nil), b.changes...), + } + b.changes = b.changes[:0] + b.batchBytes = mutationBatchBytes(sqlapi.MutationBatch{Transaction: b.transaction}) + return emit(batch) +} + func (b *sqliteBackend) remove(stream *mutationStream, err error) { b.mu.Lock() delete(b.streams, stream) @@ -1723,10 +1772,13 @@ func newMutationStream(ctx context.Context, backend *sqliteBackend, opts sqlapi. maxChanges: opts.MaxChanges, maxBytes: opts.MaxBytes, } - go stream.relay() return stream } +func (s *mutationStream) start() { + go s.relay() +} + func newSnapshotStream(ctx context.Context, backend *sqliteBackend, opts sqlapi.SnapshotOptions, watermark string, cancel context.CancelFunc) *mutationStream { stream := newMutationStream(ctx, backend, sqlapi.MutationOptions{ Tables: opts.Tables, MaxChanges: opts.MaxChanges, MaxBytes: opts.MaxBytes, diff --git a/service/sql/engine/sqlite/observer_test.go b/service/sql/engine/sqlite/observer_test.go index 6fee1c723..86cfe68a4 100644 --- a/service/sql/engine/sqlite/observer_test.go +++ b/service/sql/engine/sqlite/observer_test.go @@ -7,6 +7,7 @@ package sqlite import ( "context" "path/filepath" + "strings" "testing" "time" @@ -81,6 +82,97 @@ func TestPerPoolObserverCapturesOwnDatabase(t *testing.T) { } } +func TestPerPoolSnapshotLiveIsolation(t *testing.T) { + first, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-first.db")) + require.NoError(t, err) + defer first.Close() + second, err := openObservedDB(t, filepath.Join(t.TempDir(), "snapshot-second.db")) + require.NoError(t, err) + defer second.Close() + + for _, db := range []*openedDBForTest{first, second} { + _, err = db.opened.DB.ExecContext(context.Background(), `CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)`) + require.NoError(t, err) + _, err = db.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (1, 'initial')`) + require.NoError(t, err) + } + + type snapshotResult struct { + stream config.SnapshotStream + err error + } + firstSnapshotCh := make(chan snapshotResult, 1) + secondSnapshotCh := make(chan snapshotResult, 1) + go func() { + stream, snapshotErr := first.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 4096, + }) + firstSnapshotCh <- snapshotResult{stream: stream, err: snapshotErr} + }() + go func() { + stream, snapshotErr := second.opened.Observer.Snapshot(context.Background(), config.SnapshotOptions{ + Tables: []string{"items"}, BatchSize: 4096, + }) + secondSnapshotCh <- snapshotResult{stream: stream, err: snapshotErr} + }() + firstSnapshot := (<-firstSnapshotCh) + secondSnapshot := (<-secondSnapshotCh) + require.NoError(t, firstSnapshot.err) + require.NoError(t, secondSnapshot.err) + defer func() { _ = firstSnapshot.stream.Close() }() + defer func() { _ = secondSnapshot.stream.Close() }() + + firstBatch := receiveBatch(t, firstSnapshot.stream) + secondBatch := receiveBatch(t, secondSnapshot.stream) + require.True(t, firstBatch.Snapshot) + require.True(t, secondBatch.Snapshot) + assert.Equal(t, "0", firstBatch.Transaction) + assert.Equal(t, "0", secondBatch.Transaction) + + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'first-live')`) + require.NoError(t, err) + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (2, 'second-live')`) + require.NoError(t, err) + firstLive := receiveBatch(t, firstSnapshot.stream) + secondLive := receiveBatch(t, secondSnapshot.stream) + require.False(t, firstLive.Snapshot) + require.False(t, secondLive.Snapshot) + assert.Equal(t, "1", firstLive.Transaction) + assert.Equal(t, "1", secondLive.Transaction) + assert.Equal(t, []byte("first-live"), firstLive.Changes[0].After[1]) + assert.Equal(t, []byte("second-live"), secondLive.Changes[0].After[1]) + require.NoError(t, firstSnapshot.stream.Close()) + require.NoError(t, secondSnapshot.stream.Close()) + + firstBackpressured, err := first.opened.Observer.Subscribe(context.Background(), config.MutationOptions{MaxChanges: 1}) + require.NoError(t, err) + secondLiveStream, err := second.opened.Observer.Subscribe(context.Background(), config.MutationOptions{}) + require.NoError(t, err) + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (3, 'first-backpressure')`) + require.NoError(t, err) + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (3, 'second-live')`) + require.NoError(t, err) + require.Equal(t, int64(3), receiveBatch(t, secondLiveStream).Changes[0].RowID) + + _, err = first.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (4, 'first-overflow')`) + require.NoError(t, err) + select { + case _, ok := <-firstBackpressured.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("first pool backpressure stream did not close") + } + assert.ErrorIs(t, firstBackpressured.Err(), errObserverOverflow) + require.NoError(t, first.opened.Observer.Close()) + + _, err = second.opened.DB.ExecContext(context.Background(), `INSERT INTO items (id, value) VALUES (4, 'second-after-close')`) + require.NoError(t, err) + secondAfterClose := receiveBatch(t, secondLiveStream) + assert.Equal(t, int64(4), secondAfterClose.Changes[0].RowID) + assert.Equal(t, []byte("second-after-close"), secondAfterClose.Changes[0].After[1]) + _ = secondLiveStream.Close() +} + func TestObserverRebindsAfterConnectionExpiry(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "reconnect.db")) require.NoError(t, err) @@ -323,6 +415,55 @@ func TestObserverSnapshotHandoffIncludesInFlightWriterAsLive(t *testing.T) { } } +func TestObserverSnapshotFlushesBeforeByteBudget(t *testing.T) { + value := strings.Repeat("x", 256) + const maxBytes = 800 + batcher := newSnapshotBatcher("0", 4096, config.DefaultMaxMutationChanges, maxBytes) + var batches []config.MutationBatch + emit := func(batch config.MutationBatch) error { + batches = append(batches, batch) + return nil + } + for i := 1; i <= 8; i++ { + err := batcher.add(config.Mutation{ + Schema: "main", Table: "items", Columns: []string{"id", "value"}, + RowID: int64(i), After: []any{int64(i), []byte(value)}, Op: "snapshot", + }, emit) + require.NoError(t, err) + } + require.NoError(t, batcher.flush(emit)) + require.Len(t, batches, 8) + for _, batch := range batches { + require.True(t, batch.Snapshot) + require.NotEmpty(t, batch.Changes) + assert.LessOrEqual(t, mutationBatchBytes(batch), maxBytes) + assert.Len(t, batch.Changes, 1) + } +} + +func TestObserverCancelledStreamCannotRemainRegistered(t *testing.T) { + backend := newSQLiteBackend(config.DefaultMaxMutationChanges, config.DefaultMaxMutationBytes) + defer func() { _ = backend.Close() }() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + stream := newMutationStream(ctx, backend, config.MutationOptions{ + MaxChanges: config.DefaultMaxMutationChanges, + MaxBytes: config.DefaultMaxMutationBytes, + }) + backend.mu.Lock() + backend.streams[stream] = struct{}{} + backend.mu.Unlock() + stream.start() + waitForStreamCount(t, backend, 0) + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("cancelled stream did not close") + } + assert.ErrorIs(t, stream.Err(), context.Canceled) +} + func TestObserverRemovesCancelledAndOverflowedStreams(t *testing.T) { observed, err := openObservedDB(t, filepath.Join(t.TempDir(), "stream-churn.db")) require.NoError(t, err) From 2118b0154463bec94d4970d52f3211163d954b93 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 02:23:35 -0400 Subject: [PATCH 33/47] feat(cdc): bound driver subscriber memory --- api/service/cdc/command.go | 9 +- api/service/cdc/errors.go | 1 + api/service/cdc/size.go | 179 +++++++++++++++++++++++++ api/service/cdc/size_test.go | 70 ++++++++++ runtime/lua/modules/cdc/module.go | 27 +++- runtime/lua/modules/cdc/module_test.go | 52 +++++-- runtime/lua/modules/cdc/types.go | 1 + service/cdc/postgres/driver.go | 3 + service/cdc/postgres/stream.go | 162 +++++++++++++++------- service/cdc/postgres/stream_test.go | 66 +++++++++ service/cdc/slot.go | 3 + service/cdc/sqlite/source.go | 3 + service/cdc/sqlite/source_test.go | 2 +- service/cdc/sqlite/subscribers.go | 155 ++++++++++++++------- service/cdc/sqlite/subscribers_test.go | 70 +++++++++- service/cdc/stream.go | 21 +-- service/cdc/stream_test.go | 52 +++++++ 17 files changed, 744 insertions(+), 132 deletions(-) create mode 100644 api/service/cdc/size.go create mode 100644 api/service/cdc/size_test.go create mode 100644 service/cdc/stream_test.go diff --git a/api/service/cdc/command.go b/api/service/cdc/command.go index 164e7e6a4..3b1ec3f91 100644 --- a/api/service/cdc/command.go +++ b/api/service/cdc/command.go @@ -21,9 +21,12 @@ const ( type StreamOptions struct { // After is an opaque source cursor. A driver that cannot resume from a // cursor must return ErrUnsupported rather than silently ignore it. - After string - Tables []string - Ops []string + After string + Tables []string + Ops []string + // MaxBytes bounds the retained logical size of one subscriber backlog. + // Zero selects DefaultMaxStreamBytes; negative values are invalid. + MaxBytes int64 Buffer int Snapshot bool } diff --git a/api/service/cdc/errors.go b/api/service/cdc/errors.go index 098135a1e..4c36ecfb4 100644 --- a/api/service/cdc/errors.go +++ b/api/service/cdc/errors.go @@ -19,6 +19,7 @@ var ( ErrInvalidMaxTransactionBytes = apierror.New(apierror.Invalid, "max_transaction_bytes must be non-negative").WithRetryable(apierror.False) ErrInvalidMaxInflightChanges = apierror.New(apierror.Invalid, "max_inflight_changes must be non-negative").WithRetryable(apierror.False) ErrInvalidMaxInflightBytes = apierror.New(apierror.Invalid, "max_inflight_bytes must be non-negative").WithRetryable(apierror.False) + ErrInvalidMaxBytes = apierror.New(apierror.Invalid, "stream max_bytes must be non-negative").WithRetryable(apierror.False) ErrDBResourceRequired = apierror.New(apierror.Invalid, "db_resource is required").WithRetryable(apierror.False) ErrSourceNotFound = apierror.New(apierror.NotFound, "cdc source not found").WithRetryable(apierror.False) ErrUnsupported = apierror.New(apierror.Invalid, "cdc operation is not supported by this source").WithRetryable(apierror.False) diff --git a/api/service/cdc/size.go b/api/service/cdc/size.go new file mode 100644 index 000000000..d982e245e --- /dev/null +++ b/api/service/cdc/size.go @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import "reflect" + +const ( + // DefaultMaxStreamBytes bounds a subscriber's retained event backlog when + // MaxBytes is omitted. It is deliberately finite for every driver. + DefaultMaxStreamBytes int64 = 64 << 20 + changeStructuralBytes = 128 + valueStructuralBytes = 24 + maxEstimateDepth = 256 + maxEstimateNodes = 1 << 20 +) + +// ValidateStreamOptions validates common stream resource limits. Buffer keeps +// its historical clamping behavior; MaxBytes is the only option with a +// rejected negative value. +func (o StreamOptions) Validate() error { + if o.MaxBytes < 0 { + return ErrInvalidMaxBytes + } + return nil +} + +// EffectiveMaxBytes returns the finite subscriber backlog limit selected by +// the options. Zero means the safe common default. +func (o StreamOptions) EffectiveMaxBytes() int64 { + if o.MaxBytes > 0 { + return o.MaxBytes + } + return DefaultMaxStreamBytes +} + +// EstimateChangeBytes returns a conservative logical retained-size estimate +// for a Change and all nested values in its before/after images. It counts +// strings and byte blobs by length, includes container structure, saturates +// at MaxInt64, and terminates on cyclic pointers/maps/slices. +// +// The estimate is intentionally driver-neutral: both SQLite and PostgreSQL +// use this exact function for their sole subscriber backlog. +func EstimateChangeBytes(change Change) int64 { + e := sizeEstimator{seen: make(map[sizeVisit]struct{})} + return e.add(reflect.ValueOf(change), changeStructuralBytes) +} + +type sizeVisit struct { + typ reflect.Type + kind reflect.Kind + ptr uintptr + len int + cap int +} + +type sizeEstimator struct { + seen map[sizeVisit]struct{} + nodes int +} + +func (e *sizeEstimator) add(value reflect.Value, total int64) int64 { + return e.addDepth(value, total, 0) +} + +func (e *sizeEstimator) addDepth(value reflect.Value, total int64, depth int) int64 { + if total >= maxInt64Value { + return maxInt64Value + } + if !value.IsValid() { + return total + } + if depth > maxEstimateDepth || e.nodes >= maxEstimateNodes { + return maxInt64Value + } + e.nodes++ + + switch value.Kind() { + case reflect.Interface: + if value.IsNil() { + return total + } + return e.addDepth(value.Elem(), total, depth+1) + case reflect.Pointer: + if value.IsNil() { + return total + } + if e.visited(value, 0, 0) { + return total + } + return e.addDepth(value.Elem(), satAdd(total, valueStructuralBytes), depth+1) + case reflect.Map: + if value.IsNil() { + return total + } + if e.visited(value, 0, 0) { + return total + } + total = satAdd(total, satMul(valueStructuralBytes, int64(value.Len()))) + iter := value.MapRange() + for iter.Next() { + total = e.addDepth(iter.Key(), total, depth+1) + total = e.addDepth(iter.Value(), total, depth+1) + } + return total + case reflect.Slice: + if value.IsNil() { + return total + } + if value.Type().Elem().Kind() == reflect.Uint8 { + return satAdd(total, int64(value.Len())) + } + if e.visited(value, value.Len(), value.Cap()) { + return total + } + total = satAdd(total, satMul(valueStructuralBytes, int64(value.Len()))) + for i := 0; i < value.Len(); i++ { + total = e.addDepth(value.Index(i), total, depth+1) + } + return total + case reflect.Array: + total = satAdd(total, satMul(valueStructuralBytes, int64(value.Len()))) + for i := 0; i < value.Len(); i++ { + total = e.addDepth(value.Index(i), total, depth+1) + } + return total + case reflect.Struct: + total = satAdd(total, typeSize(value.Type())) + for i := 0; i < value.NumField(); i++ { + total = e.addDepth(value.Field(i), total, depth+1) + } + return total + case reflect.String: + return satAdd(total, int64(value.Len())) + case reflect.Chan, reflect.Func, reflect.UnsafePointer: + return satAdd(total, typeSize(value.Type())) + default: + return satAdd(total, typeSize(value.Type())) + } +} + +func (e *sizeEstimator) visited(value reflect.Value, length, capacity int) bool { + ptr := value.Pointer() + if ptr == 0 { + return false + } + key := sizeVisit{typ: value.Type(), kind: value.Kind(), ptr: ptr, len: length, cap: capacity} + if _, exists := e.seen[key]; exists { + return true + } + e.seen[key] = struct{}{} + return false +} + +const maxInt64Value = int64(^uint64(0) >> 1) + +func typeSize(typ reflect.Type) int64 { + size := typ.Size() + if uint64(size) > uint64(maxInt64Value) { + return maxInt64Value + } + return int64(size) +} + +func satAdd(a, b int64) int64 { + if a >= maxInt64Value || b >= maxInt64Value-a { + return maxInt64Value + } + return a + b +} + +func satMul(a, b int64) int64 { + if a <= 0 || b <= 0 { + return 0 + } + if a > maxInt64Value/b { + return maxInt64Value + } + return a * b +} diff --git a/api/service/cdc/size_test.go b/api/service/cdc/size_test.go new file mode 100644 index 000000000..19be1e7cb --- /dev/null +++ b/api/service/cdc/size_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStreamOptionsMaxBytesDefaultsAndValidation(t *testing.T) { + var options StreamOptions + assert.Equal(t, DefaultMaxStreamBytes, options.EffectiveMaxBytes()) + assert.NoError(t, options.Validate()) + + options.MaxBytes = 1024 + assert.Equal(t, int64(1024), options.EffectiveMaxBytes()) + assert.NoError(t, options.Validate()) + + options.MaxBytes = -1 + assert.ErrorIs(t, options.Validate(), ErrInvalidMaxBytes) +} + +func TestEstimateChangeBytesCountsNestedBlobs(t *testing.T) { + change := Change{ + Source: "source", + Before: map[string]any{ + "name": "alice", + "blob": []byte{1, 2, 3, 4}, + "nested": map[string]any{"values": []any{"nested-value", []byte{5, 6}}}, + }, + } + base := EstimateChangeBytes(Change{}) + got := EstimateChangeBytes(change) + assert.Greater(t, got, base) + assert.GreaterOrEqual(t, got-base, int64(len("alice")+4+len("nested-value")+2)) +} + +func TestEstimateChangeBytesTerminatesCyclicValues(t *testing.T) { + cyclic := map[string]any{} + cyclic["self"] = cyclic + change := Change{After: cyclic} + + assert.NotPanics(t, func() { + assert.Positive(t, EstimateChangeBytes(change)) + }) +} + +func TestEstimateChangeBytesSaturates(t *testing.T) { + assert.Equal(t, maxInt64Value, satAdd(maxInt64Value-1, 2)) + assert.Equal(t, maxInt64Value, satMul(maxInt64Value, 2)) + assert.LessOrEqual(t, EstimateChangeBytes(Change{}), maxInt64Value) +} + +func TestEstimateChangeBytesBoundsDeepAndWideValues(t *testing.T) { + deep := map[string]any{} + current := deep + for i := 0; i < maxEstimateDepth+2; i++ { + next := map[string]any{} + current["next"] = next + current = next + } + assert.Equal(t, maxInt64Value, EstimateChangeBytes(Change{After: deep})) + + wide := make([]any, maxEstimateNodes+1) + for i := range wide { + wide[i] = "x" + } + assert.Equal(t, maxInt64Value, EstimateChangeBytes(Change{After: map[string]any{"wide": wide}})) +} diff --git a/runtime/lua/modules/cdc/module.go b/runtime/lua/modules/cdc/module.go index d2c1a8e36..e5e4c660f 100644 --- a/runtime/lua/modules/cdc/module.go +++ b/runtime/lua/modules/cdc/module.go @@ -27,6 +27,10 @@ const ( // an unbounded map/slice/channel allocation in the Lua adapter. defaultStreamBuffer = 64 maxStreamItems = 65536 + // LNumber is float64. Values above this boundary are not all exactly + // representable; integer-valued Lua literals use LInteger below and retain + // the complete int64 range instead. + maxExactLuaNumber = int64(1<<53 - 1) ) var subscriptionCounter uint64 @@ -396,6 +400,27 @@ func streamOptionsFromLua(l *lua.LState, idx int) (cdcapi.StreamOptions, *lua.Er n := int(number) opts.Buffer = n } + if v := table.RawGetString("max_bytes"); v != lua.LNil { + if v.Type() != lua.LTNumber && v.Type() != lua.LTInteger { + return opts, invalidStreamOption(l, "max_bytes must be a number") + } + if v.Type() == lua.LTInteger { + number, ok := v.(lua.LInteger) + if !ok || number < 1 { + return opts, invalidStreamOption(l, "max_bytes must be a positive integer") + } + opts.MaxBytes = int64(number) + } else { + number := lua.LVAsNumber(v) + floatNumber := float64(number) + if math.IsNaN(floatNumber) || math.IsInf(floatNumber, 0) || + math.Trunc(floatNumber) != floatNumber || + number < 1 || number > lua.LNumber(maxExactLuaNumber) { + return opts, invalidStreamOption(l, "max_bytes must be an exact positive integer") + } + opts.MaxBytes = int64(number) + } + } if v := table.RawGetString("snapshot"); v != lua.LNil { if v.Type() != lua.LTBool { return opts, invalidStreamOption(l, "snapshot must be a boolean") @@ -443,7 +468,7 @@ func validateOptionKeys(table *lua.LTable) string { return } switch string(name) { - case "tables", "ops", "buffer", "snapshot", "after": + case "tables", "ops", "buffer", "max_bytes", "snapshot", "after": default: errMsg = "stream options contains unknown field: " + string(name) } diff --git a/runtime/lua/modules/cdc/module_test.go b/runtime/lua/modules/cdc/module_test.go index 897bdf581..f1f4e6a02 100644 --- a/runtime/lua/modules/cdc/module_test.go +++ b/runtime/lua/modules/cdc/module_test.go @@ -235,6 +235,7 @@ func TestStreamOpenAndRelease(t *testing.T) { tables = {"public.accounts"}, ops = {"insert", "update"}, buffer = 4, + max_bytes = 4096, snapshot = true, after = "cursor-1", }) @@ -268,18 +269,23 @@ func TestStreamRejectsInvalidBuffer(t *testing.T) { func TestStreamRejectsMalformedOptions(t *testing.T) { cases := map[string]string{ - "tables type": `{ tables = "accounts" }`, - "tables element": `{ tables = { 1 } }`, - "ops element": `{ ops = { "insert", 2 } }`, - "fractional buffer": `{ buffer = 1.5 }`, - "zero buffer": `{ buffer = 0 }`, - "oversized buffer": `{ buffer = 65537 }`, - "snapshot type": `{ snapshot = "true" }`, - "after type": `{ after = 42 }`, - "empty after": `{ after = "" }`, - "whitespace after": `{ after = " \t\n" }`, - "unknown field": `{ unsupported = true }`, - "numeric field": `{ [1] = "unsupported" }`, + "tables type": `{ tables = "accounts" }`, + "tables element": `{ tables = { 1 } }`, + "ops element": `{ ops = { "insert", 2 } }`, + "fractional buffer": `{ buffer = 1.5 }`, + "zero buffer": `{ buffer = 0 }`, + "oversized buffer": `{ buffer = 65537 }`, + "max bytes type": `{ max_bytes = "4096" }`, + "fractional max bytes": `{ max_bytes = 1.5 }`, + "zero max bytes": `{ max_bytes = 0 }`, + "negative max bytes": `{ max_bytes = -1 }`, + "infinite max bytes": `{ max_bytes = math.huge }`, + "snapshot type": `{ snapshot = "true" }`, + "after type": `{ after = 42 }`, + "empty after": `{ after = "" }`, + "whitespace after": `{ after = " \t\n" }`, + "unknown field": `{ unsupported = true }`, + "numeric field": `{ [1] = "unsupported" }`, } for name, options := range cases { t.Run(name, func(t *testing.T) { @@ -294,6 +300,25 @@ func TestStreamRejectsMalformedOptions(t *testing.T) { } } +func TestStreamMaxBytesPreservesIntegerAndRejectsInexactFloat(t *testing.T) { + l := lua.NewState() + defer l.Close() + + integerOptions := l.CreateTable(0, 1) + integerOptions.RawSetString("max_bytes", lua.LInteger(1<<63-1)) + l.Push(integerOptions) + options, luaErr := streamOptionsFromLua(l, 1) + require.Nil(t, luaErr) + require.Equal(t, int64(1<<63-1), options.MaxBytes) + l.SetTop(0) + + floatOptions := l.CreateTable(0, 1) + floatOptions.RawSetString("max_bytes", lua.LNumber(1<<53)) + l.Push(floatOptions) + _, luaErr = streamOptionsFromLua(l, 1) + require.NotNil(t, luaErr) +} + func TestStringArrayFieldRejectsOutOfRangeIndex(t *testing.T) { l := lua.NewState() defer l.Close() @@ -331,6 +356,7 @@ func TestModuleTypesUseIntegerBufferAndTypedChannel(t *testing.T) { optionsRecord, ok := streamOptions.(*typ.Record) require.True(t, ok) require.Equal(t, typ.Integer, optionsRecord.GetField("buffer").Type) + require.Equal(t, typ.Integer, optionsRecord.GetField("max_bytes").Type) require.NotEqual(t, typ.Any, cdcChannelType) } @@ -408,7 +434,7 @@ func TestModuleTypesMatchRuntimeFields(t *testing.T) { "engine", "file", "db_resource", "epoch", "error", "tables", "streaming", "failover", "temporary", "snapshot", "faulted", }) - assertRecordFields("StreamOptions", []string{"tables", "ops", "buffer", "snapshot", "after"}) + assertRecordFields("StreamOptions", []string{"tables", "ops", "buffer", "max_bytes", "snapshot", "after"}) assertRecordFields("Change", []string{ "source_id", "source", "op", "schema", "table", "relation", "lsn", "commit_lsn", "cursor", "generation", "transaction", "error", "xid", "before", "after", diff --git a/runtime/lua/modules/cdc/types.go b/runtime/lua/modules/cdc/types.go index adee8383f..fed2a3729 100644 --- a/runtime/lua/modules/cdc/types.go +++ b/runtime/lua/modules/cdc/types.go @@ -45,6 +45,7 @@ var streamOptionsType = typ.NewRecord(). OptField("tables", typ.NewArray(typ.String)). OptField("ops", typ.NewArray(typ.String)). OptField("buffer", typ.Integer). + OptField("max_bytes", typ.Integer). OptField("snapshot", typ.Boolean). OptField("after", typ.String). Build() diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index f61a4006c..d33c63ea5 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -139,6 +139,9 @@ func (s *sourceAdapter) Subscribe(ctx context.Context, opts config.StreamOptions if err := ctx.Err(); err != nil { return nil, err } + if err := opts.Validate(); err != nil { + return nil, err + } if opts.After != "" { return nil, config.ErrUnsupported } diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index b2ea38e5d..2a2330ed5 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -22,19 +22,32 @@ const ( var errSubscriberOverflow = errors.New("postgres cdc subscriber backlog overflow") type sourceSubscription struct { - err error - source *Source - out chan config.Change - tables map[string]struct{} - ops map[string]struct{} - id uint64 - once sync.Once - sendMu sync.Mutex - closed bool - errMu sync.RWMutex + err error + tables map[string]struct{} + source *Source + done chan struct{} + notify chan struct{} + relayDone chan struct{} + ops map[string]struct{} + out chan config.Change + queue []queuedChange + maxBytes int64 + maxChanges int + id uint64 + queuedBytes int64 + mu sync.Mutex + closed bool +} + +type queuedChange struct { + change config.Change + bytes int64 } func (s *Source) Subscribe(opts config.StreamOptions) config.Stream { + if err := opts.Validate(); err != nil { + return nil + } s.mu.Lock() defer s.mu.Unlock() if s.state != sourceNew && s.state != sourceRunning { @@ -77,15 +90,22 @@ func (s *Source) newSubscription(opts config.StreamOptions) config.Stream { sub := &sourceSubscription{ source: s, id: s.nextSubID, - // out is the only event queue. sendMu serializes producers with - // terminal close so a slow consumer cannot retain a second buffer. - out: make(chan config.Change, buffer), - tables: filterSet(opts.Tables), - ops: filterSet(opts.Ops), + // queue is the sole driver-owned backlog. out is an unbuffered + // delivery handoff, so bytes are released exactly after a consumer + // receives the change rather than when it is merely enqueued. + out: make(chan config.Change), + done: make(chan struct{}), + notify: make(chan struct{}, 1), + maxChanges: buffer, + maxBytes: opts.EffectiveMaxBytes(), + relayDone: make(chan struct{}), + tables: filterSet(opts.Tables), + ops: filterSet(opts.Ops), } s.subs[sub.id] = sub s.subMu.Unlock() + go sub.run() return sub } @@ -126,6 +146,9 @@ func (s *Source) closeSubscriptionsWithError(err error) { for _, sub := range subs { sub.closeWithError(err) } + for _, sub := range subs { + sub.waitRelay() + } } func (s *sourceSubscription) Changes() <-chan config.Change { @@ -134,53 +157,100 @@ func (s *sourceSubscription) Changes() <-chan config.Change { func (s *sourceSubscription) Close() { s.closeWithError(nil) + s.waitRelay() } func (s *sourceSubscription) Err() error { - s.errMu.RLock() - defer s.errMu.RUnlock() - return s.err + s.mu.Lock() + err := s.err + s.mu.Unlock() + return err } func (s *sourceSubscription) closeWithError(err error) { - s.once.Do(func() { - // Publish the terminal error before closing Changes. Err is therefore - // immediately observable when the caller receives the closed channel. - if err != nil { - s.errMu.Lock() - s.err = err - s.errMu.Unlock() + s.mu.Lock() + parent, id := s.closeLocked(err) + s.mu.Unlock() + if parent != nil { + parent.removeSubscription(id) + } +} + +func (s *sourceSubscription) waitRelay() { + <-s.relayDone +} + +func (s *sourceSubscription) closeLocked(err error) (*Source, uint64) { + if s.closed { + return nil, 0 + } + s.closed = true + s.err = err + s.queue = nil + s.queuedBytes = 0 + close(s.done) + return s.source, s.id +} + +func (s *sourceSubscription) run() { + defer close(s.relayDone) + defer close(s.out) + for { + s.mu.Lock() + if len(s.queue) == 0 { + if s.closed { + s.mu.Unlock() + return + } + notify := s.notify + done := s.done + s.mu.Unlock() + select { + case <-notify: + case <-done: + } + continue } - // Serialize the terminal transition with send and close the event - // queue synchronously. No forwarding goroutine is needed, and no - // producer can send to a closed channel. - s.sendMu.Lock() - s.closed = true - close(s.out) - s.sendMu.Unlock() - // Detach outside sendMu: source removal takes subMu and must never - // participate in the producer/close critical section. - if s.source != nil { - s.source.removeSubscription(s.id) + item := s.queue[0] + done := s.done + s.mu.Unlock() + + select { + case <-done: + return + case s.out <- item.change: + s.mu.Lock() + if len(s.queue) > 0 { + s.queuedBytes -= s.queue[0].bytes + s.queue[0] = queuedChange{} + s.queue = s.queue[1:] + } + s.mu.Unlock() } - }) + } } func (s *sourceSubscription) send(_ context.Context, change config.Change) { - s.sendMu.Lock() + bytes := config.EstimateChangeBytes(change) + s.mu.Lock() if s.closed { - s.sendMu.Unlock() + s.mu.Unlock() + return + } + if len(s.queue) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { + parent, id := s.closeLocked(errSubscriberOverflow) + s.mu.Unlock() + if parent != nil { + parent.removeSubscription(id) + } return } + s.queue = append(s.queue, queuedChange{change: change, bytes: bytes}) + s.queuedBytes += bytes + s.mu.Unlock() select { - case s.out <- change: - s.sendMu.Unlock() + case s.notify <- struct{}{}: default: - s.sendMu.Unlock() - // Never wait for a slow consumer from the replication goroutine. The - // subscription gets a terminal error and is removed; other consumers - // continue receiving the transaction. - s.closeWithError(errSubscriberOverflow) } } diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 81bfa4eef..80101a56e 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -177,3 +177,69 @@ func TestSourceSubscriptionOverflowIsBoundedAndLocal(t *testing.T) { laggard.Close() reader.Close() } + +func TestSourceSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + change := cdcapi.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + changeBytes := cdcapi.EstimateChangeBytes(change) + stream := src.newSubscription(cdcapi.StreamOptions{Buffer: 2, MaxBytes: changeBytes + 1}) + sub := stream.(*sourceSubscription) + defer sub.Close() + + sub.send(context.Background(), change) + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 1 && sub.queuedBytes == changeBytes + }, time.Second, time.Millisecond) + + select { + case got := <-sub.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("timed out receiving queued change") + } + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 0 && sub.queuedBytes == 0 + }, time.Second, time.Millisecond) + + sub.send(context.Background(), change) + assert.NotErrorIs(t, sub.Err(), errSubscriberOverflow) + select { + case <-sub.Changes(): + case <-time.After(time.Second): + t.Fatal("released byte budget did not accept the next change") + } +} + +func TestSourceSubscriptionMaxBytesOverflowIsIsolated(t *testing.T) { + change := cdcapi.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + limit := cdcapi.EstimateChangeBytes(change) - 1 + first := NewSource(SourceOptions{Name: "test:first", Slot: "slot_first"}) + second := NewSource(SourceOptions{Name: "test:second", Slot: "slot_second"}) + firstStream := first.Subscribe(cdcapi.StreamOptions{MaxBytes: limit}) + secondStream := second.Subscribe(cdcapi.StreamOptions{MaxBytes: limit + 1}) + defer firstStream.Close() + defer secondStream.Close() + + first.publishChange(context.Background(), change) + assert.ErrorIs(t, firstStream.(interface{ Err() error }).Err(), errSubscriberOverflow) + second.publishChange(context.Background(), change) + assert.NotErrorIs(t, secondStream.(interface{ Err() error }).Err(), errSubscriberOverflow) + select { + case got := <-secondStream.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("independent source did not receive change") + } +} diff --git a/service/cdc/slot.go b/service/cdc/slot.go index d603789a7..cb1de789d 100644 --- a/service/cdc/slot.go +++ b/service/cdc/slot.go @@ -117,6 +117,9 @@ func (s *sourceSlot) Info() api.SourceInfo { } func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api.Stream, error) { + if err := opts.Validate(); err != nil { + return nil, err + } s.mu.RLock() if s.state != slotRunning || isNilSource(s.current) || s.disposing || s.replacing { s.mu.RUnlock() diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index 5dbff54e2..1a3811a02 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -456,6 +456,7 @@ func (s *Source) fail(err error) { } for _, sub := range snapshotSubscriptions { sub.closeWithError(err) + sub.waitRelay() } for _, snapshotStream := range snapshotSubs { _ = snapshotStream.Close() @@ -523,6 +524,7 @@ func (s *Source) Stop(ctx context.Context) error { } for _, sub := range snapshotSubscriptions { sub.closeWithError(nil) + sub.waitRelay() } for _, snapshotStream := range snapshotStreams { _ = snapshotStream.Close() @@ -746,6 +748,7 @@ func (s *Source) subscribeSnapshot(ctx context.Context, observer sqlapi.Committe func (s *Source) runSnapshot(ctx context.Context, stream sqlapi.SnapshotStream, sub *subscription, acquisitionID uint64) { defer s.snapshotWG.Done() + defer sub.waitRelay() defer func() { s.mu.Lock() delete(s.snapshotSubs, sub) diff --git a/service/cdc/sqlite/source_test.go b/service/cdc/sqlite/source_test.go index 629f51953..5fc4dd3dd 100644 --- a/service/cdc/sqlite/source_test.go +++ b/service/cdc/sqlite/source_test.go @@ -417,7 +417,7 @@ func TestSourceSnapshotOverflowClosesUpstream(t *testing.T) { assert.Eventually(t, func() bool { subscriber.mu.Lock() defer subscriber.mu.Unlock() - return len(subscriber.changes) == 1 + return len(subscriber.queue) == 1 }, time.Second, time.Millisecond) upstream.push(change(2)) diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go index b62f97635..d37a7a62e 100644 --- a/service/cdc/sqlite/subscribers.go +++ b/service/cdc/sqlite/subscribers.go @@ -6,7 +6,6 @@ import ( "errors" "strings" "sync" - "sync/atomic" config "github.com/wippyai/runtime/api/service/cdc" ) @@ -48,13 +47,21 @@ func (s *subscribers) subscribe(sourceName string, opts config.StreamOptions) *s } func newSubscription(sourceName string, opts config.StreamOptions, buffer int) *subscription { - return &subscription{ + sub := &subscription{ sourceName: sourceName, - changes: make(chan config.Change, buffer), + // queue is the sole driver-owned backlog. changes is an unbuffered + // delivery handoff, so bytes leave the budget only after a receive. + changes: make(chan config.Change), done: make(chan struct{}), + notify: make(chan struct{}, 1), + maxChanges: buffer, + maxBytes: opts.EffectiveMaxBytes(), tables: filterSet(opts.Tables), ops: filterSet(opts.Ops), + relayDone: make(chan struct{}), } + go sub.run() + return sub } func (s *subscribers) publish(change config.Change) { @@ -92,28 +99,41 @@ func (s *subscribers) closeWithError(err error) { for _, sub := range subs { sub.closeWithError(err) } + for _, sub := range subs { + sub.waitRelay() + } } type subscription struct { - err error - parent *subscribers - changes chan config.Change - done chan struct{} - tables map[string]struct{} - ops map[string]struct{} - sourceName string - id uint64 - mu sync.Mutex - // closedFlag lets the fan-out path reject work without taking the lock in - // the common case. The lock is still held while sending/closing so a send - // cannot race close(changes). - closedFlag atomic.Bool - closed bool + err error + ops map[string]struct{} + parent *subscribers + changes chan config.Change + done chan struct{} + notify chan struct{} + relayDone chan struct{} + tables map[string]struct{} + sourceName string + queue []queuedChange + maxBytes int64 + maxChanges int + id uint64 + queuedBytes int64 + mu sync.Mutex + closed bool +} + +type queuedChange struct { + change config.Change + bytes int64 } func (s *subscription) Changes() <-chan config.Change { return s.changes } -func (s *subscription) Close() { s.closeWithError(nil) } +func (s *subscription) Close() { + s.closeWithError(nil) + s.waitRelay() +} func (s *subscription) Err() error { s.mu.Lock() @@ -123,56 +143,90 @@ func (s *subscription) Err() error { } func (s *subscription) send(change config.Change) { - if s.closedFlag.Load() { + bytes := config.EstimateChangeBytes(change) + s.mu.Lock() + if s.closed { + s.mu.Unlock() return } - s.mu.Lock() - var parent *subscribers - var id uint64 - if !s.closed { - select { - case s.changes <- change: - default: - s.closeLocked(errSubscriberOverflow) - parent = s.parent - id = s.id + if len(s.queue) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { + parent, id := s.closeLocked(errSubscriberOverflow) + s.mu.Unlock() + if parent != nil { + parent.remove(id) } + return } + s.queue = append(s.queue, queuedChange{change: change, bytes: bytes}) + s.queuedBytes += bytes s.mu.Unlock() - // Detach after releasing the subscription lock. Taking the parent lock - // while holding s.mu would invert the order used by closeWithError and - // make concurrent publish/close able to deadlock. - if parent != nil { - parent.remove(id) + select { + case s.notify <- struct{}{}: + default: } } func (s *subscription) closeWithError(err error) { s.mu.Lock() - var parent *subscribers - var id uint64 - if s.closed { - s.mu.Unlock() - return - } - s.closeLocked(err) - parent = s.parent - id = s.id + parent, id := s.closeLocked(err) s.mu.Unlock() if parent != nil { parent.remove(id) } } -func (s *subscription) closeLocked(err error) { +func (s *subscription) waitRelay() { + <-s.relayDone +} + +func (s *subscription) closeLocked(err error) (*subscribers, uint64) { if s.closed { - return + return nil, 0 } s.closed = true - s.closedFlag.Store(true) s.err = err close(s.done) - close(s.changes) + s.queue = nil + s.queuedBytes = 0 + return s.parent, s.id +} + +func (s *subscription) run() { + defer close(s.relayDone) + defer close(s.changes) + for { + s.mu.Lock() + if len(s.queue) == 0 { + if s.closed { + s.mu.Unlock() + return + } + notify := s.notify + done := s.done + s.mu.Unlock() + select { + case <-notify: + case <-done: + } + continue + } + item := s.queue[0] + done := s.done + s.mu.Unlock() + + select { + case <-done: + return + case s.changes <- item.change: + s.mu.Lock() + if len(s.queue) > 0 { + s.queuedBytes -= s.queue[0].bytes + s.queue[0] = queuedChange{} + s.queue = s.queue[1:] + } + s.mu.Unlock() + } + } } func (s *subscription) matches(change config.Change) bool { @@ -204,7 +258,12 @@ func (s *subscription) matchesSnapshot(change config.Change) bool { return ok } -func (s *subscription) isClosed() bool { return s.closedFlag.Load() } +func (s *subscription) isClosed() bool { + s.mu.Lock() + closed := s.closed + s.mu.Unlock() + return closed +} func filterSet(values []string) map[string]struct{} { if len(values) == 0 { diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go index d24b6a3bc..4fc99c4a1 100644 --- a/service/cdc/sqlite/subscribers_test.go +++ b/service/cdc/sqlite/subscribers_test.go @@ -75,16 +75,17 @@ func TestSubscribeBufferClamp(t *testing.T) { subs := newSubscribers() def := subs.subscribe("s", config.StreamOptions{Buffer: 0}) - assert.Equal(t, defaultStreamBuffer, cap(def.changes)) + assert.Equal(t, defaultStreamBuffer, def.maxChanges) neg := subs.subscribe("s", config.StreamOptions{Buffer: -5}) - assert.Equal(t, defaultStreamBuffer, cap(neg.changes)) + assert.Equal(t, defaultStreamBuffer, neg.maxChanges) exact := subs.subscribe("s", config.StreamOptions{Buffer: 7}) - assert.Equal(t, 7, cap(exact.changes)) + assert.Equal(t, 7, exact.maxChanges) huge := subs.subscribe("s", config.StreamOptions{Buffer: maxStreamBuffer + 100}) - assert.Equal(t, maxStreamBuffer, cap(huge.changes)) + assert.Equal(t, maxStreamBuffer, huge.maxChanges) + assert.Zero(t, cap(def.changes), "the common adapter must not add a second queue") } func TestSubscribeAssignsUniqueIncreasingIDs(t *testing.T) { @@ -155,6 +156,67 @@ func TestOverflowedSubscriberChurnDoesNotRetainParentEntries(t *testing.T) { assert.Zero(t, remaining) } +func TestSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { + change := config.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + changeBytes := config.EstimateChangeBytes(change) + sub := newSubscription("s", config.StreamOptions{Buffer: 2, MaxBytes: changeBytes + 1}, 2) + defer sub.Close() + + sub.send(change) + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 1 && sub.queuedBytes == changeBytes + }, time.Second, time.Millisecond) + + select { + case got := <-sub.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("timed out receiving queued change") + } + assert.Eventually(t, func() bool { + sub.mu.Lock() + defer sub.mu.Unlock() + return len(sub.queue) == 0 && sub.queuedBytes == 0 + }, time.Second, time.Millisecond) + + sub.send(change) + assert.NotErrorIs(t, sub.Err(), errSubscriberOverflow) + select { + case <-sub.Changes(): + case <-time.After(time.Second): + t.Fatal("released byte budget did not accept the next change") + } +} + +func TestSubscriptionMaxBytesOverflowIsLocal(t *testing.T) { + change := config.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("payload")}, + } + limit := config.EstimateChangeBytes(change) - 1 + subs := newSubscribers() + laggard := subs.subscribe("s", config.StreamOptions{MaxBytes: limit}) + reader := subs.subscribe("s", config.StreamOptions{MaxBytes: limit + 1}) + + subs.publish(change) + assert.ErrorIs(t, laggard.Err(), errSubscriberOverflow) + assert.NotErrorIs(t, reader.Err(), errSubscriberOverflow) + select { + case got := <-reader.Changes(): + assert.Equal(t, change.Table, got.Table) + case <-time.After(time.Second): + t.Fatal("unrelated subscriber did not receive change") + } + reader.Close() +} + func TestSubscribersFilterByOp(t *testing.T) { subs := newSubscribers() stream := subs.subscribe("s", config.StreamOptions{Ops: []string{"delete"}}) diff --git a/service/cdc/stream.go b/service/cdc/stream.go index a99bf0ee2..af6246daf 100644 --- a/service/cdc/stream.go +++ b/service/cdc/stream.go @@ -9,11 +9,6 @@ import ( api "github.com/wippyai/runtime/api/service/cdc" ) -const ( - defaultStreamBuffer = 128 - maxStreamBuffer = 65536 -) - // stampedStream is the boundary between a driver stream and the common CDC // API. Drivers own transport-specific change decoding; the stable source slot // owns the process identity and generation that consumers use for routing and @@ -27,21 +22,15 @@ type stampedStream struct { once sync.Once } -func newStampedStream(id registry.ID, generation uint64, requestedBuffer int, upstream api.Stream) *stampedStream { - buffer := requestedBuffer - if buffer <= 0 { - buffer = defaultStreamBuffer - } - if buffer > maxStreamBuffer { - buffer = maxStreamBuffer - } - +func newStampedStream(id registry.ID, generation uint64, _ int, upstream api.Stream) *stampedStream { stream := &stampedStream{ upstream: upstream, sourceID: registry.ParseID(id.String()), generation: generationString(generation), - out: make(chan api.Change, buffer), - done: make(chan struct{}), + // The driver owns the only bounded subscriber queue. Keep this common + // identity adapter unbuffered so it cannot double retained events. + out: make(chan api.Change), + done: make(chan struct{}), } go stream.run() return stream diff --git a/service/cdc/stream_test.go b/service/cdc/stream_test.go new file mode 100644 index 000000000..48a7b15c3 --- /dev/null +++ b/service/cdc/stream_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MPL-2.0 + +package cdc + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/registry" + api "github.com/wippyai/runtime/api/service/cdc" +) + +type stampedTestStream struct { + changes chan api.Change + once sync.Once +} + +func (s *stampedTestStream) Changes() <-chan api.Change { return s.changes } + +func (s *stampedTestStream) Close() { + s.once.Do(func() { close(s.changes) }) +} + +func (*stampedTestStream) Err() error { return nil } + +func TestStampedStreamUsesOnlyTheDriverQueue(t *testing.T) { + upstream := &stampedTestStream{changes: make(chan api.Change, 2)} + stream := newStampedStream(registry.NewID("app", "cdc"), 7, 65536, upstream) + require.Zero(t, cap(stream.Changes()), "the common adapter must not add a second event queue") + + upstream.changes <- api.Change{Op: "insert", Table: "users"} + select { + case change := <-stream.Changes(): + require.Equal(t, "insert", change.Op) + require.Equal(t, "users", change.Table) + require.Equal(t, "app:cdc", change.Source) + require.Equal(t, registry.NewID("app", "cdc"), change.SourceID) + require.Equal(t, "7", change.Generation) + case <-time.After(time.Second): + t.Fatal("timed out waiting for stamped change") + } + + stream.Close() + select { + case _, ok := <-stream.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("stamped stream did not close after upstream close") + } +} From 05eed6a5f0a78842bcc51b42c72c596d0745b74c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 02:25:08 -0400 Subject: [PATCH 34/47] perf(cdc): estimate fanout payload once --- service/cdc/postgres/stream.go | 11 ++++++++--- service/cdc/postgres/stream_test.go | 4 ++-- service/cdc/sqlite/source.go | 2 +- service/cdc/sqlite/subscribers.go | 11 ++++++++--- service/cdc/sqlite/subscribers_test.go | 14 ++++++++------ 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 2a2330ed5..7a61ed514 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -119,8 +119,14 @@ func (s *Source) publishChange(ctx context.Context, change config.Change) { } s.subMu.RUnlock() + if len(subs) == 0 { + return + } + // Estimate the retained size once for this source event. Fan-out must not + // repeat a recursive walk for every subscriber. + bytes := config.EstimateChangeBytes(change) for _, sub := range subs { - sub.send(ctx, change) + sub.send(ctx, change, bytes) } } @@ -230,8 +236,7 @@ func (s *sourceSubscription) run() { } } -func (s *sourceSubscription) send(_ context.Context, change config.Change) { - bytes := config.EstimateChangeBytes(change) +func (s *sourceSubscription) send(_ context.Context, change config.Change, bytes int64) { s.mu.Lock() if s.closed { s.mu.Unlock() diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 80101a56e..6e2539029 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -190,7 +190,7 @@ func TestSourceSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { sub := stream.(*sourceSubscription) defer sub.Close() - sub.send(context.Background(), change) + sub.send(context.Background(), change, cdcapi.EstimateChangeBytes(change)) assert.Eventually(t, func() bool { sub.mu.Lock() defer sub.mu.Unlock() @@ -209,7 +209,7 @@ func TestSourceSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { return len(sub.queue) == 0 && sub.queuedBytes == 0 }, time.Second, time.Millisecond) - sub.send(context.Background(), change) + sub.send(context.Background(), change, cdcapi.EstimateChangeBytes(change)) assert.NotErrorIs(t, sub.Err(), errSubscriberOverflow) select { case <-sub.Changes(): diff --git a/service/cdc/sqlite/source.go b/service/cdc/sqlite/source.go index 1a3811a02..26accf32a 100644 --- a/service/cdc/sqlite/source.go +++ b/service/cdc/sqlite/source.go @@ -792,7 +792,7 @@ func (s *Source) runSnapshot(ctx context.Context, stream sqlapi.SnapshotStream, } else if !sub.matches(change) { continue } - sub.send(change) + sub.send(change, config.EstimateChangeBytes(change)) if sub.isClosed() { return } diff --git a/service/cdc/sqlite/subscribers.go b/service/cdc/sqlite/subscribers.go index d37a7a62e..aa85b6780 100644 --- a/service/cdc/sqlite/subscribers.go +++ b/service/cdc/sqlite/subscribers.go @@ -73,8 +73,14 @@ func (s *subscribers) publish(change config.Change) { } } s.mu.RUnlock() + if len(matched) == 0 { + return + } + // Estimate the retained size once for this source event. Fan-out must not + // repeat a recursive walk for every subscriber. + bytes := config.EstimateChangeBytes(change) for _, sub := range matched { - sub.send(change) + sub.send(change, bytes) } } @@ -142,8 +148,7 @@ func (s *subscription) Err() error { return err } -func (s *subscription) send(change config.Change) { - bytes := config.EstimateChangeBytes(change) +func (s *subscription) send(change config.Change, bytes int64) { s.mu.Lock() if s.closed { s.mu.Unlock() diff --git a/service/cdc/sqlite/subscribers_test.go b/service/cdc/sqlite/subscribers_test.go index 4fc99c4a1..79c6a7e82 100644 --- a/service/cdc/sqlite/subscribers_test.go +++ b/service/cdc/sqlite/subscribers_test.go @@ -131,8 +131,9 @@ func TestOverflowedSubscriberDetachesImmediately(t *testing.T) { stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) change := config.Change{Op: "insert", Table: "users"} - stream.send(change) - stream.send(change) + bytes := config.EstimateChangeBytes(change) + stream.send(change, bytes) + stream.send(change, bytes) assert.ErrorIs(t, stream.Err(), errSubscriberOverflow) subs.mu.RLock() @@ -146,8 +147,9 @@ func TestOverflowedSubscriberChurnDoesNotRetainParentEntries(t *testing.T) { change := config.Change{Op: "insert", Table: "users"} for i := 0; i < 1000; i++ { stream := subs.subscribe("s", config.StreamOptions{Buffer: 1}) - stream.send(change) - stream.send(change) + bytes := config.EstimateChangeBytes(change) + stream.send(change, bytes) + stream.send(change, bytes) } subs.mu.RLock() @@ -166,7 +168,7 @@ func TestSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { sub := newSubscription("s", config.StreamOptions{Buffer: 2, MaxBytes: changeBytes + 1}, 2) defer sub.Close() - sub.send(change) + sub.send(change, config.EstimateChangeBytes(change)) assert.Eventually(t, func() bool { sub.mu.Lock() defer sub.mu.Unlock() @@ -185,7 +187,7 @@ func TestSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { return len(sub.queue) == 0 && sub.queuedBytes == 0 }, time.Second, time.Millisecond) - sub.send(change) + sub.send(change, config.EstimateChangeBytes(change)) assert.NotErrorIs(t, sub.Err(), errSubscriberOverflow) select { case <-sub.Changes(): From d1e53d6b44038c747e7d9d9cab60ce87dafafff4 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 02:36:01 -0400 Subject: [PATCH 35/47] fix(cdc/postgres): await failed run shutdown --- service/cdc/postgres/service.go | 28 ++++++++++---- service/cdc/postgres/service_test.go | 58 ++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 841abe163..6fbaa70cb 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -343,14 +343,21 @@ func (s *Source) Stop(ctx context.Context) error { return nil } if s.state == sourceNew || s.state == sourceFailed { - s.state = sourceStopped - s.cancel = nil - s.mu.Unlock() - s.closeSubscriptions() - if s.dropSlot.Load() { - return s.dropSlotAndCheckpoint(ctx) + if s.state == sourceNew { + s.state = sourceStopped + s.cancel = nil + s.mu.Unlock() + s.closeSubscriptions() + if s.dropSlot.Load() { + return s.dropSlotAndCheckpoint(ctx) + } + return nil } - return nil + // A replication run marks the source failed before its deferred + // cleanup closes done. Keep the source stopping until that run has + // fully released its connections; replacement and slot deletion must + // not race the failed generation. + s.state = sourceStopping } if s.state == sourceStarting || s.state == sourceRunning { s.state = sourceStopping @@ -373,6 +380,13 @@ func (s *Source) Stop(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() } + } else { + s.mu.Lock() + if s.state == sourceStopping { + s.state = sourceStopped + s.cancel = nil + } + s.mu.Unlock() } if s.dropSlot.Load() { diff --git a/service/cdc/postgres/service_test.go b/service/cdc/postgres/service_test.go index c333e4354..760dbaa73 100644 --- a/service/cdc/postgres/service_test.go +++ b/service/cdc/postgres/service_test.go @@ -67,6 +67,64 @@ func TestFailedSourceCanBeStoppedAndRetried(t *testing.T) { assert.NotErrorIs(t, err, ErrSourceClosed) } +func TestFailedSourceStopWaitsForRunCleanup(t *testing.T) { + source := NewSource(SourceOptions{}) + runDone := make(chan struct{}) + cancelCalled := make(chan struct{}) + releaseRun := make(chan struct{}) + source.mu.Lock() + source.state = sourceFailed + source.done = runDone + source.cancel = func() { close(cancelCalled) } + source.mu.Unlock() + go func() { + <-cancelCalled + <-releaseRun + close(runDone) + }() + + stopDone := make(chan error, 1) + go func() { stopDone <- source.Stop(context.Background()) }() + select { + case err := <-stopDone: + t.Fatalf("Stop returned before failed run cleanup: %v", err) + case <-cancelCalled: + } + close(releaseRun) + require.NoError(t, <-stopDone) + + source.mu.Lock() + assert.Equal(t, sourceStopped, source.state) + source.mu.Unlock() +} + +func TestFailedSourceStopCancellationIsRetryableAndIsolated(t *testing.T) { + first := NewSource(SourceOptions{Name: "db-one"}) + firstDone := make(chan struct{}) + first.mu.Lock() + first.state = sourceFailed + first.done = firstDone + first.mu.Unlock() + + second := NewSource(SourceOptions{Name: "db-two"}) + second.mu.Lock() + second.state = sourceFailed + second.mu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + assert.ErrorIs(t, first.Stop(ctx), context.DeadlineExceeded) + first.mu.Lock() + assert.Equal(t, sourceStopping, first.state) + first.mu.Unlock() + + // A blocked source must not hold lifecycle state for an independent + // database/source instance. + require.NoError(t, second.Stop(context.Background())) + close(firstDone) + require.NoError(t, first.Stop(context.Background())) +} + func TestClosePermanentlyRetiresSource(t *testing.T) { s := NewSource(SourceOptions{}) require.NoError(t, s.Close(context.Background())) From d7c52e06952d05888669edbb0686d9ad9720450d Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 02:39:49 -0400 Subject: [PATCH 36/47] fix(cdc): retain startup snapshot subscriptions Allow the stable source slot and PostgreSQL driver to establish subscriptions before a generation is running, so source-owned startup snapshots have a subscriber during startup. Preserve driver-specific rejection for sources without a pre-start handoff and clean snapshot-capable idle generations during replacement. Add deterministic multi-source handoff coverage. --- api/service/cdc/context.go | 6 ++- service/cdc/manager_test.go | 83 +++++++++++++++++++++++++++++ service/cdc/postgres/stream.go | 5 +- service/cdc/postgres/stream_test.go | 23 ++++++++ service/cdc/slot.go | 19 +++++-- 5 files changed, 129 insertions(+), 7 deletions(-) diff --git a/api/service/cdc/context.go b/api/service/cdc/context.go index 1f7501ba8..259c2f632 100644 --- a/api/service/cdc/context.go +++ b/api/service/cdc/context.go @@ -50,8 +50,10 @@ type Stream interface { type ErrStream = Stream // Source is the common source contract implemented by every CDC driver. -// Subscribe receives a context so a source can reject subscriptions while it -// is not ready and can bind snapshot work to the caller's lifetime. +// Subscribe receives a context so a source can bind snapshot work to the +// caller's lifetime. A source that supports startup snapshot handoff may +// accept subscriptions while it is idle or starting; sources that cannot +// establish that handoff return ErrSourceNotReady until they are running. type Source interface { Info() SourceInfo Subscribe(context.Context, StreamOptions) (Stream, error) diff --git a/service/cdc/manager_test.go b/service/cdc/manager_test.go index fc081af43..e6566962d 100644 --- a/service/cdc/manager_test.go +++ b/service/cdc/manager_test.go @@ -55,6 +55,11 @@ type blockingStopSource struct { stopOnce sync.Once } +type startupSnapshotSource struct { + *managedTestSource + snapshot api.Change +} + func newBlockingStopSource(source *managedTestSource) *blockingStopSource { return &blockingStopSource{ managedTestSource: source, @@ -63,6 +68,15 @@ func newBlockingStopSource(source *managedTestSource) *blockingStopSource { } } +func (s *startupSnapshotSource) Start(ctx context.Context) (<-chan any, error) { + status, err := s.managedTestSource.Start(ctx) + if err != nil { + return status, err + } + s.stream.changes <- s.snapshot + return status, nil +} + func (s *blockingStopSource) Stop(ctx context.Context) error { s.stopOnce.Do(func() { close(s.stopEntered) }) select { @@ -212,6 +226,75 @@ func TestManagerRoutesCanonicalIDsAndOwnsLifecycle(t *testing.T) { require.False(t, ok) } +func TestManagerPreStartSubscriptionsReceiveIndependentStartupSnapshots(t *testing.T) { + kind := registry.Kind("db.cdc.test") + aID := registry.NewID("app", "db-a") + bID := registry.NewID("app", "db-b") + aSource := &startupSnapshotSource{ + managedTestSource: &managedTestSource{ + info: api.SourceInfo{Snapshot: true}, + stream: &testStream{changes: make(chan api.Change, 1)}, + }, + snapshot: api.Change{Op: "snapshot", Table: "a"}, + } + bSource := &startupSnapshotSource{ + managedTestSource: &managedTestSource{ + info: api.SourceInfo{Snapshot: true}, + stream: &testStream{changes: make(chan api.Change, 1)}, + }, + snapshot: api.Change{Op: "snapshot", Table: "b"}, + } + driver := testDriver{ + kind: kind, + create: func(entry registry.Entry) (ManagedSource, error) { + switch entry.ID { + case aID: + return aSource, nil + case bID: + return bSource, nil + default: + return nil, errors.New("unexpected source id") + } + }, + } + m, _ := newManagerTest(t, driver) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: aID, Kind: kind})) + require.NoError(t, m.Add(context.Background(), registry.Entry{ID: bID, Kind: kind})) + + aStream, err := mustSlot(t, m, aID).Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + bStream, err := mustSlot(t, m, bID).Subscribe(context.Background(), api.StreamOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + aStream.Close() + bStream.Close() + }) + + _, err = mustSlot(t, m, aID).Start(context.Background()) + require.NoError(t, err) + _, err = mustSlot(t, m, bID).Start(context.Background()) + require.NoError(t, err) + + select { + case change := <-aStream.Changes(): + require.Equal(t, "a", change.Table) + require.Equal(t, aID, change.SourceID) + require.Equal(t, aID.String(), change.Source) + case <-time.After(time.Second): + t.Fatal("source A startup snapshot was lost") + } + select { + case change := <-bStream.Changes(): + require.Equal(t, "b", change.Table) + require.Equal(t, bID, change.SourceID) + require.Equal(t, bID.String(), change.Source) + case <-time.After(time.Second): + t.Fatal("source B startup snapshot was lost") + } + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: aID, Kind: kind})) + require.NoError(t, m.Delete(context.Background(), registry.Entry{ID: bID, Kind: kind})) +} + func TestManagerDeleteInvokesDisposeOnlyAfterUnregister(t *testing.T) { source := &disposableTestSource{managedTestSource: &managedTestSource{info: api.SourceInfo{Name: "source"}}} driver := testDriver{ diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 7a61ed514..770bdf0ed 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -50,7 +50,7 @@ func (s *Source) Subscribe(opts config.StreamOptions) config.Stream { } s.mu.Lock() defer s.mu.Unlock() - if s.state != sourceNew && s.state != sourceRunning { + if s.state != sourceNew && s.state != sourceStarting && s.state != sourceRunning { return nil } return s.newSubscription(opts) @@ -70,7 +70,8 @@ func (s *Source) subscribe(ctx context.Context, opts config.StreamOptions) (conf s.mu.Lock() defer s.mu.Unlock() - if s.state != sourceRunning || s.permanentlyClosed || s.sourceErr != nil { + if (s.state != sourceNew && s.state != sourceStarting && s.state != sourceRunning) || + s.permanentlyClosed || s.sourceErr != nil { return nil, config.ErrSourceNotReady } return s.newSubscription(opts), nil diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 6e2539029..8aa80de68 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -41,6 +41,29 @@ func TestSourceSubscribePublishesMatchingChanges(t *testing.T) { } } +func TestSourceSubscribePreparesStartupSnapshotBeforeStart(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + stream, err := src.subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 1}) + require.NoError(t, err) + require.NotNil(t, stream) + defer stream.Close() + + src.publishChange(context.Background(), cdcapi.Change{ + Op: "snapshot", + Table: "accounts", + After: map[string]any{"id": int64(1)}, + Source: "test:cdc", + }) + + select { + case got := <-stream.Changes(): + require.Equal(t, "snapshot", got.Op) + require.Equal(t, "accounts", got.Table) + case <-time.After(time.Second): + t.Fatal("pre-start subscription did not retain startup snapshot") + } +} + func TestSourceSubscribeFiltersChanges(t *testing.T) { src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) stream := src.Subscribe(cdcapi.StreamOptions{ diff --git a/service/cdc/slot.go b/service/cdc/slot.go index cb1de789d..edf001f3d 100644 --- a/service/cdc/slot.go +++ b/service/cdc/slot.go @@ -116,12 +116,18 @@ func (s *sourceSlot) Info() api.SourceInfo { return info } +// Subscribe delegates pre-start subscriptions to drivers that can retain the +// registration until Start establishes the generation. This is required for +// source-owned startup snapshots; drivers without that handoff return +// ErrSourceNotReady. The stable slot still rejects stopped, replacing, and +// disposing generations. func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api.Stream, error) { if err := opts.Validate(); err != nil { return nil, err } s.mu.RLock() - if s.state != slotRunning || isNilSource(s.current) || s.disposing || s.replacing { + preStart := s.state == slotIdle || s.state == slotStarting + if (!preStart && s.state != slotRunning) || isNilSource(s.current) || s.disposing || s.replacing { s.mu.RUnlock() return nil, api.ErrSourceNotReady } @@ -137,7 +143,8 @@ func (s *sourceSlot) Subscribe(ctx context.Context, opts api.StreamOptions) (api } s.mu.RLock() - stillCurrent := s.state == slotRunning && s.current == current && s.generation == generation && !s.replacing + stillCurrent := (s.state == slotIdle || s.state == slotStarting || s.state == slotRunning) && + s.current == current && s.generation == generation && !s.replacing s.mu.RUnlock() if !stillCurrent { stream.Close() @@ -365,7 +372,13 @@ func (s *sourceSlot) Replace(ctx context.Context, candidate ManagedSource, oldLe s.mu.Unlock() startCandidate := oldState == slotRunning || lifecycleAutoStart(candidate) - shouldStopOld := !isNilSource(old) && (oldState != slotStopped && oldState != slotIdle || differentResource || oldKey == "") + // A source configured with a startup snapshot may have accepted a + // pre-start subscription while the stable slot was idle. Stop that old + // generation on replacement so its driver can close the prepared stream; + // ordinary idle sources retain the historical no-op handoff. + oldHasStartupSnapshot := oldState == slotIdle && !isNilSource(old) && old.Info().Snapshot + shouldStopOld := !isNilSource(old) && + (oldState != slotStopped && oldState != slotIdle || differentResource || oldKey == "" || oldHasStartupSnapshot) var ( underlying <-chan any From 06b19d4d37d5f62ce6d73edc11cef316c9db01ee Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 03:14:44 -0400 Subject: [PATCH 37/47] fix(cdc): bound retained messages across scheduler and Lua --- api/process/errors.go | 6 + api/process/queue.go | 274 +++++++++++++++++- api/process/queue_limits_test.go | 77 +++++ api/relay/pool.go | 3 + api/relay/relay.go | 10 + api/service/cdc/size.go | 19 +- cluster/internode/codec.go | 17 +- cluster/internode/codec_test.go | 8 +- .../lua/engine/message_queue_bytes_test.go | 151 ++++++++++ runtime/lua/engine/process.go | 270 ++++++++++++++++- runtime/lua/modules/cdc/module.go | 7 +- service/cdc/dispatcher.go | 31 +- system/scheduler/actor/scheduler.go | 8 +- system/scheduler/pool/pool.go | 12 +- 14 files changed, 852 insertions(+), 41 deletions(-) create mode 100644 api/process/queue_limits_test.go create mode 100644 runtime/lua/engine/message_queue_bytes_test.go diff --git a/api/process/errors.go b/api/process/errors.go index aaf80ae07..2658d08be 100644 --- a/api/process/errors.go +++ b/api/process/errors.go @@ -27,6 +27,12 @@ var ( ErrProcessNotIdle = apierror.New(InvalidState, "process is not idle").WithRetryable(apierror.False) ErrSchedulerStopping = apierror.New(InvalidState, "scheduler is stopping").WithRetryable(apierror.False) + + // ErrMessageQueueOverflow is delivered as a terminal stream error when an + // explicitly bounded message subscription exhausts its retained backlog. + // It is intentionally an ordinary Go sentinel so consumers can use + // errors.Is on the error carried by the terminal payload. + ErrMessageQueueOverflow = errors.New("message queue limit exceeded") ) // ErrProcessReplacementRequested is an internal scheduler sentinel. A process diff --git a/api/process/queue.go b/api/process/queue.go index 12dec055a..87d758f39 100644 --- a/api/process/queue.go +++ b/api/process/queue.go @@ -5,6 +5,9 @@ package process import ( "sync" "sync/atomic" + + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/relay" ) // todo: move from api @@ -17,14 +20,38 @@ const defaultQueueCap = 16 // Generation counter ensures stale senders from previous executions // cannot push to a reused queue. type EventQueue struct { - signal chan struct{} - events []Event - drainBuf []Event + signal chan struct{} + events []Event + drainBuf []Event + + // Message accounting is opt-in. Ordinary event traffic keeps the + // historical unbounded queue semantics; CDC messages carry MaxItems and/or + // MaxBytes and are admitted through PushMessage. + messageItems map[string]int + messageBytes map[string]int64 + messageItemLimits map[string]int + messageByteLimits map[string]int64 + messageOverflowed map[string]struct{} + generation atomic.Uint64 mu sync.Mutex closed atomic.Bool } +// MessageAdmission describes ownership after PushMessage. +// +// Accepted means the queue owns the package. Dropped means the queue emitted +// its overflow terminal but retained no part of the supplied package, so the +// caller must release it. Rejected means the queue did not admit the package +// (closed or stale generation), and the caller must release it as well. +type MessageAdmission uint8 + +const ( + MessageRejected MessageAdmission = iota + MessageDropped + MessageAccepted +) + // NewEventQueue creates a queue with default capacity. func NewEventQueue() *EventQueue { q := &EventQueue{ @@ -61,12 +88,185 @@ func (q *EventQueue) Push(e Event, gen uint64) bool { q.events = append(q.events, e) q.mu.Unlock() - // Non-blocking signal + q.signalPush() + return true +} + +func (q *EventQueue) signalPush() { select { case q.signal <- struct{}{}: default: } - return true +} + +// PushMessage admits a relay package while enforcing the per-topic limits +// carried by its messages. A package can contain messages for more than one +// topic; each message is admitted independently and the package is compacted +// before ownership transfers to the queue. On the first overflow for a topic, +// one synthetic error+terminal message is appended in its position. Later +// traffic for that topic is discarded until Reset. +// +// The queue owns an accepted package and the scheduler releases it after +// processing. The caller owns rejected or fully dropped packages. +func (q *EventQueue) PushMessage(e Event, gen uint64) MessageAdmission { + if e.Type != EventMessage { + if q.Push(e, gen) { + return MessageAccepted + } + return MessageRejected + } + pkg, ok := e.Data.(*relay.Package) + if !ok || pkg == nil { + if q.Push(e, gen) { + return MessageAccepted + } + return MessageRejected + } + + if q.generation.Load() != gen || q.closed.Load() { + return MessageRejected + } + + q.mu.Lock() + if q.generation.Load() != gen || q.closed.Load() { + q.mu.Unlock() + return MessageRejected + } + accepted := q.admitPackageLocked(pkg) + if !accepted { + q.mu.Unlock() + return MessageDropped + } + q.events = append(q.events, e) + q.mu.Unlock() + + q.signalPush() + return MessageAccepted +} + +func (q *EventQueue) admitPackageLocked(pkg *relay.Package) bool { + original := pkg.Messages + if len(original) == 0 { + return true + } + + accepted := make([]*relay.Message, 0, len(original)+1) + for _, msg := range original { + if msg == nil { + accepted = append(accepted, nil) + continue + } + + topic := string(msg.Topic) + maxItems := msg.MaxItems + if maxItems <= 0 { + maxItems = q.messageItemLimits[topic] + } + maxBytes := msg.MaxBytes + if maxBytes <= 0 { + maxBytes = q.messageByteLimits[topic] + } + if maxItems > 0 { + if previous := q.messageItemLimits[topic]; previous > 0 && previous < maxItems { + maxItems = previous + } + if q.messageItemLimits == nil { + q.messageItemLimits = make(map[string]int) + } + q.messageItemLimits[topic] = maxItems + } + if maxBytes > 0 { + if previous := q.messageByteLimits[topic]; previous > 0 && previous < maxBytes { + maxBytes = previous + } + if q.messageByteLimits == nil { + q.messageByteLimits = make(map[string]int64) + } + q.messageByteLimits[topic] = maxBytes + } + + if _, overflowed := q.messageOverflowed[topic]; overflowed { + relay.ReleaseMessage(msg) + continue + } + + // A terminal never consumes backlog capacity. This also makes the + // synthetic overflow terminal admissible after a full backlog. + if !messageHasData(msg) { + if maxItems > 0 { + msg.MaxItems = maxItems + } + if maxBytes > 0 { + msg.MaxBytes = maxBytes + } + accepted = append(accepted, msg) + continue + } + + payloadBytes := msg.PayloadBytes + if maxBytes > 0 && payloadBytes <= 0 { + // Missing size metadata must not bypass a byte budget. + payloadBytes = maxBytes + } + items := q.messageItems[topic] + bytes := q.messageBytes[topic] + if (maxItems > 0 && items >= maxItems) || + (maxBytes > 0 && (payloadBytes > maxBytes || bytes > maxBytes-payloadBytes)) { + accepted = q.messageOverflowedLocked(topic, accepted, maxItems, maxBytes) + relay.ReleaseMessage(msg) + continue + } + + msg.MaxItems = maxItems + msg.MaxBytes = maxBytes + msg.PayloadBytes = payloadBytes + if maxItems > 0 { + if q.messageItems == nil { + q.messageItems = make(map[string]int) + } + q.messageItems[topic] = items + 1 + } + if maxBytes > 0 && payloadBytes > 0 { + if q.messageBytes == nil { + q.messageBytes = make(map[string]int64) + } + q.messageBytes[topic] = bytes + payloadBytes + } + accepted = append(accepted, msg) + } + + pkg.Messages = accepted + return len(accepted) > 0 +} + +func (q *EventQueue) messageOverflowedLocked(topic string, accepted []*relay.Message, maxItems int, maxBytes int64) []*relay.Message { + if q.messageOverflowed == nil { + q.messageOverflowed = make(map[string]struct{}) + } + if _, exists := q.messageOverflowed[topic]; exists { + return accepted + } + q.messageOverflowed[topic] = struct{}{} + msg := relay.AcquireMessage() + msg.Topic = topic + msg.Payloads = payload.Payloads{payload.NewError(ErrMessageQueueOverflow), payload.NewTerminal()} + msg.MaxItems = maxItems + msg.MaxBytes = maxBytes + msg.PayloadBytes = 0 + return append(accepted, msg) +} + +func messageHasData(msg *relay.Message) bool { + if msg == nil { + return false + } + for _, pl := range msg.Payloads { + if pl == nil || payload.IsTerminal(pl) || pl.Format() == payload.GoError { + continue + } + return true + } + return false } // PushDirect adds an event without generation check (for scheduler's own use). @@ -75,10 +275,7 @@ func (q *EventQueue) PushDirect(e Event) { q.events = append(q.events, e) q.mu.Unlock() - select { - case q.signal <- struct{}{}: - default: - } + q.signalPush() } // Drain returns all pending events and clears the queue. @@ -90,6 +287,9 @@ func (q *EventQueue) Drain() []Event { q.mu.Unlock() return nil } + for _, event := range q.events { + q.releaseEventLocked(event) + } // Swap buffers to avoid allocation q.drainBuf, q.events = q.events, q.drainBuf[:0] @@ -116,7 +316,12 @@ func (q *EventQueue) Signal() <-chan struct{} { func (q *EventQueue) Close() { q.mu.Lock() q.closed.Store(true) + for _, event := range q.events { + q.releaseEventLocked(event) + q.releaseEventPackageLocked(event) + } q.events = q.events[:0] + q.clearMessageAccountingLocked() q.mu.Unlock() // Wake any waiters @@ -131,8 +336,12 @@ func (q *EventQueue) Reset() { q.mu.Lock() q.generation.Add(1) // Invalidate all existing senders q.closed.Store(false) + for _, event := range q.events { + q.releaseEventPackageLocked(event) + } q.events = q.events[:0] q.drainBuf = q.drainBuf[:0] + q.clearMessageAccountingLocked() q.mu.Unlock() // Drain signal channel @@ -142,6 +351,53 @@ func (q *EventQueue) Reset() { } } +func (q *EventQueue) releaseEventLocked(event Event) { + if event.Type != EventMessage { + return + } + pkg, ok := event.Data.(*relay.Package) + if !ok || pkg == nil { + return + } + for _, msg := range pkg.Messages { + if !messageHasData(msg) { + continue + } + topic := string(msg.Topic) + if msg.MaxItems > 0 && q.messageItems != nil { + if n := q.messageItems[topic] - 1; n > 0 { + q.messageItems[topic] = n + } else { + delete(q.messageItems, topic) + } + } + if msg.MaxBytes > 0 && msg.PayloadBytes > 0 && q.messageBytes != nil { + if n := q.messageBytes[topic] - msg.PayloadBytes; n > 0 { + q.messageBytes[topic] = n + } else { + delete(q.messageBytes, topic) + } + } + } +} + +func (q *EventQueue) releaseEventPackageLocked(event Event) { + if event.Type != EventMessage { + return + } + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } +} + +func (q *EventQueue) clearMessageAccountingLocked() { + clear(q.messageItems) + clear(q.messageBytes) + clear(q.messageItemLimits) + clear(q.messageByteLimits) + clear(q.messageOverflowed) +} + // YieldScheduler is the subset of Scheduler needed for waking. type YieldScheduler interface { WakeProcessor(q *EventQueue, gen uint64) diff --git a/api/process/queue_limits_test.go b/api/process/queue_limits_test.go new file mode 100644 index 000000000..b0e07faa4 --- /dev/null +++ b/api/process/queue_limits_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MPL-2.0 + +package process + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/relay" +) + +func boundedPackage(topic string, items int, bytes int64, value string) *relay.Package { + pkg := relay.NewPackage(pid.PID{}, pid.PID{}, topic, payload.NewString(value)) + pkg.Messages[0].MaxItems = items + pkg.Messages[0].MaxBytes = bytes + pkg.Messages[0].PayloadBytes = bytes + return pkg +} + +func TestEventQueueMessageAdmissionEmitsOneTerminal(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + + first := boundedPackage("cdc:a", 1, 100, "first") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: first}, gen)) + + second := boundedPackage("cdc:a", 1, 100, "second") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: second}, gen), "overflow terminal remains admissible") + require.Len(t, second.Messages, 1) + require.True(t, payload.IsTerminal(second.Messages[0].Payloads[len(second.Messages[0].Payloads)-1])) + + late := boundedPackage("cdc:a", 1, 100, "late") + require.Equal(t, MessageDropped, q.PushMessage(Event{Type: EventMessage, Data: late}, gen)) + require.Empty(t, late.Messages, "caller owns and releases a fully dropped package") + relay.ReleasePackage(late) + + events := q.Drain() + require.Len(t, events, 2) + for _, event := range events { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } + + // A reset clears the overflow tombstone and permits a new stream + // incarnation to reuse the same topic. + q.Reset() + reuse := boundedPackage("cdc:a", 1, 100, "reuse") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: reuse}, q.Generation())) + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} + +func TestEventQueueMessageLimitsArePerTopic(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + + for _, topic := range []string{"cdc:a", "cdc:b"} { + first := boundedPackage(topic, 1, 0, "first") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: first}, gen)) + second := boundedPackage(topic, 1, 0, "second") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: second}, gen)) + } + + events := q.Drain() + require.Len(t, events, 4) + for _, event := range events { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} diff --git a/api/relay/pool.go b/api/relay/pool.go index 2ea0c2a4d..6fd333c73 100644 --- a/api/relay/pool.go +++ b/api/relay/pool.go @@ -40,6 +40,9 @@ func ReleaseMessage(m *Message) { } m.Topic = "" m.Payloads = nil + m.PayloadBytes = 0 + m.MaxBytes = 0 + m.MaxItems = 0 messagePool.Put(m) } diff --git a/api/relay/relay.go b/api/relay/relay.go index bcdd5b255..615baa774 100644 --- a/api/relay/relay.go +++ b/api/relay/relay.go @@ -39,6 +39,16 @@ type ( Message struct { Topic Topic Payloads payload.Payloads + // PayloadBytes is the logical retained size of Payloads. It is + // optional metadata used by bounded subscribers; zero preserves the + // historical unbounded relay behavior. + PayloadBytes int64 + // MaxBytes is the per-destination backlog limit for this message's + // topic. Zero means that the destination applies no byte limit. + MaxBytes int64 + // MaxItems is the per-destination message backlog limit for this + // topic. Zero means that the destination applies no item limit. + MaxItems int } // Package combines source, target and messages for delivery. diff --git a/api/service/cdc/size.go b/api/service/cdc/size.go index d982e245e..a66398ff4 100644 --- a/api/service/cdc/size.go +++ b/api/service/cdc/size.go @@ -8,10 +8,13 @@ const ( // DefaultMaxStreamBytes bounds a subscriber's retained event backlog when // MaxBytes is omitted. It is deliberately finite for every driver. DefaultMaxStreamBytes int64 = 64 << 20 - changeStructuralBytes = 128 - valueStructuralBytes = 24 - maxEstimateDepth = 256 - maxEstimateNodes = 1 << 20 + // DefaultMaxStreamItems is the historical Lua CDC stream capacity. It is + // also used by direct Go callers so process admission is always bounded. + DefaultMaxStreamItems = 64 + changeStructuralBytes = 128 + valueStructuralBytes = 24 + maxEstimateDepth = 256 + maxEstimateNodes = 1 << 20 ) // ValidateStreamOptions validates common stream resource limits. Buffer keeps @@ -33,6 +36,14 @@ func (o StreamOptions) EffectiveMaxBytes() int64 { return DefaultMaxStreamBytes } +// EffectiveMaxStreamItems returns the finite item limit selected by options. +func (o StreamOptions) EffectiveMaxStreamItems() int { + if o.Buffer > 0 { + return o.Buffer + } + return DefaultMaxStreamItems +} + // EstimateChangeBytes returns a conservative logical retained-size estimate // for a Change and all nested values in its before/after images. It counts // strings and byte blobs by length, includes container structure, saturates diff --git a/cluster/internode/codec.go b/cluster/internode/codec.go index 6c43ce319..85868ee54 100644 --- a/cluster/internode/codec.go +++ b/cluster/internode/codec.go @@ -21,8 +21,11 @@ type encodedPayload struct { } type encodedMessage struct { - Topic string - Payloads []encodedPayload + Topic string + Payloads []encodedPayload + PayloadBytes int64 + MaxBytes int64 + MaxItems int } type encodedPackage struct { @@ -101,8 +104,11 @@ func (c *MessageCodec) Encode(pkg *relay.Package) ([]byte, error) { for i, msg := range pkg.Messages { encMsg := &encodedMessage{ - Topic: msg.Topic, - Payloads: make([]encodedPayload, len(msg.Payloads)), + Topic: msg.Topic, + Payloads: make([]encodedPayload, len(msg.Payloads)), + PayloadBytes: msg.PayloadBytes, + MaxBytes: msg.MaxBytes, + MaxItems: msg.MaxItems, } for j, p := range msg.Payloads { @@ -159,6 +165,9 @@ func (c *MessageCodec) Decode(data []byte) (*relay.Package, error) { for i, encMsg := range encPkg.Messages { finalMsg := relay.AcquireMessage() finalMsg.Topic = encMsg.Topic + finalMsg.PayloadBytes = encMsg.PayloadBytes + finalMsg.MaxBytes = encMsg.MaxBytes + finalMsg.MaxItems = encMsg.MaxItems finalMsg.Payloads = make(payload.Payloads, len(encMsg.Payloads)) for j, encP := range encMsg.Payloads { diff --git a/cluster/internode/codec_test.go b/cluster/internode/codec_test.go index 7a6d1a0aa..92211920d 100644 --- a/cluster/internode/codec_test.go +++ b/cluster/internode/codec_test.go @@ -85,7 +85,10 @@ func TestMessageCodec_PackagePIDs_SourceTarget(t *testing.T) { Target: targetPID, Messages: []*relay.Message{ { - Topic: "test.topic", + Topic: "test.topic", + PayloadBytes: 4096, + MaxBytes: 8192, + MaxItems: 7, Payloads: []payload.Payload{ payload.NewString("test message"), }, @@ -140,6 +143,9 @@ func TestMessageCodec_PackagePIDs_SourceTarget(t *testing.T) { if decoded.Messages[0].Topic != "test.topic" { t.Errorf("Topic mismatch. Expected 'test.topic', got %q", decoded.Messages[0].Topic) } + if decoded.Messages[0].PayloadBytes != 4096 || decoded.Messages[0].MaxBytes != 8192 || decoded.Messages[0].MaxItems != 7 { + t.Errorf("retention metadata mismatch: bytes=%d max_bytes=%d max_items=%d", decoded.Messages[0].PayloadBytes, decoded.Messages[0].MaxBytes, decoded.Messages[0].MaxItems) + } } func TestMessageCodec_ConcurrentEncodeSharedMapPayload(t *testing.T) { diff --git a/runtime/lua/engine/message_queue_bytes_test.go b/runtime/lua/engine/message_queue_bytes_test.go new file mode 100644 index 000000000..9a7c765fb --- /dev/null +++ b/runtime/lua/engine/message_queue_bytes_test.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "bytes" + "context" + "sync/atomic" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" +) + +func TestMessageQueueByteLimitRejectsLargePayload(t *testing.T) { + proc := mustNewProcess(t, WithScript("return 1", "test.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + const topic = "cdc.large" + const limit = int64(1 << 20) + ch := NewChannel(1) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, func(context.Context, *lua.LState, pid.PID, string, []payload.Payload) lua.LValue { + return lua.LTrue + }) + var cleanup atomic.Bool + if !proc.SetSubscriptionCleanup(ch, func() { cleanup.Store(true) }) { + t.Fatal("subscription cleanup was not installed") + } + + blob := bytes.Repeat([]byte{'x'}, int(limit)+1) + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.New(blob)}, + PayloadBytes: int64(len(blob)), + MaxBytes: limit, + }) + + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("expected one synthetic terminal, got %d queued messages", got) + } + if got := proc.messageQueue[0].Payloads[0].Format(); got != payload.GoError { + t.Fatalf("expected overflow error payload, got %q", got) + } + if got := proc.messageQueueBytes[topic]; got != 0 { + t.Fatalf("overflow retained %d bytes", got) + } + if !cleanup.Load() { + t.Fatal("overflow did not stop the existing producer") + } + + proc.flushMessageQueue(proc.subs) + if !ch.IsClosed() { + t.Fatal("overflow terminal did not close the subscription") + } +} + +func TestMessageQueueByteLimitBoundsSlowConsumer(t *testing.T) { + proc := mustNewProcess(t, WithScript("return 1", "test.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + const topic = "cdc.slow" + const messageBytes = int64(100) + const limit = int64(128) + ch := NewChannel(0) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, func(context.Context, *lua.LState, pid.PID, string, []payload.Payload) lua.LValue { + return lua.LTrue + }) + var cleanup atomic.Bool + if !proc.SetSubscriptionCleanup(ch, func() { cleanup.Store(true) }) { + t.Fatal("subscription cleanup was not installed") + } + + message := func() queuedMessage { + return queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("change")}, + PayloadBytes: messageBytes, + MaxBytes: limit, + } + } + + // With a rendezvous channel and no waiting consumer, the first value stays + // in Process.messageQueue; the next value exceeds the byte budget. + proc.enqueueMessage(message()) + proc.flushMessageQueue(proc.subs) + proc.enqueueMessage(message()) + if got := proc.messageQueueBytes[topic]; got != messageBytes { + t.Fatalf("expected %d retained bytes, got %d", messageBytes, got) + } + + // Further values are dropped after exactly one terminal is queued. + for i := 0; i < 100; i++ { + proc.enqueueMessage(message()) + } + if got := proc.messageQueueBytes[topic]; got > limit { + t.Fatalf("retained %d bytes above limit %d", got, limit) + } + if got := len(proc.messageQueue); got != 2 { + t.Fatalf("expected retained value plus terminal, got %d messages", got) + } + if !cleanup.Load() { + t.Fatal("overflow did not stop the existing producer") + } +} + +func TestMessageQueueItemLimitIsPerTopicAndTerminalOrdered(t *testing.T) { + proc := mustNewProcess(t, WithScript("return 1", "test.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + const limit = 2 + for _, topic := range []string{"cdc.one", "cdc.two"} { + ch := NewChannel(0) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + for i := 0; i < limit+4; i++ { + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("change")}, + MaxItems: limit, + }) + } + } + + if got := len(proc.messageQueue); got != 2*(limit+1) { + t.Fatalf("expected two bounded queues with data plus terminal, got %d", got) + } + if got := len(proc.messageQueueOverflowed); got != 2 { + t.Fatalf("expected independent overflow tombstones, got %d", got) + } +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 7a91b9a88..c3e23bf56 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -110,8 +110,17 @@ type Process struct { externalTasks []*Task yieldBuf []*Task messageQueue []queuedMessage - threads []*Task - yieldSeq uint64 + // messageQueueBytes accounts only messages that opt into a byte limit via + // relay metadata. Ordinary process messages keep their historical behavior. + messageQueueItems map[string]int + messageQueueBytes map[string]int64 + messageQueueItemLimits map[string]int + messageQueueLimits map[string]int64 + messageQueueOverflowed map[string]struct{} + messageQueueDiscarded map[string]struct{} + flushingMessages bool + threads []*Task + yieldSeq uint64 // epoch is the monotonic incarnation counter. Incremented on every // Init / clearExecution / Close drain and on Abort. Producers stamp // every SubscriptionFrame with the epoch they were registered under; @@ -124,9 +133,12 @@ type Process struct { // queuedMessage stores a message waiting to be delivered type queuedMessage struct { - Source pid.PID - Topic string - Payloads []payload.Payload + Source pid.PID + Topic string + Payloads []payload.Payload + MaxItems int + PayloadBytes int64 + MaxBytes int64 } // GetProcess retrieves the Process from LState via Owner. @@ -196,16 +208,29 @@ func (p *Process) SetSubscriptionCleanup(ch *Channel, fn func()) bool { return false } p.subs.mu.Lock() - defer p.subs.mu.Unlock() topic, ok := p.subs.byChannel[ch] if !ok { + p.subs.mu.Unlock() return false } sub := p.subs.byTopic[topic] if sub == nil { + p.subs.mu.Unlock() return false } sub.cleanup = fn + overflowed := false + if _, exists := p.messageQueueOverflowed[topic]; exists { + overflowed = true + } + p.subs.mu.Unlock() + // If admission overflowed before the Lua subscription yield completed, + // stop the source as soon as its cleanup hook becomes available. Do this + // outside the subscription lock because cleanup may unsubscribe the same + // channel. + if overflowed { + sub.callCleanup() + } return true } @@ -253,6 +278,14 @@ func (p *Process) closeChannel(ch *Channel) bool { sub.gen.Add(1) sub.callCleanup() } + if p.flushingMessages { + if p.messageQueueDiscarded == nil { + p.messageQueueDiscarded = make(map[string]struct{}) + } + p.messageQueueDiscarded[topic] = struct{}{} + } else { + p.discardMessageTopic(topic) + } if !ch.IsClosed() { p.applyExternalChannelResult(ch.Close(nil)) } @@ -537,6 +570,13 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload // Clear message queue p.messageQueue = p.messageQueue[:0] + clear(p.messageQueueItems) + clear(p.messageQueueBytes) + clear(p.messageQueueItemLimits) + clear(p.messageQueueLimits) + clear(p.messageQueueOverflowed) + clear(p.messageQueueDiscarded) + p.flushingMessages = false p.pendingOutdated = nil // Seal the frame - no more modifications allowed after this @@ -672,10 +712,13 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // Add incoming messages to queue first (before any processing) for _, pkg := range messages { for _, msg := range pkg.Messages { - p.messageQueue = append(p.messageQueue, queuedMessage{ - Source: pkg.Source, - Topic: msg.Topic, - Payloads: msg.Payloads, + p.enqueueMessage(queuedMessage{ + Source: pkg.Source, + Topic: msg.Topic, + Payloads: msg.Payloads, + MaxItems: msg.MaxItems, + PayloadBytes: msg.PayloadBytes, + MaxBytes: msg.MaxBytes, }) } relay.ReleasePackage(pkg) @@ -1057,12 +1100,17 @@ func (p *Process) flushMessageQueue(subs *subscribeContext) { // Process queue, retaining undelivered messages in order. remaining := p.messageQueue[:0] + p.flushingMessages = true for _, qm := range p.messageQueue { if p.deliverMessage(subs, qm) { remaining = append(remaining, qm) // retain in queue + } else { + p.releaseQueuedMessage(qm) } } + p.flushingMessages = false p.messageQueue = remaining + p.finishDiscardedTopics() } // A coalesced OUTDATED event lives outside the queue in a single slot and is @@ -1072,6 +1120,198 @@ func (p *Process) flushMessageQueue(subs *subscribeContext) { } } +// enqueueMessage is the single handoff from relay delivery into the process +// mailbox. Limits are opt-in: only messages carrying MaxItems/MaxBytes are +// bounded, so unrelated process topics retain their historical behavior. +func (p *Process) enqueueMessage(qm queuedMessage) { + if _, discarded := p.messageQueueDiscarded[qm.Topic]; discarded { + return + } + if qm.MaxItems <= 0 { + qm.MaxItems = p.messageQueueItemLimits[qm.Topic] + } + if qm.MaxBytes <= 0 { + qm.MaxBytes = p.messageQueueLimits[qm.Topic] + } + if qm.MaxItems > 0 { + if p.messageQueueItemLimits == nil { + p.messageQueueItemLimits = make(map[string]int) + } + if previous := p.messageQueueItemLimits[qm.Topic]; previous > 0 && previous < qm.MaxItems { + qm.MaxItems = previous + } + p.messageQueueItemLimits[qm.Topic] = qm.MaxItems + } + if qm.MaxBytes > 0 { + if p.messageQueueLimits == nil { + p.messageQueueLimits = make(map[string]int64) + } + if previous := p.messageQueueLimits[qm.Topic]; previous > 0 && previous < qm.MaxBytes { + qm.MaxBytes = previous + } + p.messageQueueLimits[qm.Topic] = qm.MaxBytes + } + if !hasDataPayload(qm.Payloads) { + if _, overflowed := p.messageQueueOverflowed[qm.Topic]; overflowed { + return + } + if isOverflowTerminal(qm.Payloads) { + if p.messageQueueOverflowed == nil { + p.messageQueueOverflowed = make(map[string]struct{}) + } + p.messageQueueOverflowed[qm.Topic] = struct{}{} + if p.subs != nil { + if sub, ok := p.subs.get(qm.Topic); ok { + sub.callCleanup() + } + } + } + // Terminals are always admissible and never consume backlog capacity. + p.messageQueue = append(p.messageQueue, qm) + return + } + + if _, overflowed := p.messageQueueOverflowed[qm.Topic]; overflowed { + return + } + + if qm.MaxItems > 0 || qm.MaxBytes > 0 { + // A bounded producer must provide a conservative size. If it does not, + // charge the whole budget rather than retaining an unaccounted value. + if qm.PayloadBytes <= 0 && hasDataPayload(qm.Payloads) { + if qm.MaxBytes > 0 { + qm.PayloadBytes = qm.MaxBytes + } + } + queuedItems := p.messageQueueItems[qm.Topic] + queuedBytes := p.messageQueueBytes[qm.Topic] + if (qm.MaxItems > 0 && queuedItems >= qm.MaxItems) || + (qm.MaxBytes > 0 && (qm.PayloadBytes > qm.MaxBytes || queuedBytes > qm.MaxBytes-qm.PayloadBytes)) { + p.overflowMessageQueue(qm) + return + } + if qm.MaxItems > 0 { + if p.messageQueueItems == nil { + p.messageQueueItems = make(map[string]int) + } + p.messageQueueItems[qm.Topic] = queuedItems + 1 + } + if qm.MaxBytes > 0 && qm.PayloadBytes > 0 { + if p.messageQueueBytes == nil { + p.messageQueueBytes = make(map[string]int64) + } + p.messageQueueBytes[qm.Topic] = queuedBytes + qm.PayloadBytes + } + } + p.messageQueue = append(p.messageQueue, qm) +} + +func (p *Process) overflowMessageQueue(qm queuedMessage) { + if p.messageQueueOverflowed == nil { + p.messageQueueOverflowed = make(map[string]struct{}) + } + if _, exists := p.messageQueueOverflowed[qm.Topic]; exists { + return + } + p.messageQueueOverflowed[qm.Topic] = struct{}{} + + // Stop the producer through the subscription's existing ownership hook. + // The channel is closed by the terminal below on the process step goroutine. + if p.subs != nil { + if sub, ok := p.subs.get(qm.Topic); ok { + sub.callCleanup() + } + } + p.messageQueue = append(p.messageQueue, queuedMessage{ + Source: qm.Source, + Topic: qm.Topic, + Payloads: payload.Payloads{payload.NewError(process.ErrMessageQueueOverflow), payload.NewTerminal()}, + MaxItems: qm.MaxItems, + MaxBytes: qm.MaxBytes, + }) +} + +func (p *Process) releaseQueuedMessage(qm queuedMessage) { + if qm.MaxItems > 0 && p.messageQueueItems != nil { + remaining := p.messageQueueItems[qm.Topic] - 1 + if remaining > 0 { + p.messageQueueItems[qm.Topic] = remaining + } else { + delete(p.messageQueueItems, qm.Topic) + } + } + if qm.MaxBytes > 0 && qm.PayloadBytes > 0 && p.messageQueueBytes != nil { + remaining := p.messageQueueBytes[qm.Topic] - qm.PayloadBytes + if remaining > 0 { + p.messageQueueBytes[qm.Topic] = remaining + } else { + delete(p.messageQueueBytes, qm.Topic) + } + } +} + +func (p *Process) discardMessageTopic(topic string) { + remaining := p.messageQueue[:0] + for _, qm := range p.messageQueue { + if qm.Topic == topic { + p.releaseQueuedMessage(qm) + continue + } + remaining = append(remaining, qm) + } + p.messageQueue = remaining + delete(p.messageQueueItems, topic) + delete(p.messageQueueBytes, topic) + delete(p.messageQueueItemLimits, topic) + delete(p.messageQueueLimits, topic) + delete(p.messageQueueOverflowed, topic) + delete(p.messageQueueDiscarded, topic) +} + +func (p *Process) finishDiscardedTopics() { + for topic := range p.messageQueueDiscarded { + found := false + for _, qm := range p.messageQueue { + if qm.Topic == topic { + found = true + break + } + } + if !found { + delete(p.messageQueueItems, topic) + delete(p.messageQueueBytes, topic) + delete(p.messageQueueItemLimits, topic) + delete(p.messageQueueLimits, topic) + delete(p.messageQueueOverflowed, topic) + delete(p.messageQueueDiscarded, topic) + } + } +} + +func hasDataPayload(payloads payload.Payloads) bool { + for _, pl := range payloads { + if pl == nil || payload.IsTerminal(pl) || pl.Format() == payload.GoError { + continue + } + return true + } + return false +} + +func isOverflowTerminal(payloads payload.Payloads) bool { + if len(payloads) == 0 || !payload.IsTerminal(payloads[len(payloads)-1]) { + return false + } + for _, pl := range payloads { + if pl == nil || pl.Format() != payload.GoError { + continue + } + err, ok := pl.Data().(error) + return ok && errors.Is(err, process.ErrMessageQueueOverflow) + } + return false +} + // markStalled records that a channel retained a message in the current flush // pass, lazily creating the set. func (p *Process) markStalled(ch *Channel) { @@ -1100,6 +1340,9 @@ func (p *Process) isStalled(ch *Channel) bool { // stall; pure data retries normally and a full buffer preserves its order. func (p *Process) deliverMessage(subs *subscribeContext, qm queuedMessage) (keep bool) { topic := qm.Topic + if _, discarded := p.messageQueueDiscarded[topic]; discarded { + return false + } handlerTopic := topic frame, hasFrame := subscriptionFrameFromPayloads(qm.Payloads) @@ -1583,6 +1826,13 @@ func (p *Process) Close() { p.subs = nil p.handlers = nil p.messageQueue = nil + p.messageQueueItems = nil + p.messageQueueBytes = nil + p.messageQueueItemLimits = nil + p.messageQueueLimits = nil + p.messageQueueOverflowed = nil + p.messageQueueDiscarded = nil + p.flushingMessages = false p.stalledChans = nil p.trapLinks = false p.upgradable = false diff --git a/runtime/lua/modules/cdc/module.go b/runtime/lua/modules/cdc/module.go index e5e4c660f..69cf7f4c2 100644 --- a/runtime/lua/modules/cdc/module.go +++ b/runtime/lua/modules/cdc/module.go @@ -189,7 +189,12 @@ func openStream(l *lua.LState) int { return 2 } - ch := engine.NewChannel(streamBufferCapacity(opts.Buffer)) + // CDC backlog capacity is enforced by the source/relay/process mailbox. + // Keep the Lua channel rendezvous-only so buffered Lua values cannot form a + // second queue outside that shared budget. Normalize the historical default + // before the command crosses the Lua boundary. + opts.Buffer = streamBufferCapacity(opts.Buffer) + ch := engine.NewChannel(0) engine.PushChannel(l, ch) l.Pop(1) diff --git a/service/cdc/dispatcher.go b/service/cdc/dispatcher.go index f56b12212..8462e2e3e 100644 --- a/service/cdc/dispatcher.go +++ b/service/cdc/dispatcher.go @@ -315,6 +315,14 @@ func (d *Dispatcher) execute(dispatchCtx context.Context, job dispatchJob) { } func (d *Dispatcher) executeSubscribe(dispatchCtx, requestCtx context.Context, cmd cdcapi.SubscribeCmd, tag uint64, receiver dispatcher.ResultReceiver) { + if err := cmd.Options.Validate(); err != nil { + complete(receiver, tag, nil, err) + return + } + // Keep source, relay, and process admission on one finite item limit. + // Lua normalizes this before yielding; direct Go callers get the same + // bounded default here. + cmd.Options.Buffer = cmd.Options.EffectiveMaxStreamItems() ctx, cancelContext := linkedContext(dispatchCtx, requestCtx) node := relay.GetNode(ctx) if node == nil { @@ -340,6 +348,8 @@ func (d *Dispatcher) executeSubscribe(dispatchCtx, requestCtx context.Context, c loopCtx, cancelLoop := context.WithCancel(ctx) session := &relaySession{ + maxBytes: cmd.Options.EffectiveMaxBytes(), + maxItems: cmd.Options.EffectiveMaxStreamItems(), cancel: func() { cancelLoop() cancelContext() @@ -440,14 +450,17 @@ func (d *Dispatcher) relay(ctx context.Context, session *relaySession, changes < case change, ok := <-changes: if !ok { if err := streamError(stream); err != nil { - d.sendTerminal(ctx, node, target, topic, err) + d.sendTerminal(ctx, node, target, topic, session.maxItems, session.maxBytes, err) } else { - d.sendTerminal(ctx, node, target, topic, nil) + d.sendTerminal(ctx, node, target, topic, session.maxItems, session.maxBytes, nil) } return } pkg := relay.NewPackage(pid.Zero(), target, topic, payload.New(change)) + pkg.Messages[0].PayloadBytes = cdcapi.EstimateChangeBytes(change) + pkg.Messages[0].MaxBytes = session.maxBytes + pkg.Messages[0].MaxItems = session.maxItems if err := sendRelay(ctx, node, pkg); err != nil { d.log.Debug("failed to relay cdc change", zap.String("source", source), @@ -469,13 +482,15 @@ func (d *Dispatcher) relayDone(id uint64) { d.mu.Unlock() } -func (d *Dispatcher) sendTerminal(ctx context.Context, node relay.Node, target pid.PID, topic string, err error) { +func (d *Dispatcher) sendTerminal(ctx context.Context, node relay.Node, target pid.PID, topic string, maxItems int, maxBytes int64, err error) { var terminal payload.Payloads if err != nil { terminal = append(terminal, payload.NewError(err)) } terminal = append(terminal, payload.NewTerminal()) pkg := relay.NewPackage(pid.Zero(), target, topic, terminal...) + pkg.Messages[0].MaxItems = maxItems + pkg.Messages[0].MaxBytes = maxBytes if sendErr := sendRelay(ctx, node, pkg); sendErr != nil { d.log.Debug("failed to send cdc terminal", zap.String("topic", topic), @@ -524,10 +539,12 @@ func complete(receiver dispatcher.ResultReceiver, tag uint64, data any, err erro } type relaySession struct { - cancel context.CancelFunc - close func() - id uint64 - once sync.Once + cancel context.CancelFunc + close func() + maxBytes int64 + maxItems int + id uint64 + once sync.Once } func (s *relaySession) stop() { diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 415612042..3eb23ac45 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -453,12 +453,16 @@ func (s *Scheduler) Send(pkg *relay.Package) error { // callers holding an out-of-band snapshot never deliver to a different process // that has since inherited the slot. Returns whether the push succeeded. func (s *Scheduler) deliverToProc(proc *Processor, gen uint64, pkg *relay.Package) bool { - if !proc.queue.Push(process.Event{ + admission := proc.queue.PushMessage(process.Event{ Type: process.EventMessage, Data: pkg, - }, gen) { + }, gen) + if admission == process.MessageRejected { return false } + if admission == process.MessageDropped { + relay.ReleasePackage(pkg) + } // Wake process if waiting for messages. // CAS ensures exactly-once wake even with concurrent senders. diff --git a/system/scheduler/pool/pool.go b/system/scheduler/pool/pool.go index 6f5e9386e..0ad1bdddf 100644 --- a/system/scheduler/pool/pool.go +++ b/system/scheduler/pool/pool.go @@ -121,13 +121,19 @@ func (e *Executor) CompleteYield(tag uint64, data any, err error) { // Send implements relay.Receiver. Delivers message via EventQueue. // Safe to call concurrently. Messages can be queued before Run() starts. func (e *Executor) Send(pkg *relay.Package) error { - // Push message event to queue with generation check - if !e.queue.Push(process.Event{ + // Push through the bounded message admission path. A dropped package has + // already caused one terminal event to be queued; the original pooled + // package is no longer owned by the queue and must be released here. + admission := e.queue.PushMessage(process.Event{ Type: process.EventMessage, Data: pkg, - }, e.gen.Load()) { + }, e.gen.Load()) + if admission == process.MessageRejected { return process.ErrProcessNotFound } + if admission == process.MessageDropped { + relay.ReleasePackage(pkg) + } // Signal wake select { case e.wake <- struct{}{}: From 3d3ddb32c59562fcc1906364b87e9806f19242ae Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 03:21:40 -0400 Subject: [PATCH 38/47] chore(cdc): align bounded queue state --- api/process/queue.go | 18 ++++---- runtime/lua/engine/process.go | 83 ++++++++++++++++++----------------- 2 files changed, 50 insertions(+), 51 deletions(-) diff --git a/api/process/queue.go b/api/process/queue.go index 87d758f39..410f132aa 100644 --- a/api/process/queue.go +++ b/api/process/queue.go @@ -20,10 +20,7 @@ const defaultQueueCap = 16 // Generation counter ensures stale senders from previous executions // cannot push to a reused queue. type EventQueue struct { - signal chan struct{} - events []Event - drainBuf []Event - + signal chan struct{} // Message accounting is opt-in. Ordinary event traffic keeps the // historical unbounded queue semantics; CDC messages carry MaxItems and/or // MaxBytes and are admitted through PushMessage. @@ -32,10 +29,11 @@ type EventQueue struct { messageItemLimits map[string]int messageByteLimits map[string]int64 messageOverflowed map[string]struct{} - - generation atomic.Uint64 - mu sync.Mutex - closed atomic.Bool + events []Event + drainBuf []Event + generation atomic.Uint64 + mu sync.Mutex + closed atomic.Bool } // MessageAdmission describes ownership after PushMessage. @@ -157,7 +155,7 @@ func (q *EventQueue) admitPackageLocked(pkg *relay.Package) bool { continue } - topic := string(msg.Topic) + topic := msg.Topic maxItems := msg.MaxItems if maxItems <= 0 { maxItems = q.messageItemLimits[topic] @@ -363,7 +361,7 @@ func (q *EventQueue) releaseEventLocked(event Event) { if !messageHasData(msg) { continue } - topic := string(msg.Topic) + topic := msg.Topic if msg.MaxItems > 0 && q.messageItems != nil { if n := q.messageItems[topic] - 1; n > 0 { q.messageItems[topic] = n diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index c3e23bf56..6de52d8ad 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -79,56 +79,57 @@ func WithStateOptions(opts lua.Options) ProcessOption { // Combines VM + CVM + Runner into a single unit. // Module binders and state options are stored in Factory for sharing across processes. type Process struct { - ctx context.Context - linkDownError error - execErr error - result payload.Payload - channelQueue *TaskQueue - subs *subscribeContext - mainTask *Task - upgradeRequest *UpgradeRequest - proto *lua.FunctionProto - queue *TaskQueue - factory *Factory + ctx context.Context + linkDownError error + execErr error + result payload.Payload + // Message queue limits apply only to messages that opt into bounded + // retention through relay metadata. Ordinary process messages preserve + // their historical behavior. + messageQueueLimits map[string]int64 + messageQueueItemLimits map[string]int + mainTask *Task + upgradeRequest *UpgradeRequest + proto *lua.FunctionProto + queue *TaskQueue + factory *Factory // pendingOutdated holds the single coalesced OUTDATED event awaiting // delivery to an upgradable process's events channel. Nil when none pending. - pendingOutdated *topology.OutdatedEvent - pendingYields map[uint64]*Task - channels map[*Channel]int - state *lua.LState - handlers map[string]TopicHandler - // stalledChans tracks channels that retained an undeliverable message in - // the current flush pass, keyed on the resolved *Channel. Once a channel - // stalls, every later mailbox message for it (including a terminal) is also - // retained so a terminal cannot overtake earlier retained data on the same - // channel. Lazily created on first stall, cleared at the start of each flush. - stalledChans map[*Channel]struct{} - exported map[string]*lua.LFunction - scriptName string - script string - outTasks []*Task - externalTasks []*Task - yieldBuf []*Task - messageQueue []queuedMessage + pendingOutdated *topology.OutdatedEvent + pendingYields map[uint64]*Task + channels map[*Channel]int + state *lua.LState + handlers map[string]TopicHandler + // stalledChans records channels that could not accept a message during the + // current flush. Later messages for the same channel, including terminals, + // remain queued so delivery order cannot be inverted. + stalledChans map[*Channel]struct{} + exported map[string]*lua.LFunction + messageQueueDiscarded map[string]struct{} + messageQueueOverflowed map[string]struct{} + channelQueue *TaskQueue + subs *subscribeContext // messageQueueBytes accounts only messages that opt into a byte limit via // relay metadata. Ordinary process messages keep their historical behavior. - messageQueueItems map[string]int - messageQueueBytes map[string]int64 - messageQueueItemLimits map[string]int - messageQueueLimits map[string]int64 - messageQueueOverflowed map[string]struct{} - messageQueueDiscarded map[string]struct{} - flushingMessages bool - threads []*Task - yieldSeq uint64 + messageQueueBytes map[string]int64 + messageQueueItems map[string]int + script string + scriptName string + messageQueue []queuedMessage + yieldBuf []*Task + externalTasks []*Task + outTasks []*Task + threads []*Task + yieldSeq uint64 // epoch is the monotonic incarnation counter. Incremented on every // Init / clearExecution / Close drain and on Abort. Producers stamp // every SubscriptionFrame with the epoch they were registered under; // deliverMessage compares atomically so frames from prior incarnations // are dropped without locking. - epoch atomic.Uint64 - trapLinks bool - upgradable bool + epoch atomic.Uint64 + flushingMessages bool + trapLinks bool + upgradable bool } // queuedMessage stores a message waiting to be delivered From a3edf6416d2975f06f7a2daa078b486ff5ede1e6 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 03:37:20 -0400 Subject: [PATCH 39/47] fix(lua): retain bounded CDC delivery invariants --- .../lua/engine/cdc_process_regression_test.go | 252 ++++++++++++++++++ runtime/lua/engine/process.go | 144 +++++++--- runtime/lua/engine/subscribe.go | 65 ++++- 3 files changed, 418 insertions(+), 43 deletions(-) create mode 100644 runtime/lua/engine/cdc_process_regression_test.go diff --git a/runtime/lua/engine/cdc_process_regression_test.go b/runtime/lua/engine/cdc_process_regression_test.go new file mode 100644 index 000000000..16a881fee --- /dev/null +++ b/runtime/lua/engine/cdc_process_regression_test.go @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/topology" +) + +func newCDCRegressionProcess(t *testing.T) *Process { + t.Helper() + proc := mustNewProcess(t, WithScript(`return 1`, "cdc_process_regression.lua")) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + proc.Close() + t.Fatalf("process init failed: %v", err) + } + return proc +} + +func cdcRegressionHandler(_ context.Context, _ *lua.LState, _ pid.PID, _ string, _ []payload.Payload) lua.LValue { + return lua.LTrue +} + +// A bounded relay message must wait for its exact subscription. In particular, +// a startup message cannot be consumed by the process inbox before the CDC +// subscription has finished registering. +func TestBoundedMessageDoesNotFallbackToInbox(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + inbox := NewChannel(1) + if err := proc.SubscribeExisting(topology.TopicInbox, inbox); err != nil { + t.Fatal(err) + } + const topic = "cdc.startup" + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("snapshot")}, + PayloadBytes: 8, + MaxItems: 4, + MaxBytes: 64, + }) + proc.flushMessageQueue(proc.subs) + + if got := inbox.Size(); got != 0 { + t.Fatalf("bounded startup message fell back to inbox, size=%d", got) + } + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("bounded startup message was not retained, queue=%d", got) + } + + cdc := NewChannel(1) + if err := proc.SubscribeExisting(topic, cdc); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, cdcRegressionHandler) + proc.flushMessageQueue(proc.subs) + if got := cdc.Size(); got != 1 { + t.Fatalf("exact subscription did not receive startup message, size=%d", got) + } + if got := len(proc.messageQueue); got != 0 { + t.Fatalf("startup message remained after exact subscription, queue=%d", got) + } +} + +func TestBoundedTerminalDoesNotCloseInbox(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + inbox := NewChannel(1) + if err := proc.SubscribeExisting(topology.TopicInbox, inbox); err != nil { + t.Fatal(err) + } + const topic = "cdc.startup-terminal" + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewError(errors.New("snapshot failed")), payload.NewTerminal()}, + MaxItems: 1, + MaxBytes: 64, + }) + proc.flushMessageQueue(proc.subs) + + if inbox.IsClosed() { + t.Fatal("bounded terminal closed the inbox fallback channel") + } + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("bounded terminal was not retained for exact subscription, queue=%d", got) + } + + cdc := NewChannel(1) + if err := proc.SubscribeExisting(topic, cdc); err != nil { + t.Fatal(err) + } + proc.flushMessageQueue(proc.subs) + if !cdc.IsClosed() { + t.Fatal("exact subscription did not receive terminal close") + } + if inbox.IsClosed() { + t.Fatal("inbox was closed while delivering exact terminal") + } +} + +func TestOrdinaryClosePreservesQueuedMessage(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + const topic = "ordinary.close" + old := NewChannel(1) + if err := proc.SubscribeExisting(topic, old); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, cdcRegressionHandler) + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("ordinary")}, + }) + if !proc.closeChannel(old) { + t.Fatal("closeChannel did not remove ordinary subscription") + } + if got := len(proc.messageQueue); got != 1 { + t.Fatalf("ordinary queued message was discarded on close, queue=%d", got) + } + if old.IsClosed() == false { + t.Fatal("closed ordinary channel remains open") + } + + current := NewChannel(1) + if err := proc.SubscribeExisting(topic, current); err != nil { + t.Fatal(err) + } + proc.SetTopicHandler(topic, cdcRegressionHandler) + proc.flushMessageQueue(proc.subs) + if got := current.Size(); got != 1 { + t.Fatalf("preserved ordinary message was not delivered, size=%d", got) + } +} + +func TestGoErrorWithoutTerminalConsumesBoundedCapacity(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + const topic = "cdc.error-data" + message := func() queuedMessage { + return queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewError(errors.New("row failed"))}, + MaxItems: 1, + } + } + proc.enqueueMessage(message()) + proc.enqueueMessage(message()) + + if got := len(proc.messageQueue); got != 2 { + t.Fatalf("GoError-only data bypassed bounded admission, queue=%d", got) + } + if got := proc.messageQueueItems[topic]; got != 1 { + t.Fatalf("GoError-only data was not charged as an item, items=%d", got) + } + if !isOverflowTerminal(proc.messageQueue[1].Payloads) { + t.Fatalf("second GoError-only message did not produce overflow terminal") + } +} + +func TestCleanupRegisteredAfterOverflowStillRuns(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + const topic = "cdc.late-cleanup" + ch := NewChannel(1) + if err := proc.SubscribeExisting(topic, ch); err != nil { + t.Fatal(err) + } + proc.enqueueMessage(queuedMessage{ + Topic: topic, + Payloads: payload.Payloads{payload.NewString("too large")}, + PayloadBytes: 2, + MaxBytes: 1, + }) + var calls atomic.Int32 + if !proc.SetSubscriptionCleanup(ch, func() { calls.Add(1) }) { + t.Fatal("SetSubscriptionCleanup failed") + } + if got := calls.Load(); got != 1 { + t.Fatalf("late cleanup callback count=%d, want 1", got) + } + proc.closeChannel(ch) + proc.drainSubscriptionChannels() + if got := calls.Load(); got != 1 { + t.Fatalf("cleanup callback ran more than once: %d", got) + } +} + +func TestProcessQueueBackingReferencesClearOnExecutionReset(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + oldPayloads := payload.Payloads{payload.NewString("large retained value")} + proc.enqueueMessage(queuedMessage{Topic: "unsubscribed", Payloads: oldPayloads}) + backing := proc.messageQueue + if len(backing) != 1 || backing[0].Payloads == nil { + t.Fatal("test message was not queued") + } + + proc.clearExecution() + if len(proc.messageQueue) != 0 { + t.Fatalf("clearExecution left queued messages: %d", len(proc.messageQueue)) + } + if backing[0].Payloads != nil { + t.Fatal("clearExecution left payloads reachable through queue backing array") + } +} + +type cdcTestLease struct { + calls atomic.Int32 +} + +func (l *cdcTestLease) Release() { + l.calls.Add(1) +} + +func TestLeasedMessageUsesUpstreamBudgetAndReleasesOnClear(t *testing.T) { + proc := newCDCRegressionProcess(t) + defer proc.Close() + + lease := &cdcTestLease{} + proc.enqueueMessage(queuedMessage{ + Topic: "cdc.leased", + Payloads: payload.Payloads{payload.NewString("leased")}, + MaxItems: 1, + MaxBytes: 32, + Lease: lease, + }) + if got := proc.messageQueueItems["cdc.leased"]; got != 0 { + t.Fatalf("leased message consumed a second local item budget: %d", got) + } + if got := proc.messageQueueBytes["cdc.leased"]; got != 0 { + t.Fatalf("leased message consumed a second local byte budget: %d", got) + } + proc.clearMessageQueue() + if got := lease.calls.Load(); got != 1 { + t.Fatalf("leased message release count=%d, want 1", got) + } +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 6de52d8ad..b5eba7b85 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -95,11 +95,11 @@ type Process struct { factory *Factory // pendingOutdated holds the single coalesced OUTDATED event awaiting // delivery to an upgradable process's events channel. Nil when none pending. - pendingOutdated *topology.OutdatedEvent - pendingYields map[uint64]*Task - channels map[*Channel]int - state *lua.LState - handlers map[string]TopicHandler + pendingOutdated *topology.OutdatedEvent + pendingYields map[uint64]*Task + channels map[*Channel]int + state *lua.LState + handlers map[string]TopicHandler // stalledChans records channels that could not accept a message during the // current flush. Later messages for the same channel, including terminals, // remain queued so delivery order cannot be inverted. @@ -140,6 +140,11 @@ type queuedMessage struct { MaxItems int PayloadBytes int64 MaxBytes int64 + // Lease transfers the upstream EventQueue reservation into this mailbox. + // It is released only when this queued message is delivered, discarded, or + // the process execution is reset. A leased message is already bounded by + // the upstream queue and therefore does not consume a second local budget. + Lease relay.RetentionLease } // GetProcess retrieves the Process from LState via Owner. @@ -219,12 +224,12 @@ func (p *Process) SetSubscriptionCleanup(ch *Channel, fn func()) bool { p.subs.mu.Unlock() return false } - sub.cleanup = fn overflowed := false if _, exists := p.messageQueueOverflowed[topic]; exists { overflowed = true } p.subs.mu.Unlock() + sub.setCleanup(fn) // If admission overflowed before the Lua subscription yield completed, // stop the source as soon as its cleanup hook becomes available. Do this // outside the subscription lock because cleanup may unsubscribe the same @@ -279,13 +284,15 @@ func (p *Process) closeChannel(ch *Channel) bool { sub.gen.Add(1) sub.callCleanup() } - if p.flushingMessages { - if p.messageQueueDiscarded == nil { - p.messageQueueDiscarded = make(map[string]struct{}) + if p.topicHasBoundedMessages(topic) { + if p.flushingMessages { + if p.messageQueueDiscarded == nil { + p.messageQueueDiscarded = make(map[string]struct{}) + } + p.messageQueueDiscarded[topic] = struct{}{} + } else { + p.discardMessageTopic(topic) } - p.messageQueueDiscarded[topic] = struct{}{} - } else { - p.discardMessageTopic(topic) } if !ch.IsClosed() { p.applyExternalChannelResult(ch.Close(nil)) @@ -569,15 +576,10 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload p.channelQueue.Drain() } - // Clear message queue - p.messageQueue = p.messageQueue[:0] - clear(p.messageQueueItems) - clear(p.messageQueueBytes) - clear(p.messageQueueItemLimits) - clear(p.messageQueueLimits) - clear(p.messageQueueOverflowed) - clear(p.messageQueueDiscarded) - p.flushingMessages = false + // Clear message queue and all accounting, including payload references in + // the retained backing array. Processes are pooled, so truncating the slice + // alone would keep the previous execution's data alive. + p.clearMessageQueue() p.pendingOutdated = nil // Seal the frame - no more modifications allowed after this @@ -713,6 +715,9 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // Add incoming messages to queue first (before any processing) for _, pkg := range messages { for _, msg := range pkg.Messages { + if msg == nil { + continue + } p.enqueueMessage(queuedMessage{ Source: pkg.Source, Topic: msg.Topic, @@ -720,6 +725,7 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { MaxItems: msg.MaxItems, PayloadBytes: msg.PayloadBytes, MaxBytes: msg.MaxBytes, + Lease: msg.TakeRetentionLease(), }) } relay.ReleasePackage(pkg) @@ -1111,6 +1117,7 @@ func (p *Process) flushMessageQueue(subs *subscribeContext) { } p.flushingMessages = false p.messageQueue = remaining + clear(p.messageQueue[len(remaining):]) p.finishDiscardedTopics() } @@ -1121,13 +1128,30 @@ func (p *Process) flushMessageQueue(subs *subscribeContext) { } } +// clearMessageQueue releases queued payload references before truncating the +// reusable slice. This is required on both execution reset and process-pool +// reuse; otherwise a single large message remains reachable through the +// backing array until that slice grows past its old capacity. +func (p *Process) clearMessageQueue() { + for _, qm := range p.messageQueue { + p.releaseQueuedMessage(qm) + } + clear(p.messageQueue) + p.messageQueue = p.messageQueue[:0] + clear(p.messageQueueItems) + clear(p.messageQueueBytes) + clear(p.messageQueueItemLimits) + clear(p.messageQueueLimits) + clear(p.messageQueueOverflowed) + clear(p.messageQueueDiscarded) + clear(p.stalledChans) + p.flushingMessages = false +} + // enqueueMessage is the single handoff from relay delivery into the process // mailbox. Limits are opt-in: only messages carrying MaxItems/MaxBytes are // bounded, so unrelated process topics retain their historical behavior. func (p *Process) enqueueMessage(qm queuedMessage) { - if _, discarded := p.messageQueueDiscarded[qm.Topic]; discarded { - return - } if qm.MaxItems <= 0 { qm.MaxItems = p.messageQueueItemLimits[qm.Topic] } @@ -1152,8 +1176,13 @@ func (p *Process) enqueueMessage(qm queuedMessage) { } p.messageQueueLimits[qm.Topic] = qm.MaxBytes } + if _, discarded := p.messageQueueDiscarded[qm.Topic]; discarded && messageIsBounded(qm) { + releaseMessageLease(qm) + return + } if !hasDataPayload(qm.Payloads) { if _, overflowed := p.messageQueueOverflowed[qm.Topic]; overflowed { + releaseMessageLease(qm) return } if isOverflowTerminal(qm.Payloads) { @@ -1173,10 +1202,11 @@ func (p *Process) enqueueMessage(qm queuedMessage) { } if _, overflowed := p.messageQueueOverflowed[qm.Topic]; overflowed { + releaseMessageLease(qm) return } - if qm.MaxItems > 0 || qm.MaxBytes > 0 { + if qm.Lease == nil && (qm.MaxItems > 0 || qm.MaxBytes > 0) { // A bounded producer must provide a conservative size. If it does not, // charge the whole budget rather than retaining an unaccounted value. if qm.PayloadBytes <= 0 && hasDataPayload(qm.Payloads) { @@ -1208,6 +1238,7 @@ func (p *Process) enqueueMessage(qm queuedMessage) { } func (p *Process) overflowMessageQueue(qm queuedMessage) { + releaseMessageLease(qm) if p.messageQueueOverflowed == nil { p.messageQueueOverflowed = make(map[string]struct{}) } @@ -1233,6 +1264,10 @@ func (p *Process) overflowMessageQueue(qm queuedMessage) { } func (p *Process) releaseQueuedMessage(qm queuedMessage) { + releaseMessageLease(qm) + if qm.Lease != nil { + return + } if qm.MaxItems > 0 && p.messageQueueItems != nil { remaining := p.messageQueueItems[qm.Topic] - 1 if remaining > 0 { @@ -1251,16 +1286,47 @@ func (p *Process) releaseQueuedMessage(qm queuedMessage) { } } -func (p *Process) discardMessageTopic(topic string) { - remaining := p.messageQueue[:0] +func releaseMessageLease(qm queuedMessage) { + if qm.Lease != nil { + qm.Lease.Release() + } +} + +func messageIsBounded(qm queuedMessage) bool { + return qm.Lease != nil || qm.MaxItems > 0 || qm.MaxBytes > 0 +} + +// topicHasBoundedMessages reports whether closing a subscription must leave a +// bounded-topic tombstone. The limit maps persist after delivery so a producer +// that is still racing with close is treated consistently; ordinary messages +// on the same topic remain eligible for their historical inbox behavior. +func (p *Process) topicHasBoundedMessages(topic string) bool { + if _, ok := p.messageQueueOverflowed[topic]; ok { + return true + } + if p.messageQueueItemLimits[topic] > 0 || p.messageQueueLimits[topic] > 0 { + return true + } for _, qm := range p.messageQueue { - if qm.Topic == topic { + if qm.Topic == topic && messageIsBounded(qm) { + return true + } + } + return false +} + +func (p *Process) discardMessageTopic(topic string) { + queued := p.messageQueue + remaining := queued[:0] + for _, qm := range queued { + if qm.Topic == topic && messageIsBounded(qm) { p.releaseQueuedMessage(qm) continue } remaining = append(remaining, qm) } p.messageQueue = remaining + clear(queued[len(remaining):]) delete(p.messageQueueItems, topic) delete(p.messageQueueBytes, topic) delete(p.messageQueueItemLimits, topic) @@ -1273,7 +1339,7 @@ func (p *Process) finishDiscardedTopics() { for topic := range p.messageQueueDiscarded { found := false for _, qm := range p.messageQueue { - if qm.Topic == topic { + if qm.Topic == topic && messageIsBounded(qm) { found = true break } @@ -1290,8 +1356,15 @@ func (p *Process) finishDiscardedTopics() { } func hasDataPayload(payloads payload.Payloads) bool { + hasTerminal := len(payloads) > 0 && payload.IsTerminal(payloads[len(payloads)-1]) for _, pl := range payloads { - if pl == nil || payload.IsTerminal(pl) || pl.Format() == payload.GoError { + if pl == nil || payload.IsTerminal(pl) { + continue + } + // A Go error is data unless it is part of the terminal result shape. + // Treating every GoError as control would allow an unbounded stream of + // non-terminal errors to bypass the producer budget. + if hasTerminal && pl.Format() == payload.GoError { continue } return true @@ -1341,7 +1414,7 @@ func (p *Process) isStalled(ch *Channel) bool { // stall; pure data retries normally and a full buffer preserves its order. func (p *Process) deliverMessage(subs *subscribeContext, qm queuedMessage) (keep bool) { topic := qm.Topic - if _, discarded := p.messageQueueDiscarded[topic]; discarded { + if _, discarded := p.messageQueueDiscarded[topic]; discarded && messageIsBounded(qm) { return false } handlerTopic := topic @@ -1376,6 +1449,13 @@ func (p *Process) deliverMessage(subs *subscribeContext, qm queuedMessage) (keep if hasFrame { return false } + // Bounded producer messages must wait for their exact subscription. The + // inbox is a compatibility fallback for ordinary process messages, but + // routing a bounded startup message there can consume/close the wrong + // channel before the producer's subscription is registered. + if messageIsBounded(qm) { + return true + } // Fallback to inbox for non-@ topics if !strings.HasPrefix(topic, "@") { sub, exists = subs.get(topology.TopicInbox) @@ -1810,6 +1890,7 @@ func (p *Process) Close() { p.yieldBuf = p.yieldBuf[:0] p.externalTasks = p.externalTasks[:0] p.outTasks = p.outTasks[:0] + p.clearMessageQueue() // Clear all references p.ctx = nil @@ -1950,6 +2031,7 @@ func (p *Process) clearExecution() { if p.channelQueue != nil { p.channelQueue.Drain() } + p.clearMessageQueue() // Clear yield buffer p.yieldBuf = p.yieldBuf[:0] diff --git a/runtime/lua/engine/subscribe.go b/runtime/lua/engine/subscribe.go index 4ea07b65d..d581e9e66 100644 --- a/runtime/lua/engine/subscribe.go +++ b/runtime/lua/engine/subscribe.go @@ -147,24 +147,65 @@ func (m *subscribeContext) snapshotSubscriptions() []*subscription { // subscription links a topic to a channel. type subscription struct { - cleanup func() - channel *Channel - topic string - id uint64 - gen atomic.Uint64 - cleanupOnce sync.Once + channel *Channel + topic string + id uint64 + gen atomic.Uint64 + + // Cleanup can be requested before the producer has finished registering + // its hook (for example when a bounded relay overflows during startup). + // Keep that request pending until the hook is installed instead of + // consuming a one-shot guard while cleanup is nil. + cleanupMu sync.Mutex + cleanup func() + cleanupRequested bool + cleanupDone bool } func (s *subscription) callCleanup() { if s == nil { return } - s.cleanupOnce.Do(func() { - if s.cleanup != nil { - s.cleanup() - s.cleanup = nil - } - }) + s.cleanupMu.Lock() + if s.cleanupDone { + s.cleanupMu.Unlock() + return + } + s.cleanupRequested = true + cleanup := s.cleanup + if cleanup != nil { + s.cleanup = nil + s.cleanupDone = true + } + s.cleanupMu.Unlock() + + if cleanup != nil { + cleanup() + } +} + +// setCleanup installs a producer cleanup hook. A cleanup request that arrived +// before registration is fulfilled exactly once after the hook is visible. +// The callback always runs outside cleanupMu so it may safely tear down the +// subscription or call back into the process. +func (s *subscription) setCleanup(cleanup func()) { + if s == nil || cleanup == nil { + return + } + s.cleanupMu.Lock() + if s.cleanupDone { + s.cleanupMu.Unlock() + return + } + s.cleanup = cleanup + if s.cleanupRequested { + s.cleanup = nil + s.cleanupDone = true + s.cleanupMu.Unlock() + cleanup() + return + } + s.cleanupMu.Unlock() } // SubscriptionFrame carries process-epoch, subscription-id, and generation From a01b566f0262d5734365e93cc5fba8f404785652 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 03:38:21 -0400 Subject: [PATCH 40/47] fix(cdc): transfer queue retention ownership --- api/process/queue.go | 231 +++++++++++++++++++--------- api/process/queue_retention_test.go | 156 +++++++++++++++++++ api/relay/pool.go | 7 + api/relay/relay.go | 51 ++++++ 4 files changed, 371 insertions(+), 74 deletions(-) create mode 100644 api/process/queue_retention_test.go diff --git a/api/process/queue.go b/api/process/queue.go index 410f132aa..6bd2199dd 100644 --- a/api/process/queue.go +++ b/api/process/queue.go @@ -24,16 +24,50 @@ type EventQueue struct { // Message accounting is opt-in. Ordinary event traffic keeps the // historical unbounded queue semantics; CDC messages carry MaxItems and/or // MaxBytes and are admitted through PushMessage. - messageItems map[string]int - messageBytes map[string]int64 - messageItemLimits map[string]int - messageByteLimits map[string]int64 - messageOverflowed map[string]struct{} - events []Event - drainBuf []Event - generation atomic.Uint64 - mu sync.Mutex - closed atomic.Bool + messageTopics map[string]*messageTopicState + events []Event + drainBuf []Event + generation atomic.Uint64 + mu sync.Mutex + closed atomic.Bool +} + +// messageTopicState is the accounting identity for one bounded topic +// incarnation. It must not be reused after a terminal is drained: a process +// may hold a message reservation past that point while a new stream with the +// same topic is admitted. Per-message leases retain this state until the +// consumer releases the message, so old traffic cannot debit a replacement +// stream's counters. +type messageTopicState struct { + items atomic.Int64 + bytes atomic.Int64 + maxItems int64 + maxBytes int64 + overflowed bool +} + +// messageRetentionLease transfers one EventQueue reservation to the process +// mailbox. Release is idempotent because either the process or relay's pooled +// package cleanup may be the first owner to finish the handoff. +type messageRetentionLease struct { + state *messageTopicState + items int64 + bytes int64 + once sync.Once +} + +func (l *messageRetentionLease) Release() { + if l == nil || l.state == nil { + return + } + l.once.Do(func() { + if l.items > 0 { + l.state.items.Add(-l.items) + } + if l.bytes > 0 { + l.state.bytes.Add(-l.bytes) + } + }) } // MessageAdmission describes ownership after PushMessage. @@ -157,33 +191,39 @@ func (q *EventQueue) admitPackageLocked(pkg *relay.Package) bool { topic := msg.Topic maxItems := msg.MaxItems - if maxItems <= 0 { - maxItems = q.messageItemLimits[topic] - } maxBytes := msg.MaxBytes - if maxBytes <= 0 { - maxBytes = q.messageByteLimits[topic] - } - if maxItems > 0 { - if previous := q.messageItemLimits[topic]; previous > 0 && previous < maxItems { - maxItems = previous + state := q.messageTopics[topic] + if state != nil { + if maxItems <= 0 { + maxItems = int(state.maxItems) + } else if state.maxItems > 0 && state.maxItems < int64(maxItems) { + maxItems = int(state.maxItems) } - if q.messageItemLimits == nil { - q.messageItemLimits = make(map[string]int) + if maxBytes <= 0 { + maxBytes = state.maxBytes + } else if state.maxBytes > 0 && state.maxBytes < maxBytes { + maxBytes = state.maxBytes } - q.messageItemLimits[topic] = maxItems } - if maxBytes > 0 { - if previous := q.messageByteLimits[topic]; previous > 0 && previous < maxBytes { - maxBytes = previous + if maxItems > 0 || maxBytes > 0 { + if state == nil { + if q.messageTopics == nil { + q.messageTopics = make(map[string]*messageTopicState) + } + state = &messageTopicState{} + q.messageTopics[topic] = state } - if q.messageByteLimits == nil { - q.messageByteLimits = make(map[string]int64) + if state.maxItems <= 0 || (maxItems > 0 && int64(maxItems) < state.maxItems) { + state.maxItems = int64(maxItems) } - q.messageByteLimits[topic] = maxBytes + if state.maxBytes <= 0 || (maxBytes > 0 && maxBytes < state.maxBytes) { + state.maxBytes = maxBytes + } + maxItems = int(state.maxItems) + maxBytes = state.maxBytes } - if _, overflowed := q.messageOverflowed[topic]; overflowed { + if state != nil && state.overflowed { relay.ReleaseMessage(msg) continue } @@ -206,11 +246,14 @@ func (q *EventQueue) admitPackageLocked(pkg *relay.Package) bool { // Missing size metadata must not bypass a byte budget. payloadBytes = maxBytes } - items := q.messageItems[topic] - bytes := q.messageBytes[topic] - if (maxItems > 0 && items >= maxItems) || + var items, bytes int64 + if state != nil { + items = state.items.Load() + bytes = state.bytes.Load() + } + if (maxItems > 0 && items >= int64(maxItems)) || (maxBytes > 0 && (payloadBytes > maxBytes || bytes > maxBytes-payloadBytes)) { - accepted = q.messageOverflowedLocked(topic, accepted, maxItems, maxBytes) + accepted = q.messageOverflowedLocked(state, topic, accepted, maxItems, maxBytes) relay.ReleaseMessage(msg) continue } @@ -218,17 +261,22 @@ func (q *EventQueue) admitPackageLocked(pkg *relay.Package) bool { msg.MaxItems = maxItems msg.MaxBytes = maxBytes msg.PayloadBytes = payloadBytes - if maxItems > 0 { - if q.messageItems == nil { - q.messageItems = make(map[string]int) + if state != nil { + if maxItems > 0 { + state.items.Add(1) } - q.messageItems[topic] = items + 1 - } - if maxBytes > 0 && payloadBytes > 0 { - if q.messageBytes == nil { - q.messageBytes = make(map[string]int64) + if maxBytes > 0 && payloadBytes > 0 { + state.bytes.Add(payloadBytes) + } + reservationBytes := int64(0) + if maxBytes > 0 { + reservationBytes = payloadBytes } - q.messageBytes[topic] = bytes + payloadBytes + msg.SetRetentionLease(&messageRetentionLease{ + state: state, + items: boolInt64(maxItems > 0), + bytes: reservationBytes, + }) } accepted = append(accepted, msg) } @@ -237,14 +285,25 @@ func (q *EventQueue) admitPackageLocked(pkg *relay.Package) bool { return len(accepted) > 0 } -func (q *EventQueue) messageOverflowedLocked(topic string, accepted []*relay.Message, maxItems int, maxBytes int64) []*relay.Message { - if q.messageOverflowed == nil { - q.messageOverflowed = make(map[string]struct{}) +func boolInt64(v bool) int64 { + if v { + return 1 } - if _, exists := q.messageOverflowed[topic]; exists { + return 0 +} + +func (q *EventQueue) messageOverflowedLocked(state *messageTopicState, topic string, accepted []*relay.Message, maxItems int, maxBytes int64) []*relay.Message { + if state == nil { + if q.messageTopics == nil { + q.messageTopics = make(map[string]*messageTopicState) + } + state = &messageTopicState{maxItems: int64(maxItems), maxBytes: maxBytes} + q.messageTopics[topic] = state + } + if state.overflowed { return accepted } - q.messageOverflowed[topic] = struct{}{} + state.overflowed = true msg := relay.AcquireMessage() msg.Topic = topic msg.Payloads = payload.Payloads{payload.NewError(ErrMessageQueueOverflow), payload.NewTerminal()} @@ -254,6 +313,18 @@ func (q *EventQueue) messageOverflowedLocked(topic string, accepted []*relay.Mes return append(accepted, msg) } +func messageHasTerminal(msg *relay.Message) bool { + if msg == nil { + return false + } + for _, pl := range msg.Payloads { + if pl != nil && payload.IsTerminal(pl) { + return true + } + } + return false +} + func messageHasData(msg *relay.Message) bool { if msg == nil { return false @@ -268,12 +339,25 @@ func messageHasData(msg *relay.Message) bool { } // PushDirect adds an event without generation check (for scheduler's own use). -func (q *EventQueue) PushDirect(e Event) { +// It returns false when the queue is closed. A rejected message package is +// released here because PushDirect is an ownership-taking scheduler path; a +// successful message remains owned by the queue/process as usual. +func (q *EventQueue) PushDirect(e Event) bool { q.mu.Lock() + if q.closed.Load() { + q.mu.Unlock() + if e.Type == EventMessage { + if pkg, ok := e.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } + return false + } q.events = append(q.events, e) q.mu.Unlock() q.signalPush() + return true } // Drain returns all pending events and clears the queue. @@ -281,12 +365,19 @@ func (q *EventQueue) PushDirect(e Event) { // Single consumer only (scheduler). func (q *EventQueue) Drain() []Event { q.mu.Lock() + // The previous drain result is caller-owned. The single-consumer contract + // means the caller has finished with it before asking for the next batch; + // clear it now so the queue does not retain arbitrary Data values. + for i := range q.drainBuf { + q.drainBuf[i] = Event{} + } + q.drainBuf = q.drainBuf[:0] if len(q.events) == 0 { q.mu.Unlock() return nil } for _, event := range q.events { - q.releaseEventLocked(event) + q.retireEventTopicsLocked(event) } // Swap buffers to avoid allocation @@ -314,12 +405,17 @@ func (q *EventQueue) Signal() <-chan struct{} { func (q *EventQueue) Close() { q.mu.Lock() q.closed.Store(true) - for _, event := range q.events { - q.releaseEventLocked(event) + for i, event := range q.events { + q.retireEventTopicsLocked(event) q.releaseEventPackageLocked(event) + q.events[i] = Event{} } q.events = q.events[:0] q.clearMessageAccountingLocked() + // A drained batch belongs to the scheduler. Drop the queue's reference + // without mutating the caller's slice, which may still be in flight while + // Close is called by a supervisor. + q.drainBuf = nil q.mu.Unlock() // Wake any waiters @@ -334,11 +430,15 @@ func (q *EventQueue) Reset() { q.mu.Lock() q.generation.Add(1) // Invalidate all existing senders q.closed.Store(false) - for _, event := range q.events { + for i, event := range q.events { + q.retireEventTopicsLocked(event) q.releaseEventPackageLocked(event) + q.events[i] = Event{} } q.events = q.events[:0] - q.drainBuf = q.drainBuf[:0] + // See Close: the previous Drain result is consumer-owned. Detach it + // rather than touching a potentially concurrent scheduler slice. + q.drainBuf = nil q.clearMessageAccountingLocked() q.mu.Unlock() @@ -349,7 +449,7 @@ func (q *EventQueue) Reset() { } } -func (q *EventQueue) releaseEventLocked(event Event) { +func (q *EventQueue) retireEventTopicsLocked(event Event) { if event.Type != EventMessage { return } @@ -358,24 +458,11 @@ func (q *EventQueue) releaseEventLocked(event Event) { return } for _, msg := range pkg.Messages { - if !messageHasData(msg) { + if !messageHasTerminal(msg) { continue } topic := msg.Topic - if msg.MaxItems > 0 && q.messageItems != nil { - if n := q.messageItems[topic] - 1; n > 0 { - q.messageItems[topic] = n - } else { - delete(q.messageItems, topic) - } - } - if msg.MaxBytes > 0 && msg.PayloadBytes > 0 && q.messageBytes != nil { - if n := q.messageBytes[topic] - msg.PayloadBytes; n > 0 { - q.messageBytes[topic] = n - } else { - delete(q.messageBytes, topic) - } - } + delete(q.messageTopics, topic) } } @@ -389,11 +476,7 @@ func (q *EventQueue) releaseEventPackageLocked(event Event) { } func (q *EventQueue) clearMessageAccountingLocked() { - clear(q.messageItems) - clear(q.messageBytes) - clear(q.messageItemLimits) - clear(q.messageByteLimits) - clear(q.messageOverflowed) + clear(q.messageTopics) } // YieldScheduler is the subset of Scheduler needed for waking. diff --git a/api/process/queue_retention_test.go b/api/process/queue_retention_test.go new file mode 100644 index 000000000..1599164f4 --- /dev/null +++ b/api/process/queue_retention_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MPL-2.0 + +package process + +import ( + "strconv" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/relay" +) + +type countingRetentionLease struct { + releases atomic.Int32 +} + +func (l *countingRetentionLease) Release() { + l.releases.Add(1) +} + +func TestEventQueuePushDirectClosedReleasesPackage(t *testing.T) { + q := NewEventQueue() + q.Close() + + lease := &countingRetentionLease{} + msg := relay.AcquireMessage() + msg.Topic = "closed" + msg.Payloads = payload.Payloads{payload.NewString("value")} + msg.SetRetentionLease(lease) + pkg := relay.NewMessagePackage(pid.PID{}, pid.PID{}, msg) + + require.False(t, q.PushDirect(Event{Type: EventMessage, Data: pkg})) + require.Equal(t, int32(1), lease.releases.Load(), "closed PushDirect must consume package ownership") +} + +func TestEventQueueRetentionSurvivesDrainAndTopicReuse(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + + first := boundedPackage("reuse", 1, 100, "first") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: first}, gen)) + second := boundedPackage("reuse", 1, 100, "second") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: second}, gen)) + + events := q.Drain() + require.Len(t, events, 2) + firstPkg, ok := events[0].Data.(*relay.Package) + require.True(t, ok) + firstLease := firstPkg.Messages[0].TakeRetentionLease() + require.NotNil(t, firstLease) + require.Len(t, q.messageTopics, 0, "terminal drain retires the topic state") + + // Release the terminal package but keep the first data reservation alive. + secondPkg, ok := events[1].Data.(*relay.Package) + require.True(t, ok) + relay.ReleasePackage(secondPkg) + relay.ReleasePackage(firstPkg) + + // The new stream gets a new topic state. The old lease must not debit its + // counters when it is released after the replacement has admitted data. + replacement := boundedPackage("reuse", 1, 100, "replacement") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: replacement}, q.Generation())) + overflow := boundedPackage("reuse", 1, 100, "overflow") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: overflow}, q.Generation())) + require.Len(t, overflow.Messages, 1) + require.True(t, payload.IsTerminal(overflow.Messages[0].Payloads[len(overflow.Messages[0].Payloads)-1])) + + firstLease.Release() + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} + +func TestEventQueueRetiresUniqueTopicState(t *testing.T) { + q := NewEventQueue() + for i := 0; i < 256; i++ { + topic := "churn:" + strconv.Itoa(i) + data := boundedPackage(topic, 1, 32, "data") + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: data}, q.Generation())) + terminal := relay.NewPackage(pid.PID{}, pid.PID{}, topic, payload.NewTerminal()) + terminal.Messages[0].MaxItems = 1 + terminal.Messages[0].MaxBytes = 32 + require.Equal(t, MessageAccepted, q.PushMessage(Event{Type: EventMessage, Data: terminal}, q.Generation())) + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } + require.Empty(t, q.messageTopics, "terminal topic state must not grow with churn") + } +} + +func TestEventQueueDrainClearsPreviousData(t *testing.T) { + q := NewEventQueue() + gen := q.Generation() + require.True(t, q.Push(Event{Data: &struct{ Value string }{"stale"}}, gen)) + events := q.Drain() + require.Len(t, events, 1) + require.NotNil(t, events[0].Data) + + require.Nil(t, q.Drain()) + require.Nil(t, events[0].Data, "queue must clear the reusable drain buffer before reuse") +} + +func TestEventQueueCloseReleasesQueuedPackageOnce(t *testing.T) { + q := NewEventQueue() + lease := &countingRetentionLease{} + msg := relay.AcquireMessage() + msg.Topic = "close" + msg.Payloads = payload.Payloads{payload.NewString("value")} + msg.SetRetentionLease(lease) + pkg := relay.NewMessagePackage(pid.PID{}, pid.PID{}, msg) + require.True(t, q.PushDirect(Event{Type: EventMessage, Data: pkg})) + + q.Close() + q.Close() + require.Equal(t, int32(1), lease.releases.Load(), "repeated close must not release a queued package twice") +} + +func TestEventQueueConcurrentDirectCloseOwnsEachPackage(t *testing.T) { + q := NewEventQueue() + const n = 128 + leases := make([]*countingRetentionLease, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + lease := &countingRetentionLease{} + leases[i] = lease + msg := relay.AcquireMessage() + msg.Topic = "race" + msg.Payloads = payload.Payloads{payload.NewString("value")} + msg.SetRetentionLease(lease) + pkg := relay.NewMessagePackage(pid.PID{}, pid.PID{}, msg) + wg.Add(1) + go func(pkg *relay.Package) { + defer wg.Done() + q.PushDirect(Event{Type: EventMessage, Data: pkg}) + }(pkg) + } + q.Close() + wg.Wait() + + for _, lease := range leases { + require.Equal(t, int32(1), lease.releases.Load()) + } + for _, event := range q.Drain() { + if pkg, ok := event.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} diff --git a/api/relay/pool.go b/api/relay/pool.go index 6fd333c73..af6c1f17c 100644 --- a/api/relay/pool.go +++ b/api/relay/pool.go @@ -38,6 +38,13 @@ func ReleaseMessage(m *Message) { if m == nil { return } + // A package may be released by the scheduler before its consumer takes + // ownership of a bounded-retention reservation. Consume that handoff here + // so the reservation is released exactly once and pooled messages never + // retain a callback into a retired queue. + if lease := m.TakeRetentionLease(); lease != nil { + lease.Release() + } m.Topic = "" m.Payloads = nil m.PayloadBytes = 0 diff --git a/api/relay/relay.go b/api/relay/relay.go index 615baa774..7ef8967b2 100644 --- a/api/relay/relay.go +++ b/api/relay/relay.go @@ -5,6 +5,7 @@ package relay import ( "context" + "sync/atomic" "github.com/wippyai/runtime/api/event" "github.com/wippyai/runtime/api/payload" @@ -35,6 +36,23 @@ type ( // Topic represents a message channel identifier. Topic = string + // RetentionLease represents ownership of a bounded message-retention + // reservation. The producer of a bounded message attaches a lease before + // handing the message to a queue. A consumer takes the lease when it + // transfers the message into its own mailbox and releases it when the + // message is delivered or discarded. + // + // The interface deliberately lives in relay rather than process: relay + // packages can cross scheduler and internode boundaries without depending + // on any particular consumer implementation. + RetentionLease interface { + Release() + } + + messageRetentionLease struct { + lease RetentionLease + } + // Message represents a single message with topic and payload. Message struct { Topic Topic @@ -49,6 +67,10 @@ type ( // MaxItems is the per-destination message backlog limit for this // topic. Zero means that the destination applies no item limit. MaxItems int + // retention is an atomic ownership handoff for the reservation charged + // by a bounded destination. It is intentionally not serialized: leases + // are local to a process handoff and must never cross the wire. + retention atomic.Pointer[messageRetentionLease] } // Package combines source, target and messages for delivery. @@ -66,6 +88,35 @@ type ( } ) +// SetRetentionLease attaches a bounded-retention ownership token to m. +// +// A message has at most one lease. Replacing an existing lease releases the +// old token first, which keeps pooled messages leak-free even when a caller +// accidentally reuses a message that was already admitted elsewhere. +func (m *Message) SetRetentionLease(lease RetentionLease) { + if m == nil || lease == nil { + return + } + old := m.retention.Swap(&messageRetentionLease{lease: lease}) + if old != nil && old.lease != nil { + old.lease.Release() + } +} + +// TakeRetentionLease transfers the message's reservation to its consumer. +// It is safe for a concurrent package release; exactly one caller receives +// the token and the other observes nil. +func (m *Message) TakeRetentionLease() RetentionLease { + if m == nil { + return nil + } + entry := m.retention.Swap(nil) + if entry == nil { + return nil + } + return entry.lease +} + type ( // Receiver defines the interface for message delivery. Receiver interface { From 3435eb2ac8373f247c39dd937e43b63ee25f267d Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 03:51:09 -0400 Subject: [PATCH 41/47] fix(relay): route cancellable local deliveries --- api/relay/relay.go | 16 +-- runtime/lua/engine/process.go | 10 +- runtime/lua/engine/subscribe.go | 9 +- service/cdc/dispatcher.go | 18 +++- service/host/host.go | 16 ++- service/host/host_test.go | 76 +++++++++++++++ service/terminal/host.go | 15 ++- system/relay/mailbox.go | 102 +++++++++++++++++++- system/relay/mailbox_test.go | 69 +++++++++++-- system/scheduler/actor/scheduler.go | 21 ++++ system/scheduler/actor/send_context_test.go | 76 +++++++++++++++ system/scheduler/pool/adaptive/adaptive.go | 13 ++- system/scheduler/pool/inline/inline.go | 14 ++- system/scheduler/pool/lazy/lazy.go | 14 ++- system/scheduler/pool/pool.go | 18 ++++ system/scheduler/pool/static/static.go | 13 ++- 16 files changed, 467 insertions(+), 33 deletions(-) create mode 100644 system/scheduler/actor/send_context_test.go diff --git a/api/relay/relay.go b/api/relay/relay.go index 7ef8967b2..41d489527 100644 --- a/api/relay/relay.go +++ b/api/relay/relay.go @@ -55,8 +55,12 @@ type ( // Message represents a single message with topic and payload. Message struct { - Topic Topic - Payloads payload.Payloads + // retention is an atomic ownership handoff for the reservation charged + // by a bounded destination. It is intentionally not serialized: leases + // are local to a process handoff and must never cross the wire. + retention atomic.Pointer[messageRetentionLease] + Topic Topic + Payloads payload.Payloads // PayloadBytes is the logical retained size of Payloads. It is // optional metadata used by bounded subscribers; zero preserves the // historical unbounded relay behavior. @@ -67,10 +71,6 @@ type ( // MaxItems is the per-destination message backlog limit for this // topic. Zero means that the destination applies no item limit. MaxItems int - // retention is an atomic ownership handoff for the reservation charged - // by a bounded destination. It is intentionally not serialized: leases - // are local to a process handoff and must never cross the wire. - retention atomic.Pointer[messageRetentionLease] } // Package combines source, target and messages for delivery. @@ -128,6 +128,10 @@ type ( // existing receivers keep the original Send contract; lifecycle-sensitive // dispatchers can require this capability instead of detaching a blocked // Send goroutine. + // + // Ownership is transactional: a nil error transfers the package to the + // receiver (or its accepted queue); a non-nil error means the receiver did + // not retain it and the caller must release it exactly once. ContextSender interface { SendContext(context.Context, *Package) error } diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index b5eba7b85..63a518424 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -134,17 +134,17 @@ type Process struct { // queuedMessage stores a message waiting to be delivered type queuedMessage struct { + // Lease transfers the upstream EventQueue reservation into this mailbox. + // It is released only when this queued message is delivered, discarded, or + // the process execution is reset. A leased message is already bounded by + // the upstream queue and therefore does not consume a second local budget. + Lease relay.RetentionLease Source pid.PID Topic string Payloads []payload.Payload MaxItems int PayloadBytes int64 MaxBytes int64 - // Lease transfers the upstream EventQueue reservation into this mailbox. - // It is released only when this queued message is delivered, discarded, or - // the process execution is reset. A leased message is already bounded by - // the upstream queue and therefore does not consume a second local budget. - Lease relay.RetentionLease } // GetProcess retrieves the Process from LState via Owner. diff --git a/runtime/lua/engine/subscribe.go b/runtime/lua/engine/subscribe.go index d581e9e66..b7419660a 100644 --- a/runtime/lua/engine/subscribe.go +++ b/runtime/lua/engine/subscribe.go @@ -148,16 +148,15 @@ func (m *subscribeContext) snapshotSubscriptions() []*subscription { // subscription links a topic to a channel. type subscription struct { channel *Channel - topic string - id uint64 - gen atomic.Uint64 - // Cleanup can be requested before the producer has finished registering // its hook (for example when a bounded relay overflows during startup). // Keep that request pending until the hook is installed instead of // consuming a one-shot guard while cleanup is nil. - cleanupMu sync.Mutex cleanup func() + topic string + id uint64 + gen atomic.Uint64 + cleanupMu sync.Mutex cleanupRequested bool cleanupDone bool } diff --git a/service/cdc/dispatcher.go b/service/cdc/dispatcher.go index 8462e2e3e..8b4f548e5 100644 --- a/service/cdc/dispatcher.go +++ b/service/cdc/dispatcher.go @@ -503,17 +503,33 @@ func (d *Dispatcher) sendTerminal(ctx context.Context, node relay.Node, target p // goroutine retained the stream, node, and process context indefinitely; a // node without the capability therefore fails the delivery explicitly. func sendRelay(ctx context.Context, node relay.Node, pkg *relay.Package) error { + if pkg == nil { + return ErrNoRelayNode + } if ctx == nil { ctx = context.Background() } if err := ctx.Err(); err != nil { + relay.ReleasePackage(pkg) return err } + if node == nil { + relay.ReleasePackage(pkg) + return ErrNoRelayNode + } sender, ok := node.(relay.ContextSender) if !ok { + relay.ReleasePackage(pkg) return ErrRelayNotCancellable } - return sender.SendContext(ctx, pkg) + if err := sender.SendContext(ctx, pkg); err != nil { + // ContextSender ownership is transactional: a nil result transfers + // ownership to the destination queue; an error leaves it with the + // caller. CDC is the caller at this boundary, so release exactly once. + relay.ReleasePackage(pkg) + return err + } + return nil } // streamError is an optional extension implemented by streams that can diff --git a/service/host/host.go b/service/host/host.go index cc74171f4..89efe5aaf 100644 --- a/service/host/host.go +++ b/service/host/host.go @@ -163,6 +163,7 @@ func processName(start *process.Start) string { func (h *Host) sendMessages(target pid.PID, messages []*relay.Message) { pkg := relay.NewMessagePackage(pid.PID{}, target, messages...) if err := h.scheduler.Send(pkg); err != nil { + relay.ReleasePackage(pkg) h.log.Warn("failed to send messages", zap.String("target", target.String()), zap.Error(err)) @@ -177,10 +178,23 @@ func (h *Host) Terminate(_ context.Context, processID pid.PID) error { // Send implements relay.Receiver. func (h *Host) Send(pkg *relay.Package) error { + return h.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. The actor scheduler admits +// messages without a blocking goroutine, so cancellation can stop a relay +// directly at the host boundary. +func (h *Host) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } if h.shutdown.Load() { return ErrHostShuttingDown } - return h.scheduler.Send(pkg) + return h.scheduler.SendContext(ctx, pkg) } // Start implements supervisor.Service. diff --git a/service/host/host_test.go b/service/host/host_test.go index 4b11cad93..5a930ce92 100644 --- a/service/host/host_test.go +++ b/service/host/host_test.go @@ -24,6 +24,7 @@ import ( hostapi "github.com/wippyai/runtime/api/service/host" "github.com/wippyai/runtime/api/topology" "github.com/wippyai/runtime/internal/uniqid" + relaysys "github.com/wippyai/runtime/system/relay" "github.com/wippyai/runtime/system/scheduler/actor" securitysys "github.com/wippyai/runtime/system/security" "go.uber.org/zap" @@ -692,6 +693,81 @@ func TestHost_SendMessagesEmpty(t *testing.T) { th.host.sendMessages(target, nil) } +type contextMessageProcess struct { + ready chan struct{} + received chan struct{} + readyOnce sync.Once + recvOnce sync.Once +} + +func (p *contextMessageProcess) Init(context.Context, string, payload.Payloads) error { return nil } + +func (p *contextMessageProcess) Step(events []process.Event, out *process.StepOutput) error { + p.readyOnce.Do(func() { close(p.ready) }) + for _, event := range events { + if event.Type != process.EventMessage { + continue + } + pkg, ok := event.Data.(*relay.Package) + if !ok { + continue + } + relay.ReleasePackage(pkg) + p.recvOnce.Do(func() { close(p.received) }) + out.Done(nil) + return nil + } + out.Idle() + return nil +} + +func (p *contextMessageProcess) Close() {} + +func TestHostSendContextThroughLocalNode(t *testing.T) { + th := newTestHost() + th.start(t) + defer th.stop() + + processID := pid.PID{Node: "test-node", Host: "test:host", UniqID: "context-send"} + processID = processID.Precomputed() + proc := &contextMessageProcess{ready: make(chan struct{}), received: make(chan struct{})} + _, err := th.scheduler.Submit(context.Background(), processID, proc, "", nil) + require.NoError(t, err) + select { + case <-proc.ready: + case <-time.After(time.Second): + t.Fatal("process did not reach idle state") + } + + node := relaysys.NewNode("test-node") + require.NoError(t, node.RegisterHost("test:host", th.host)) + pkg := relay.NewPackage(pid.PID{}, processID, "context", payload.New("value")) + require.NoError(t, node.SendContext(context.Background(), pkg)) + select { + case <-proc.received: + case <-time.After(time.Second): + t.Fatal("local host did not receive cancellable delivery") + } +} + +func TestHostSendContextRejectsUnknownAndCanceledDelivery(t *testing.T) { + h := newTestHost() + h.start(t) + defer h.stop() + + unknown := relay.NewPackage(pid.PID{}, pid.PID{Host: "test:host", UniqID: "missing"}, "context", payload.New("value")) + err := h.host.SendContext(context.Background(), unknown) + require.ErrorIs(t, err, process.ErrProcessNotFound) + relay.ReleasePackage(unknown) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + canceled := relay.NewPackage(pid.PID{}, pid.PID{Host: "test:host", UniqID: "missing"}, "context", payload.New("value")) + err = h.host.SendContext(ctx, canceled) + require.ErrorIs(t, err, context.Canceled) + relay.ReleasePackage(canceled) +} + // --- Concurrent Operation Tests --- func TestHost_ConcurrentRun(t *testing.T) { diff --git a/service/terminal/host.go b/service/terminal/host.go index 534ae26f8..949f10807 100644 --- a/service/terminal/host.go +++ b/service/terminal/host.go @@ -239,10 +239,23 @@ func (h *Host) Terminate(_ context.Context, processID pid.PID) error { // Send implements relay.Receiver. func (h *Host) Send(pkg *relay.Package) error { + return h.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender through the actor scheduler. +// Admission is non-blocking, so cancellation never requires a detached +// delivery goroutine. +func (h *Host) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } if h.shutdown.Load() { return ErrHostShuttingDown } - return h.scheduler.Send(pkg) + return h.scheduler.SendContext(ctx, pkg) } // Start implements supervisor.Service. diff --git a/system/relay/mailbox.go b/system/relay/mailbox.go index 94d44e5a8..ce458a587 100644 --- a/system/relay/mailbox.go +++ b/system/relay/mailbox.go @@ -49,6 +49,9 @@ type Mailbox struct { ctx context.Context receivers sync.Map jobQueues []chan *api.Package + lifecycle sync.RWMutex + delivery sync.RWMutex + closed bool } // NewMailbox creates a new Mailbox instance with the provided options. @@ -103,6 +106,8 @@ func hashString(s string) uint32 { // Attach attaches a receiver channel for Package messages. // Only one receiver may be attached per PID; if one already exists, an error is returned. func (m *Mailbox) Attach(p pid.PID, ch chan *api.Package) (context.CancelFunc, error) { + m.lifecycle.RLock() + defer m.lifecycle.RUnlock() key := p.String() _, loaded := m.receivers.LoadOrStore(key, ch) if loaded { @@ -113,13 +118,19 @@ func (m *Mailbox) Attach(p pid.PID, ch chan *api.Package) (context.CancelFunc, e return nil, NewAlreadyAttachedError(p) } - return func() { m.receivers.Delete(key) }, nil + return func() { m.Detach(p) }, nil } // Detach removes a receiver channel from a pid. func (m *Mailbox) Detach(p pid.PID) { key := p.String() - m.receivers.Delete(key) + m.lifecycle.Lock() + if rec, ok := m.receivers.LoadAndDelete(key); ok { + if ch, ok := rec.(chan *api.Package); ok { + drainReceiver(ch) + } + } + m.lifecycle.Unlock() m.config.logger.Debug("receiver detached", zap.String("pid", key)) } @@ -144,7 +155,6 @@ func (m *Mailbox) SendContext(ctx context.Context, pkg *api.Package) error { return err } - // Check context before attempting to send to avoid sending to closed channels if err := m.ctx.Err(); err != nil { m.config.logger.Warn("send after mailbox shutdown", zap.String("pid", pkg.Target.String())) return err @@ -153,12 +163,24 @@ func (m *Mailbox) SendContext(ctx context.Context, pkg *api.Package) error { // Hash by Source.UniqID to preserve per-sender ordering workerIndex := int(hashString(pkg.Source.UniqID)) % m.config.workerCount + // The lifecycle read lock linearizes queue admission with worker shutdown. + // A worker drains queued packages only after acquiring the write lock, so a + // successful send always has a live owner and a rejected send remains owned + // by its caller. + m.lifecycle.RLock() + if m.closed { + m.lifecycle.RUnlock() + return m.ctx.Err() + } select { case m.jobQueues[workerIndex] <- pkg: + m.lifecycle.RUnlock() return nil case <-ctx.Done(): + m.lifecycle.RUnlock() return ctx.Err() case <-m.ctx.Done(): + m.lifecycle.RUnlock() m.config.logger.Warn("send after mailbox shutdown", zap.String("pid", pkg.Target.String())) return m.ctx.Err() } @@ -175,13 +197,54 @@ func (m *Mailbox) worker(queueIndex int) { case pkg := <-queue: m.deliver(pkg) case <-m.ctx.Done(): + m.shutdown() return } } } +// shutdown closes admission and releases packages still waiting in every +// worker queue. The queues remain open because SendContext may be racing the +// context cancellation; the lifecycle lock makes that race deterministic. +func (m *Mailbox) shutdown() { + m.lifecycle.Lock() + if m.closed { + m.lifecycle.Unlock() + return + } + m.closed = true + for _, queue := range m.jobQueues { + drain: + for { + select { + case pkg := <-queue: + api.ReleasePackage(pkg) + default: + break drain + } + } + } + m.lifecycle.Unlock() + + // Wait for in-flight deliveries to observe the canceled mailbox context + // before draining attached channels. This lock is separate from lifecycle: + // Detach must remain able to remove a blocked receiver without waiting for a + // potentially slow consumer. + m.delivery.Lock() + m.receivers.Range(func(_, value any) bool { + if ch, ok := value.(chan *api.Package); ok { + drainReceiver(ch) + } + return true + }) + m.delivery.Unlock() +} + // deliver sends the package to the target's receiver channel. func (m *Mailbox) deliver(pkg *api.Package) { + if pkg == nil { + return + } targetKey := pkg.Target.String() rec, ok := m.receivers.Load(targetKey) if !ok { @@ -193,6 +256,7 @@ func (m *Mailbox) deliver(pkg *api.Package) { zap.String("target", targetKey), zap.String("source", pkg.Source.String()), zap.String("topic", topic)) + api.ReleasePackage(pkg) return } @@ -200,6 +264,7 @@ func (m *Mailbox) deliver(pkg *api.Package) { if !ok { m.config.logger.Error("receiver has invalid type", zap.String("target", targetKey)) + api.ReleasePackage(pkg) return } @@ -212,17 +277,48 @@ func (m *Mailbox) deliver(pkg *api.Package) { // take down the worker, so it is recovered: a closed receiver means the process // is gone and the package is dropped. func (m *Mailbox) deliverTo(ch chan *api.Package, pkg *api.Package, targetKey string) { + m.delivery.RLock() + defer m.delivery.RUnlock() + + delivered := false defer func() { if r := recover(); r != nil { m.config.logger.Debug("dropped delivery to closed receiver", zap.String("target", targetKey)) } + if !delivered { + api.ReleasePackage(pkg) + } }() + if err := m.ctx.Err(); err != nil { + m.config.logger.Debug("delivery canceled", + zap.String("target", targetKey), zap.Error(err)) + return + } + select { case ch <- pkg: + delivered = true case <-m.ctx.Done(): m.config.logger.Debug("delivery canceled", zap.String("target", targetKey)) } } + +// drainReceiver releases packages that were already accepted by an attached +// channel after its owner detaches. Detach serializes with deliverTo, so no +// package can become unreachable between the final send and this drain. +func drainReceiver(ch chan *api.Package) { + for { + select { + case pkg, ok := <-ch: + if !ok { + return + } + api.ReleasePackage(pkg) + default: + return + } + } +} diff --git a/system/relay/mailbox_test.go b/system/relay/mailbox_test.go index ae2a80dfc..4ede8d865 100644 --- a/system/relay/mailbox_test.go +++ b/system/relay/mailbox_test.go @@ -5,6 +5,7 @@ package relay import ( "context" "fmt" + "sync/atomic" "testing" "time" @@ -15,6 +16,18 @@ import ( "go.uber.org/zap" ) +type mailboxReleaseProbe struct{ releases atomic.Int32 } + +func (p *mailboxReleaseProbe) Release() { p.releases.Add(1) } + +func probedPackage(target pidapi.PID) (*relay.Package, *mailboxReleaseProbe) { + probe := &mailboxReleaseProbe{} + msg := relay.AcquireMessage() + msg.Topic = "probe" + msg.SetRetentionLease(probe) + return relay.NewMessagePackage(pidapi.PID{}, target, msg), probe +} + func TestMailbox_NewMailbox(t *testing.T) { ctx := context.Background() logger := zap.NewNop() @@ -145,7 +158,8 @@ func TestMailbox_SendContextHonorsCallerCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - err := mailbox.SendContext(ctx, pkg) + blocked := &relay.Package{Target: target} + err := mailbox.SendContext(ctx, blocked) assert.ErrorIs(t, err, context.Canceled) } @@ -165,14 +179,49 @@ func TestMailbox_NoReceiver(t *testing.T) { } // send message without attaching a receiver - pkg := &relay.Package{ - Target: pid, - Messages: []*relay.Message{ - {Topic: "test"}, - }, - } + pkg, probe := probedPackage(pid) err := mailbox.Send(pkg) assert.NoError(t, err) // send should succeed even without receiver + assert.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond, + "accepted package without a receiver must be released by the mailbox") +} + +func TestMailbox_DetachReleasesBufferedPackages(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(4), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "detach"} + receiverCh := make(chan *relay.Package, 4) + _, err := mailbox.Attach(target, receiverCh) + require.NoError(t, err) + + pkg, probe := probedPackage(target) + require.NoError(t, mailbox.Send(pkg)) + mailbox.Detach(target) + + assert.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond, + "detaching a receiver must release packages already accepted by its channel") +} + +func TestMailbox_ShutdownReleasesQueuedPackages(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + mailbox := NewMailbox(ctx, WithBufferSize(8), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "shutdown"} + // Keep the worker from handing packages to a receiver; shutdown must own + // and release packages remaining in its internal queue. + packages := make([]*relay.Package, 8) + probes := make([]*mailboxReleaseProbe, 8) + for i := range packages { + packages[i], probes[i] = probedPackage(target) + require.NoError(t, mailbox.Send(packages[i])) + } + cancel() + assert.Eventually(t, func() bool { + for _, probe := range probes { + if probe.releases.Load() != 1 { + return false + } + } + return true + }, time.Second, time.Millisecond, "mailbox shutdown leaked queued packages") } func TestMailbox_DetachDuringDelivery(t *testing.T) { @@ -334,6 +383,12 @@ func TestMailbox_Shutdown(t *testing.T) { } err = mailbox.Send(pkg) assert.NoError(t, err) + select { + case delivered := <-receiverCh: + relay.ReleasePackage(delivered) + case <-time.After(time.Second): + t.Fatal("timeout waiting for pre-shutdown delivery") + } // Now cancel the mailbox context cancel() diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 3eb23ac45..df18b051a 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -12,6 +12,7 @@ import ( "github.com/wippyai/runtime/api/attrs" "github.com/wippyai/runtime/api/dispatcher" + apierror "github.com/wippyai/runtime/api/error" "github.com/wippyai/runtime/api/payload" "github.com/wippyai/runtime/api/pid" "github.com/wippyai/runtime/api/process" @@ -24,6 +25,8 @@ import ( type Option func(*Scheduler) +var errNilPackage = apierror.New(apierror.Invalid, "cannot send nil package").WithRetryable(apierror.False) + func WithWorkers(n int) Option { return func(s *Scheduler) { if n > 0 { @@ -431,6 +434,24 @@ func (s *Scheduler) ReleaseProcessor(proc *Processor) { // Send implements relay.Receiver. Routes package to target process. // Wakes the process if it's idle or blocked waiting for messages. func (s *Scheduler) Send(pkg *relay.Package) error { + return s.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. Admission into a process queue +// is non-blocking, so cancellation is checked before the target lookup and +// before admission. Once PushMessage accepts a package, ownership transfers +// to the process queue and a later cancellation cannot undo that transfer. +func (s *Scheduler) SendContext(ctx context.Context, pkg *relay.Package) error { + if pkg == nil { + return errNilPackage + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + target := pkg.Target // copy before push - pkg may be released after queue receives it v, ok := s.byPID.Load(target.String()) diff --git a/system/scheduler/actor/send_context_test.go b/system/scheduler/actor/send_context_test.go new file mode 100644 index 000000000..e6389b165 --- /dev/null +++ b/system/scheduler/actor/send_context_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" +) + +func TestSchedulerSendContextUnknownTargetKeepsCallerOwnership(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + pkg := relay.NewPackage(pid.PID{}, pid.PID{UniqID: "missing"}, "test") + + err := s.SendContext(context.Background(), pkg) + require.ErrorIs(t, err, process.ErrProcessNotFound) + // A failed admission leaves the package with the caller, so it is safe to + // inspect/release it here rather than relying on a detached sender. + require.Len(t, pkg.Messages, 1) + relay.ReleasePackage(pkg) +} + +func TestSchedulerSendContextCanceledBeforeAdmission(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + pkg := relay.NewPackage(pid.PID{}, pid.PID{UniqID: "missing"}, "test") + + err := s.SendContext(ctx, pkg) + require.ErrorIs(t, err, context.Canceled) + require.Len(t, pkg.Messages, 1) + relay.ReleasePackage(pkg) +} + +func TestSchedulerSendRejectsStaleGeneration(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + target := pid.PID{UniqID: "stale-generation"} + proc, err := s.Submit(context.Background(), target, &IdleProcess{}, "", nil) + require.NoError(t, err) + + oldGeneration := proc.gen.Load() + proc.queue.Reset() + pkg := relay.NewPackage(pid.PID{}, target, "test") + require.False(t, s.deliverToProc(proc, oldGeneration, pkg)) + // The stale sender never transfers ownership. + require.Len(t, pkg.Messages, 1) + relay.ReleasePackage(pkg) + + s.completeNoPool(proc, nil, context.Canceled) + _, ok := s.byPID.Load(target.String()) + require.False(t, ok) +} + +func TestSchedulerSendContextAcceptedQueueOwnsPackage(t *testing.T) { + s := NewScheduler(nil, WithWorkers(1)) + target := pid.PID{UniqID: "accepted"} + proc, err := s.Submit(context.Background(), target, &IdleProcess{}, "", nil) + require.NoError(t, err) + + pkg := relay.NewPackage(pid.PID{}, target, "test") + require.NoError(t, s.SendContext(context.Background(), pkg)) + // Queue admission transfers ownership. Closing the queue must release the + // package exactly once and must not require the sender to release it. + proc.queue.Close() + require.Empty(t, pkg.Messages) + + s.completeNoPool(proc, nil, context.Canceled) + if err := s.SendContext(context.Background(), pkg); !errors.Is(err, process.ErrProcessNotFound) { + t.Fatalf("expected completed process to be absent, got %v", err) + } +} diff --git a/system/scheduler/pool/adaptive/adaptive.go b/system/scheduler/pool/adaptive/adaptive.go index 123958155..00b7ea80c 100644 --- a/system/scheduler/pool/adaptive/adaptive.go +++ b/system/scheduler/pool/adaptive/adaptive.go @@ -222,11 +222,22 @@ func (a *Pool) QueueLen() int { // Send implements relay.Receiver for message routing. func (a *Pool) Send(pkg *relay.Package) error { + return a.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender for message routing. +func (a *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := a.active.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // Call executes a function call using an available worker. diff --git a/system/scheduler/pool/inline/inline.go b/system/scheduler/pool/inline/inline.go index 5a59324c8..d6438de19 100644 --- a/system/scheduler/pool/inline/inline.go +++ b/system/scheduler/pool/inline/inline.go @@ -98,11 +98,23 @@ func (i *Pool) replaceProcess() error { // Send implements relay.Receiver. Routes package to target execution. func (i *Pool) Send(pkg *relay.Package) error { + return i.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. Routes package to target +// execution while honoring cancellation before queue admission. +func (i *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := i.active.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // Start is a no-op for inline execution. diff --git a/system/scheduler/pool/lazy/lazy.go b/system/scheduler/pool/lazy/lazy.go index 7ea5db001..e67de5cb8 100644 --- a/system/scheduler/pool/lazy/lazy.go +++ b/system/scheduler/pool/lazy/lazy.go @@ -146,11 +146,23 @@ func (l *Pool) Call(ctx context.Context, method string, input payload.Payloads) // Send implements relay.Receiver. Routes package to target execution. func (l *Pool) Send(pkg *relay.Package) error { + return l.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. Routes package to target +// execution while honoring cancellation before queue admission. +func (l *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := l.activeExec.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // acquire gets an idle process or creates a new one. diff --git a/system/scheduler/pool/pool.go b/system/scheduler/pool/pool.go index 0ad1bdddf..e69ef1271 100644 --- a/system/scheduler/pool/pool.go +++ b/system/scheduler/pool/pool.go @@ -121,6 +121,24 @@ func (e *Executor) CompleteYield(tag uint64, data any, err error) { // Send implements relay.Receiver. Delivers message via EventQueue. // Safe to call concurrently. Messages can be queued before Run() starts. func (e *Executor) Send(pkg *relay.Package) error { + return e.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender. EventQueue admission is +// non-blocking, so cancellation is checked before admission. Once the queue +// accepts a package it owns it; cancellation after that point cannot revoke +// ownership or leave a detached sender behind. +func (e *Executor) SendContext(ctx context.Context, pkg *relay.Package) error { + if pkg == nil { + return process.ErrProcessNotFound + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + // Push through the bounded message admission path. A dropped package has // already caused one terminal event to be queued; the original pooled // package is no longer owned by the queue and must be released here. diff --git a/system/scheduler/pool/static/static.go b/system/scheduler/pool/static/static.go index cfdfaade3..3f1db6e14 100644 --- a/system/scheduler/pool/static/static.go +++ b/system/scheduler/pool/static/static.go @@ -138,11 +138,22 @@ func (s *Pool) Stop() { // Send implements relay.Receiver for message routing. func (s *Pool) Send(pkg *relay.Package) error { + return s.SendContext(context.Background(), pkg) +} + +// SendContext implements relay.ContextSender for message routing. +func (s *Pool) SendContext(ctx context.Context, pkg *relay.Package) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } v, ok := s.active.Load(pkg.Target.UniqID) if !ok { return process.ErrProcessNotFound } - return v.(*pool.Executor).Send(pkg) + return v.(*pool.Executor).SendContext(ctx, pkg) } // Call executes a function call using an available worker. From c3e085ed5c15017057d02fd08e562caa42104e87 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 03:57:09 -0400 Subject: [PATCH 42/47] fix(lua): preserve completed mailbox visibility --- runtime/lua/engine/process.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 63a518424..70490dbcf 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -2031,7 +2031,9 @@ func (p *Process) clearExecution() { if p.channelQueue != nil { p.channelQueue.Drain() } - p.clearMessageQueue() + // Keep undelivered ordinary messages observable until the scheduler retires + // or reinitializes this execution. Init and Close clear the backing storage, + // including bounded retention leases, before a pooled process can be reused. // Clear yield buffer p.yieldBuf = p.yieldBuf[:0] From e977a24beee80892fb1b36283ee025773843879e Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 03:59:58 -0400 Subject: [PATCH 43/47] test(lua): preserve mailbox semantics across completion --- runtime/lua/engine/cdc_process_regression_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/runtime/lua/engine/cdc_process_regression_test.go b/runtime/lua/engine/cdc_process_regression_test.go index 16a881fee..f8df23cd6 100644 --- a/runtime/lua/engine/cdc_process_regression_test.go +++ b/runtime/lua/engine/cdc_process_regression_test.go @@ -199,7 +199,7 @@ func TestCleanupRegisteredAfterOverflowStillRuns(t *testing.T) { } } -func TestProcessQueueBackingReferencesClearOnExecutionReset(t *testing.T) { +func TestProcessQueueBackingReferencesClearBeforeProcessReuse(t *testing.T) { proc := newCDCRegressionProcess(t) defer proc.Close() @@ -211,11 +211,19 @@ func TestProcessQueueBackingReferencesClearOnExecutionReset(t *testing.T) { } proc.clearExecution() + if len(proc.messageQueue) != 1 { + t.Fatalf("completed execution lost observable queued messages: %d", len(proc.messageQueue)) + } + + nextCtx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(nextCtx, "", nil); err != nil { + t.Fatalf("process reinit failed: %v", err) + } if len(proc.messageQueue) != 0 { - t.Fatalf("clearExecution left queued messages: %d", len(proc.messageQueue)) + t.Fatalf("process reuse left queued messages: %d", len(proc.messageQueue)) } if backing[0].Payloads != nil { - t.Fatal("clearExecution left payloads reachable through queue backing array") + t.Fatal("process reuse left payloads reachable through queue backing array") } } From 32f36d6c0b86d6fee05c1758c105ac019e2e9d7e Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 04:16:25 -0400 Subject: [PATCH 44/47] fix(relay): linearize mailbox receiver generations --- system/relay/mailbox.go | 202 +++++++++++++++++++++++++---------- system/relay/mailbox_test.go | 120 ++++++++++++++++++++- 2 files changed, 258 insertions(+), 64 deletions(-) diff --git a/system/relay/mailbox.go b/system/relay/mailbox.go index ce458a587..ec67589fe 100644 --- a/system/relay/mailbox.go +++ b/system/relay/mailbox.go @@ -45,13 +45,59 @@ func WithLogger(logger *zap.Logger) MailboxOption { // Mailbox implements a local message relay with asynchronous delivery. // It routes packages to attached receivers via worker goroutines. type Mailbox struct { - config mailboxConfig - ctx context.Context - receivers sync.Map - jobQueues []chan *api.Package - lifecycle sync.RWMutex - delivery sync.RWMutex - closed bool + config mailboxConfig + ctx context.Context + receivers sync.Map + jobQueues []chan mailboxJob + lifecycle sync.RWMutex + admissions sync.WaitGroup + closed bool +} + +type mailboxJob struct { + pkg *api.Package + receiver *mailboxReceiver +} + +// mailboxReceiver is one attachment incarnation. Deliveries hold an active +// reference while they are sending to the channel; Detach marks the +// incarnation closed, waits for those sends to finish, then drains anything +// they accepted. This makes a buffered channel safe to detach and reattach +// without letting an old delivery strand a package in the old channel. +type mailboxReceiver struct { + ch chan *api.Package + done chan struct{} + active sync.WaitGroup + mu sync.Mutex + detached bool +} + +func newMailboxReceiver(ch chan *api.Package) *mailboxReceiver { + return &mailboxReceiver{ch: ch, done: make(chan struct{})} +} + +func (r *mailboxReceiver) begin() bool { + r.mu.Lock() + defer r.mu.Unlock() + if r.detached { + return false + } + r.active.Add(1) + return true +} + +func (r *mailboxReceiver) end() { + r.active.Done() +} + +func (r *mailboxReceiver) stop() { + r.mu.Lock() + if !r.detached { + r.detached = true + close(r.done) + } + r.mu.Unlock() + r.active.Wait() } // NewMailbox creates a new Mailbox instance with the provided options. @@ -74,9 +120,9 @@ func NewMailbox(ctx context.Context, opts ...MailboxOption) *Mailbox { config.workerCount = 1 } - jobQueues := make([]chan *api.Package, config.workerCount) + jobQueues := make([]chan mailboxJob, config.workerCount) for i := 0; i < config.workerCount; i++ { - jobQueues[i] = make(chan *api.Package, config.bufferSize) + jobQueues[i] = make(chan mailboxJob, config.bufferSize) } m := &Mailbox{ @@ -108,8 +154,12 @@ func hashString(s string) uint32 { func (m *Mailbox) Attach(p pid.PID, ch chan *api.Package) (context.CancelFunc, error) { m.lifecycle.RLock() defer m.lifecycle.RUnlock() + if m.closed { + return nil, m.ctx.Err() + } key := p.String() - _, loaded := m.receivers.LoadOrStore(key, ch) + receiver := newMailboxReceiver(ch) + _, loaded := m.receivers.LoadOrStore(key, receiver) if loaded { m.config.logger.Warn("attempt to attach an already existing package receiver", zap.String("pid", key), @@ -118,20 +168,35 @@ func (m *Mailbox) Attach(p pid.PID, ch chan *api.Package) (context.CancelFunc, e return nil, NewAlreadyAttachedError(p) } - return func() { m.Detach(p) }, nil + return func() { m.detach(key, receiver) }, nil } // Detach removes a receiver channel from a pid. func (m *Mailbox) Detach(p pid.PID) { key := p.String() + m.detach(key, nil) + m.config.logger.Debug("receiver detached", zap.String("pid", key)) +} + +// detach removes one receiver incarnation and waits for all deliveries that +// loaded it before the removal. expected is used by Attach's cancellation +// callback so an old callback cannot detach a newer reattachment. +func (m *Mailbox) detach(key string, expected *mailboxReceiver) { m.lifecycle.Lock() - if rec, ok := m.receivers.LoadAndDelete(key); ok { - if ch, ok := rec.(chan *api.Package); ok { - drainReceiver(ch) + var receiver *mailboxReceiver + if value, ok := m.receivers.Load(key); ok { + current, valid := value.(*mailboxReceiver) + if valid && (expected == nil || current == expected) { + m.receivers.Delete(key) + receiver = current } } m.lifecycle.Unlock() - m.config.logger.Debug("receiver detached", zap.String("pid", key)) + if receiver == nil { + return + } + receiver.stop() + drainReceiver(receiver.ch) } // Send enqueues a package for delivery. Messages from the same source @@ -163,24 +228,37 @@ func (m *Mailbox) SendContext(ctx context.Context, pkg *api.Package) error { // Hash by Source.UniqID to preserve per-sender ordering workerIndex := int(hashString(pkg.Source.UniqID)) % m.config.workerCount - // The lifecycle read lock linearizes queue admission with worker shutdown. - // A worker drains queued packages only after acquiring the write lock, so a - // successful send always has a live owner and a rejected send remains owned - // by its caller. + // The lifecycle read lock linearizes the receiver snapshot and admission + // with worker shutdown. It is released before the queue select: a full queue + // must not prevent Detach or shutdown from canceling the sender. m.lifecycle.RLock() if m.closed { m.lifecycle.RUnlock() return m.ctx.Err() } + m.admissions.Add(1) + targetKey := pkg.Target.String() + + // Capture the attachment incarnation under the same lock as admission. A + // worker must never look up the target again after Detach/reattach, or a + // package accepted for an old channel could be delivered to a new one. + var receiver *mailboxReceiver + if value, ok := m.receivers.Load(targetKey); ok { + receiver, _ = value.(*mailboxReceiver) + } + job := mailboxJob{pkg: pkg, receiver: receiver} + defer func() { + m.lifecycle.Lock() + m.admissions.Done() + m.lifecycle.Unlock() + }() + m.lifecycle.RUnlock() select { - case m.jobQueues[workerIndex] <- pkg: - m.lifecycle.RUnlock() + case m.jobQueues[workerIndex] <- job: return nil case <-ctx.Done(): - m.lifecycle.RUnlock() return ctx.Err() case <-m.ctx.Done(): - m.lifecycle.RUnlock() m.config.logger.Warn("send after mailbox shutdown", zap.String("pid", pkg.Target.String())) return m.ctx.Err() } @@ -194,8 +272,8 @@ func (m *Mailbox) worker(queueIndex int) { for { select { - case pkg := <-queue: - m.deliver(pkg) + case job := <-queue: + m.deliver(job) case <-m.ctx.Done(): m.shutdown() return @@ -205,7 +283,7 @@ func (m *Mailbox) worker(queueIndex int) { // shutdown closes admission and releases packages still waiting in every // worker queue. The queues remain open because SendContext may be racing the -// context cancellation; the lifecycle lock makes that race deterministic. +// queue send; the admission count makes that race deterministic. func (m *Mailbox) shutdown() { m.lifecycle.Lock() if m.closed { @@ -213,41 +291,50 @@ func (m *Mailbox) shutdown() { return } m.closed = true + var receivers []*mailboxReceiver + m.receivers.Range(func(_, value any) bool { + if receiver, ok := value.(*mailboxReceiver); ok { + receivers = append(receivers, receiver) + } + return true + }) + m.lifecycle.Unlock() + + // SendContext may have captured an attachment and be waiting for queue + // capacity. Closed admission prevents new senders from entering; wait for + // existing senders before draining so every accepted job has a clear owner. + m.admissions.Wait() for _, queue := range m.jobQueues { drain: for { select { - case pkg := <-queue: - api.ReleasePackage(pkg) + case job := <-queue: + api.ReleasePackage(job.pkg) default: break drain } } } - m.lifecycle.Unlock() - // Wait for in-flight deliveries to observe the canceled mailbox context - // before draining attached channels. This lock is separate from lifecycle: - // Detach must remain able to remove a blocked receiver without waiting for a - // potentially slow consumer. - m.delivery.Lock() - m.receivers.Range(func(_, value any) bool { - if ch, ok := value.(chan *api.Package); ok { - drainReceiver(ch) - } - return true - }) - m.delivery.Unlock() + // Stop every attachment outside lifecycle so Detach/Attach do not hold the + // global lock across a potentially blocked channel send. Each receiver's + // own active counter makes this wait finite once the mailbox context is + // canceled. + for _, receiver := range receivers { + receiver.stop() + drainReceiver(receiver.ch) + } } // deliver sends the package to the target's receiver channel. -func (m *Mailbox) deliver(pkg *api.Package) { +func (m *Mailbox) deliver(job mailboxJob) { + pkg := job.pkg if pkg == nil { return } targetKey := pkg.Target.String() - rec, ok := m.receivers.Load(targetKey) - if !ok { + receiver := job.receiver + if receiver == nil { var topic string if len(pkg.Messages) > 0 { topic = pkg.Messages[0].Topic @@ -260,26 +347,20 @@ func (m *Mailbox) deliver(pkg *api.Package) { return } - ch, ok := rec.(chan *api.Package) - if !ok { - m.config.logger.Error("receiver has invalid type", - zap.String("target", targetKey)) + if !receiver.begin() { api.ReleasePackage(pkg) return } - - m.deliverTo(ch, pkg, targetKey) + defer receiver.end() + m.deliverTo(receiver, pkg, targetKey) } // deliverTo performs the receiver-channel send. The channel is owned by the -// attached process, which may close it concurrently with this send (Detach only -// removes the map entry). A send on a closed channel panics, which must never -// take down the worker, so it is recovered: a closed receiver means the process -// is gone and the package is dropped. -func (m *Mailbox) deliverTo(ch chan *api.Package, pkg *api.Package, targetKey string) { - m.delivery.RLock() - defer m.delivery.RUnlock() - +// attached process, which may close it concurrently with this send. Detach +// cancels the receiver-local delivery context; a send on a closed channel +// panics, which must never take down the worker, so it is recovered: a closed +// receiver means the process is gone and the package is dropped. +func (m *Mailbox) deliverTo(receiver *mailboxReceiver, pkg *api.Package, targetKey string) { delivered := false defer func() { if r := recover(); r != nil { @@ -298,8 +379,11 @@ func (m *Mailbox) deliverTo(ch chan *api.Package, pkg *api.Package, targetKey st } select { - case ch <- pkg: + case receiver.ch <- pkg: delivered = true + case <-receiver.done: + m.config.logger.Debug("delivery detached", + zap.String("target", targetKey)) case <-m.ctx.Done(): m.config.logger.Debug("delivery canceled", zap.String("target", targetKey)) diff --git a/system/relay/mailbox_test.go b/system/relay/mailbox_test.go index 4ede8d865..5a6289026 100644 --- a/system/relay/mailbox_test.go +++ b/system/relay/mailbox_test.go @@ -5,6 +5,7 @@ package relay import ( "context" "fmt" + "sync" "sync/atomic" "testing" "time" @@ -79,7 +80,7 @@ func TestMailbox_Attach(t *testing.T) { // Test cancellation cancel1() time.Sleep(time.Millisecond * 10) // Allow time for the delete operation - _, exists := mailbox.receivers.Load(pid) + _, exists := mailbox.receivers.Load(pid.String()) assert.False(t, exists) } @@ -201,6 +202,115 @@ func TestMailbox_DetachReleasesBufferedPackages(t *testing.T) { "detaching a receiver must release packages already accepted by its channel") } +func TestMailbox_DetachReattachDoesNotCrossReceiverGeneration(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(8), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "generation"} + oldCh := make(chan *relay.Package, 8) + _, err := mailbox.Attach(target, oldCh) + require.NoError(t, err) + + oldPkg, oldProbe := probedPackage(target) + require.NoError(t, mailbox.Send(oldPkg)) + mailbox.Detach(target) + require.Eventually(t, func() bool { return oldProbe.releases.Load() == 1 }, time.Second, time.Millisecond) + select { + case <-oldCh: + t.Fatal("detached receiver retained a package") + default: + } + + newCh := make(chan *relay.Package, 1) + _, err = mailbox.Attach(target, newCh) + require.NoError(t, err) + newPkg, newProbe := probedPackage(target) + require.NoError(t, mailbox.Send(newPkg)) + select { + case delivered := <-newCh: + relay.ReleasePackage(delivered) + case <-time.After(time.Second): + t.Fatal("reattached receiver did not receive its package") + } + require.Equal(t, int32(1), newProbe.releases.Load()) + select { + case <-oldCh: + t.Fatal("old receiver received a package after reattach") + default: + } + mailbox.Detach(target) +} + +func TestMailbox_DetachReattachConcurrentStress(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(32), WithWorkerCount(2)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "stress"} + + for round := 0; round < 100; round++ { + oldCh := make(chan *relay.Package, 32) + _, err := mailbox.Attach(target, oldCh) + require.NoError(t, err) + probes := make([]*mailboxReleaseProbe, 8) + var sends sync.WaitGroup + for i := range probes { + pkg, probe := probedPackage(target) + probes[i] = probe + sends.Add(1) + go func(pkg *relay.Package) { + defer sends.Done() + _ = mailbox.Send(pkg) + }(pkg) + } + mailbox.Detach(target) + sends.Wait() + for _, probe := range probes { + require.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond) + } + newCh := make(chan *relay.Package, 1) + _, err = mailbox.Attach(target, newCh) + require.NoError(t, err) + mailbox.Detach(target) + } +} + +func TestMailbox_DetachDoesNotWaitForBlockedAdmission(t *testing.T) { + mailbox := NewMailbox(context.Background(), WithBufferSize(1), WithWorkerCount(1)) + target := pidapi.PID{Node: "node1", Host: "host1", UniqID: "blocked-admission"} + _, err := mailbox.Attach(target, make(chan *relay.Package)) + require.NoError(t, err) + + packages := make([]*relay.Package, 3) + probes := make([]*mailboxReleaseProbe, 3) + for i := range packages { + packages[i], probes[i] = probedPackage(target) + } + // The first job blocks in the unbuffered receiver; the second fills the + // mailbox queue, leaving the third sender blocked on admission. + require.NoError(t, mailbox.Send(packages[0])) + require.NoError(t, mailbox.Send(packages[1])) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + errCh := make(chan error, 1) + go func() { errCh <- mailbox.SendContext(ctx, packages[2]) }() + require.Eventually(t, func() bool { + return len(mailbox.jobQueues[0]) == 1 + }, time.Second, time.Millisecond, "second package did not remain queued behind blocked delivery") + + detached := make(chan struct{}) + go func() { + mailbox.Detach(target) + close(detached) + }() + select { + case <-detached: + case <-time.After(time.Second): + t.Fatal("Detach waited on a blocked queue admission") + } + if err := <-errCh; err != nil { + relay.ReleasePackage(packages[2]) + } + for _, probe := range probes { + require.Eventually(t, func() bool { return probe.releases.Load() == 1 }, time.Second, time.Millisecond) + } +} + func TestMailbox_ShutdownReleasesQueuedPackages(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mailbox := NewMailbox(ctx, WithBufferSize(8), WithWorkerCount(1)) @@ -262,10 +372,10 @@ func TestMailbox_DetachDuringDelivery(t *testing.T) { // Message should be dropped without error } -// A receiver's owner closes its own channel on teardown (Detach only removes the -// map entry). A delivery racing that close hits a send-on-closed-channel, which -// must be dropped, never panicking the worker — proven by a subsequent delivery -// to a live receiver on the same worker still arriving. +// A receiver's owner closes its own channel on teardown. A delivery racing that +// close hits a send-on-closed-channel, which must be dropped, never panicking +// the worker — proven by a subsequent delivery to a live receiver on the same +// worker still arriving. func TestMailbox_ClosedReceiverDoesNotKillWorker(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() From 2160500d01f81047bb5a107a131b1b9516be0d00 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 04:31:16 -0400 Subject: [PATCH 45/47] fix(cdc/postgres): make snapshots per-subscriber --- service/cdc/postgres/driver.go | 12 +- service/cdc/postgres/driver_test.go | 12 +- .../cdc/postgres/integration_snapshot_test.go | 144 +++++++++-- service/cdc/postgres/service.go | 236 +++++++++++++++--- service/cdc/postgres/stream.go | 165 ++++++++++-- service/cdc/postgres/stream_test.go | 138 +++++++++- 6 files changed, 601 insertions(+), 106 deletions(-) diff --git a/service/cdc/postgres/driver.go b/service/cdc/postgres/driver.go index d33c63ea5..aacd50545 100644 --- a/service/cdc/postgres/driver.go +++ b/service/cdc/postgres/driver.go @@ -4,7 +4,6 @@ package postgres import ( "context" - "fmt" "net" "strconv" "strings" @@ -113,10 +112,9 @@ func (s *sourceAdapter) Info() config.SourceInfo { Snapshot: source.snapshot, State: postgresSourceState(state), Capabilities: config.Capabilities{ - // Snapshot is a source-start bootstrap operation. Subscribe rejects - // per-consumer snapshot requests, so it is not a common API - // capability of this adapter. - Snapshot: false, + // Snapshot is an atomic per-subscriber handoff. The source Snapshot + // field is only the entry default for that capability. + Snapshot: true, Durable: !source.temporary, Replayable: false, CapturesExternalWrites: true, @@ -145,10 +143,6 @@ func (s *sourceAdapter) Subscribe(ctx context.Context, opts config.StreamOptions if opts.After != "" { return nil, config.ErrUnsupported } - if opts.Snapshot { - return nil, fmt.Errorf("%w: snapshot is configured on the source", config.ErrUnsupported) - } - s.mu.RLock() source := s.source s.mu.RUnlock() diff --git a/service/cdc/postgres/driver_test.go b/service/cdc/postgres/driver_test.go index e4ece526e..a618a1a27 100644 --- a/service/cdc/postgres/driver_test.go +++ b/service/cdc/postgres/driver_test.go @@ -3,6 +3,7 @@ package postgres import ( + "context" "errors" "testing" @@ -31,14 +32,21 @@ func TestSourceAdapterInfoReportsConservativeCapabilities(t *testing.T) { assert.Equal(t, cconfig.SourceStateFaulted, info.State) assert.True(t, info.Faulted) assert.Equal(t, terminalErr.Error(), info.Error) - assert.True(t, info.Snapshot, "legacy snapshot field preserves configured bootstrap mode") + assert.True(t, info.Snapshot, "entry snapshot field preserves the configured subscriber default") assert.True(t, info.Streaming, "legacy streaming field preserves configured protocol mode") - assert.False(t, info.Capabilities.Snapshot, "per-consumer snapshots are unsupported") + assert.True(t, info.Capabilities.Snapshot, "per-consumer snapshots use the atomic handoff") assert.False(t, info.Capabilities.Replayable, "After cursors are unsupported") assert.False(t, info.Capabilities.Durable, "temporary slots are not durable") assert.False(t, info.Capabilities.BeforeImages) } +func TestSourceAdapterSnapshotRequiresRunningGeneration(t *testing.T) { + source := NewSource(SourceOptions{Slot: "events_slot"}) + adapter := &sourceAdapter{source: source} + _, err := adapter.Subscribe(context.Background(), cconfig.StreamOptions{Snapshot: true}) + assert.ErrorIs(t, err, cconfig.ErrSourceNotReady) +} + func TestSourceStopAfterDisposeCleanupIsIdempotent(t *testing.T) { source := NewSource(SourceOptions{Slot: "events_slot"}) source.dropDone.Store(true) diff --git a/service/cdc/postgres/integration_snapshot_test.go b/service/cdc/postgres/integration_snapshot_test.go index a9dd05d24..549da4aee 100644 --- a/service/cdc/postgres/integration_snapshot_test.go +++ b/service/cdc/postgres/integration_snapshot_test.go @@ -6,11 +6,13 @@ import ( "context" "database/sql" "errors" + "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + cdcapi "github.com/wippyai/runtime/api/service/cdc" ) func waitForSnapshotEmail(t *testing.T, b *changeCapture, email string, op Op, timeout time.Duration) RowChange { @@ -29,6 +31,31 @@ func waitForSnapshotEmail(t *testing.T, b *changeCapture, email string, op Op, t } } +func attachSnapshotCapture(t *testing.T, ctx context.Context, src *Source, capture *changeCapture, capacity ...int) { + t.Helper() + size := 8192 + if len(capacity) > 0 && capacity[0] > 0 { + size = capacity[0] + } + stream := src.Subscribe(cdcapi.StreamOptions{Snapshot: true, Buffer: size}) + require.NotNil(t, stream) + t.Cleanup(stream.Close) + go func() { + for { + select { + case change, ok := <-stream.Changes(): + if !ok { + return + } + capture.send(rowChangeFromAPI(change)) + case <-ctx.Done(): + stream.Close() + return + } + } + }() +} + func TestSnapshotBootstrapsExistingRows(t *testing.T) { repl, admin := dsns(t) db, err := sql.Open("postgres", admin) @@ -50,10 +77,9 @@ func TestSnapshotBootstrapsExistingRows(t *testing.T) { }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - attachCapture(t, ctx, src, capture) - _, err = src.Start(ctx) require.NoError(t, err) + attachSnapshotCapture(t, ctx, src, capture) seen := map[string]Op{} deadline := time.After(15 * time.Second) @@ -113,9 +139,9 @@ func TestSnapshotPreservesNull(t *testing.T) { }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - attachCapture(t, ctx, src, capture) _, err = src.Start(ctx) require.NoError(t, err) + attachSnapshotCapture(t, ctx, src, capture) rc := waitForSnapshotEmail(t, capture, "null@w.ai", OpSnapshot, 15*time.Second) assert.Nil(t, rc.After["note"], "NULL column must map to nil in snapshot row") @@ -125,7 +151,7 @@ func TestSnapshotPreservesNull(t *testing.T) { stopCancel() } -func TestSnapshotSkippedOnResume(t *testing.T) { +func TestSnapshotDefaultAppliesPerSubscriberAfterResume(t *testing.T) { repl, admin := dsns(t) db, err := sql.Open("postgres", admin) require.NoError(t, err) @@ -148,9 +174,9 @@ func TestSnapshotSkippedOnResume(t *testing.T) { } src := mk(capture) ctx, cancel := context.WithCancel(context.Background()) - attachCapture(t, ctx, src, capture) _, err = src.Start(ctx) require.NoError(t, err) + attachSnapshotCapture(t, ctx, src, capture) waitForSnapshotEmail(t, capture, "resume-base@w.ai", OpSnapshot, 15*time.Second) require.Eventually(t, func() bool { var raw string @@ -166,22 +192,25 @@ func TestSnapshotSkippedOnResume(t *testing.T) { src2 := mk(capture2) ctx2, cancel2 := context.WithCancel(context.Background()) defer cancel2() - attachCapture(t, ctx2, src2, capture2) _, err = src2.Start(ctx2) require.NoError(t, err) + attachSnapshotCapture(t, ctx2, src2, capture2) _, err = db.Exec(`INSERT INTO accounts (email, balance) VALUES ('resume-new@w.ai', 2)`) require.NoError(t, err) deadline := time.After(15 * time.Second) - got := false - for !got { + gotNew := false + for !gotNew { select { case rc := <-capture2.ch: - assert.NotEqual(t, OpSnapshot, rc.Op, "resume must not re-snapshot existing rows") - if em, _ := rc.After["email"].(string); em == "resume-new@w.ai" { + em, _ := rc.After["email"].(string) + if em == "resume-base@w.ai" { + assert.Equal(t, OpSnapshot, rc.Op, "entry snapshot is per subscriber, including resumed sources") + } + if em == "resume-new@w.ai" { assert.Equal(t, OpInsert, rc.Op) - got = true + gotNew = true } case <-deadline: t.Fatal("resumed source did not stream the new insert") @@ -212,23 +241,26 @@ func TestSnapshotFailureDropsSlotForCleanRetry(t *testing.T) { src := NewSource(SourceOptions{ ReplDSN: repl, AdminDSN: admin, Slot: itSlot, Publication: "wippy_cdc_pub", - Snapshot: true, StandbyInterval: 200 * time.Millisecond, StatusInterval: time.Hour, + StandbyInterval: 200 * time.Millisecond, StatusInterval: time.Hour, }) ctx, cancel := context.WithCancel(context.Background()) - status, err := src.Start(ctx) + _, err = src.Start(ctx) require.NoError(t, err) - + stream := src.Subscribe(cdcapi.StreamOptions{Snapshot: true, Buffer: 8}) + require.NotNil(t, stream) select { - case <-statusClosed(status): + case _, ok := <-stream.Changes(): + require.False(t, ok) case <-time.After(15 * time.Second): - t.Fatal("run did not exit after injected snapshot failure") + t.Fatal("subscriber snapshot did not fail") } + assert.ErrorContains(t, stream.Err(), "injected snapshot failure") + stream.Close() + assert.Equal(t, 1, slotCount(t, db, itSlot), "subscriber snapshot failure must not drop the source slot") cancel() - - assert.Equal(t, 0, slotCount(t, db, itSlot), "snapshot failure must drop the fresh slot for a clean retry") - var offsets int - require.NoError(t, db.QueryRow(`SELECT count(*) FROM wippy_cdc_offsets WHERE slot=$1`, itSlot).Scan(&offsets)) - assert.Equal(t, 0, offsets, "snapshot failure must delete the checkpoint") + stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second) + require.NoError(t, src.Stop(stopCtx)) + stopCancel() snapshotFailpoint = nil capture2 := newChangeCapture() @@ -238,12 +270,80 @@ func TestSnapshotFailureDropsSlotForCleanRetry(t *testing.T) { }) ctx2, cancel2 := context.WithCancel(context.Background()) defer cancel2() - attachCapture(t, ctx2, src2, capture2) _, err = src2.Start(ctx2) require.NoError(t, err) + attachSnapshotCapture(t, ctx2, src2, capture2) waitForSnapshotEmail(t, capture2, "retry@w.ai", OpSnapshot, 15*time.Second) stopCtx, sc := context.WithTimeout(context.Background(), 5*time.Second) require.NoError(t, src2.Stop(stopCtx)) sc() } + +func TestPerSubscriberSnapshotHandoffUsesCommitFence(t *testing.T) { + repl, admin := dsns(t) + db, err := sql.Open("postgres", admin) + require.NoError(t, err) + defer func() { _ = db.Close() }() + setupSchema(t, db) + const slot = "wippy_cdc_dynamic_snapshot" + dropNamedSlot(t, repl, slot) + defer dropNamedSlot(t, repl, slot) + + _, err = db.Exec(`DELETE FROM accounts`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO accounts (email, balance) VALUES ('before@w.ai', 1)`) + require.NoError(t, err) + + src := NewSource(SourceOptions{ + ReplDSN: repl, AdminDSN: admin, Slot: slot, Publication: "wippy_cdc_pub", + StandbyInterval: 200 * time.Millisecond, StatusInterval: time.Hour, + }) + _, err = src.Start(context.Background()) + require.NoError(t, err) + defer func() { _ = src.Stop(context.Background()) }() + + fenceReady := make(chan struct{}) + releaseSnapshot := make(chan struct{}) + var once sync.Once + snapshotFailpoint = func() error { + once.Do(func() { close(fenceReady) }) + <-releaseSnapshot + return nil + } + defer func() { snapshotFailpoint = nil }() + + stream := src.Subscribe(cdcapi.StreamOptions{Snapshot: true, Buffer: 64}) + require.NotNil(t, stream) + defer stream.Close() + select { + case <-fenceReady: + case <-time.After(15 * time.Second): + t.Fatal("subscriber snapshot did not establish its exported fence") + } + _, err = db.Exec(`INSERT INTO accounts (email, balance) VALUES ('after@w.ai', 2)`) + require.NoError(t, err) + close(releaseSnapshot) + + seenBefore := false + seenAfter := false + deadline := time.After(15 * time.Second) + for !seenBefore || !seenAfter { + select { + case change, ok := <-stream.Changes(): + require.True(t, ok, "snapshot stream closed: %v", stream.Err()) + email, _ := change.After["email"].(string) + switch email { + case "before@w.ai": + require.Equal(t, OpSnapshot, Op(change.Op)) + seenBefore = true + case "after@w.ai": + require.Equal(t, OpInsert, Op(change.Op)) + require.NotEmpty(t, change.CommitLSN) + seenAfter = true + } + case <-deadline: + t.Fatalf("snapshot/live handoff incomplete: before=%v after=%v", seenBefore, seenAfter) + } + } +} diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 6fbaa70cb..a3c8107d9 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -4,6 +4,7 @@ package postgres import ( "context" + "crypto/sha256" "database/sql" "errors" "fmt" @@ -53,9 +54,12 @@ type SourceOptions struct { StatusInterval time.Duration SnapshotFetchSize int Temporary bool - Snapshot bool - Streaming bool - Failover bool + // Snapshot makes the atomic snapshot handoff the default for each + // subscriber. It is an entry default; Start never emits a source-global + // snapshot. + Snapshot bool + Streaming bool + Failover bool // MaxTransactionChanges bounds the number of row changes retained before // an ordinary or streamed transaction commits. Zero uses the safe default. MaxTransactionChanges int @@ -78,6 +82,7 @@ type Source struct { cancel context.CancelFunc done chan struct{} subs map[uint64]*sourceSubscription + streamNotify chan struct{} replDSN string adminDSN string name string @@ -92,6 +97,8 @@ type Source struct { maxTransactionBytes int64 maxInflightChanges int maxInflightBytes int64 + snapshotWG sync.WaitGroup + streamPosition pglogrepl.LSN subMu sync.RWMutex mu sync.Mutex dropMu sync.Mutex @@ -179,6 +186,7 @@ func NewSource(opts SourceOptions) *Source { maxTransactionBytes: limits.maxBytes, maxInflightChanges: limits.maxInflightChanges, maxInflightBytes: limits.maxInflightBytes, + streamNotify: make(chan struct{}), } } @@ -282,7 +290,7 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { return nil, startErr } - startLSN, snapshotName, slotCreated, err := s.prepareSlot(runCtx, conn, adminDB, cp, sysident.XLogPos) + startLSN, slotCreated, err := s.prepareSlot(runCtx, conn, adminDB, cp, sysident.XLogPos) if err != nil { _ = conn.Close(context.Background()) _ = adminDB.Close() @@ -311,20 +319,25 @@ func (s *Source) Start(ctx context.Context) (<-chan any, error) { } s.state = sourceRunning s.sourceErr = nil + s.publication = publication + s.streamPosition = startLSN + if s.streamNotify == nil { + s.streamNotify = make(chan struct{}) + } s.mu.Unlock() s.log.Info("cdc source started", zap.String("slot", s.slot), zap.String("publication", publication), zap.String("start_lsn", startLSN.String()), - zap.Bool("snapshot", snapshotName != "")) + zap.Bool("snapshot", s.snapshot)) select { case status <- "cdc replication started": default: } s.coll = metrics.GetCollector(runCtx) - go s.run(runCtx, conn, adminDB, cp, startLSN, snapshotName, slotCreated, publication, s.coll, status, done) + go s.run(runCtx, conn, adminDB, cp, startLSN, slotCreated, publication, s.coll, status, done) return status, nil } @@ -337,6 +350,9 @@ func (s *Source) Stop(ctx context.Context) error { if s.state == sourceStopped { drop := s.dropSlot.Load() s.mu.Unlock() + if err := s.waitSnapshots(ctx); err != nil { + return err + } if drop { return s.dropSlotAndCheckpoint(ctx) } @@ -348,6 +364,9 @@ func (s *Source) Stop(ctx context.Context) error { s.cancel = nil s.mu.Unlock() s.closeSubscriptions() + if err := s.waitSnapshots(ctx); err != nil { + return err + } if s.dropSlot.Load() { return s.dropSlotAndCheckpoint(ctx) } @@ -388,6 +407,9 @@ func (s *Source) Stop(ctx context.Context) error { } s.mu.Unlock() } + if err := s.waitSnapshots(ctx); err != nil { + return err + } if s.dropSlot.Load() { return s.dropSlotAndCheckpoint(ctx) @@ -395,13 +417,84 @@ func (s *Source) Stop(ctx context.Context) error { return nil } +func (s *Source) startSnapshot(ctx context.Context, sub *sourceSubscription) { + snapshotCtx, cancel := context.WithCancel(ctx) + s.snapshotWG.Add(1) + go func() { + defer s.snapshotWG.Done() + watchDone := make(chan struct{}) + go func() { + select { + case <-sub.done: + cancel() + case <-snapshotCtx.Done(): + case <-watchDone: + } + }() + defer close(watchDone) + + fence, err := s.snapshotCurrentTo(snapshotCtx, sub) + if err != nil { + sub.finishSnapshot(0, err) + return + } + sub.finishSnapshot(fence, nil) + }() +} + +func (s *Source) waitSnapshots(ctx context.Context) error { + done := make(chan struct{}) + go func() { + s.snapshotWG.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// advanceStreamPosition publishes the replication receive watermark after a +// complete XLogData message has been decoded and emitted. Snapshot handoff +// waits for this watermark before releasing its pending live queue, so a +// change at or before the exported snapshot fence cannot arrive late and be +// duplicated after the handoff. +func (s *Source) advanceStreamPosition(done chan struct{}, position pglogrepl.LSN) { + s.mu.Lock() + defer s.mu.Unlock() + if s.done != done || position <= s.streamPosition { + return + } + s.streamPosition = position + close(s.streamNotify) + s.streamNotify = make(chan struct{}) +} + +func (s *Source) waitStreamPosition(ctx context.Context, fence pglogrepl.LSN) error { + for { + s.mu.Lock() + if s.streamPosition >= fence { + s.mu.Unlock() + return nil + } + notify := s.streamNotify + s.mu.Unlock() + select { + case <-notify: + case <-ctx.Done(): + return ctx.Err() + } + } +} + func (s *Source) run( ctx context.Context, conn *pgconn.PgConn, adminDB *sql.DB, cp Checkpointer, startLSN pglogrepl.LSN, - snapshotName string, slotCreated bool, publication string, mc metrics.Collector, @@ -433,14 +526,6 @@ func (s *Source) run( defer func() { _ = adminDB.Close() }() defer func() { _ = conn.Close(context.Background()) }() - if snapshotName != "" { - if err := s.snapshotExisting(ctx, adminDB, publication, snapshotName); err != nil { - s.abortFreshSlot(conn, slotCreated) - s.fail(ctx, status, err) - return - } - } - protoVersion := config.ProtocolVersion if s.streaming { protoVersion = config.StreamingProtocolVersion @@ -617,6 +702,7 @@ func (s *Source) run( safePos = end } } + s.advanceStreamPosition(done, xld.WALStart+pglogrepl.LSN(len(xld.WALData))) default: s.fail(ctx, status, fmt.Errorf("%w: copy data kind %q", ErrUnsupportedMessage, cd.Data[0])) return @@ -683,11 +769,11 @@ func (s *Source) prepareSlot( adminDB *sql.DB, cp Checkpointer, fallback pglogrepl.LSN, -) (pglogrepl.LSN, string, bool, error) { +) (pglogrepl.LSN, bool, error) { var start pglogrepl.LSN resumed := false if cpLSN, ok, err := cp.Load(ctx, s.slot); err != nil { - return 0, "", false, err + return 0, false, err } else if ok { start = cpLSN resumed = true @@ -698,14 +784,14 @@ func (s *Source) prepareSlot( var err error exists, err = slotExists(ctx, adminDB, s.slot) if err != nil { - return 0, "", false, err + return 0, false, err } if !exists && resumed { // A local offset is meaningful only for the server-side slot // incarnation that produced it. If that slot disappeared, do not // reuse the old LSN for a newly-created slot. if err := cp.Delete(ctx, s.slot); err != nil { - return 0, "", false, fmt.Errorf("delete stale cdc checkpoint: %w", err) + return 0, false, fmt.Errorf("delete stale cdc checkpoint: %w", err) } start = 0 resumed = false @@ -716,7 +802,7 @@ func (s *Source) prepareSlot( // state is missing; doing so can skip retained logical changes. confirmed, valid, err := slotConfirmedFlush(ctx, adminDB, s.slot) if err != nil { - return 0, "", false, err + return 0, false, err } if valid { start = confirmed @@ -726,51 +812,42 @@ func (s *Source) prepareSlot( // Temporary slots are destroyed with their replication connection, so // any persisted offset belongs to an older slot incarnation. if err := cp.Delete(ctx, s.slot); err != nil { - return 0, "", false, fmt.Errorf("delete stale cdc checkpoint: %w", err) + return 0, false, fmt.Errorf("delete stale cdc checkpoint: %w", err) } start = 0 - resumed = false } - snapshotName := "" slotCreated := false if !exists { slotIdentifier, err := quoteReplicationSlotName(s.slot) if err != nil { - return 0, "", false, err + return 0, false, err } opts := pglogrepl.CreateReplicationSlotOptions{Temporary: s.temporary} - wantSnapshot := s.snapshot && !resumed - if wantSnapshot { - opts.SnapshotAction = "EXPORT_SNAPSHOT" - } res, err := pglogrepl.CreateReplicationSlot(ctx, conn, slotIdentifier, config.OutputPlugin, opts) if err != nil { - return 0, "", false, fmt.Errorf("create replication slot: %w", err) + return 0, false, fmt.Errorf("create replication slot: %w", err) } slotCreated = true cpoint, err := pglogrepl.ParseLSN(res.ConsistentPoint) if err != nil { - return 0, "", slotCreated, fmt.Errorf("parse consistent point %q: %w", res.ConsistentPoint, err) + return 0, slotCreated, fmt.Errorf("parse consistent point %q: %w", res.ConsistentPoint, err) } if cpoint > start { start = cpoint } - if wantSnapshot { - snapshotName = res.SnapshotName - } } if s.failover && !s.temporary { if err := s.setSlotFailover(ctx, conn); err != nil { - return 0, "", slotCreated, err + return 0, slotCreated, err } } if start == 0 { start = fallback } - return start, snapshotName, slotCreated, nil + return start, slotCreated, nil } func (s *Source) setSlotFailover(ctx context.Context, conn *pgconn.PgConn) error { @@ -795,7 +872,7 @@ func (t tableRef) quoted() string { return pq.QuoteIdentifier(t.schema) + "." + pq.QuoteIdentifier(t.name) } -func (s *Source) snapshotExisting(ctx context.Context, adminDB *sql.DB, publication, snapshotName string) error { +func (s *Source) snapshotWithSink(ctx context.Context, adminDB *sql.DB, publication, snapshotName string, sink snapshotSink) error { conn, err := adminDB.Conn(ctx) if err != nil { return fmt.Errorf("snapshot connection: %w", err) @@ -837,7 +914,7 @@ func (s *Source) snapshotExisting(ctx context.Context, adminDB *sql.DB, publicat total := 0 for _, tbl := range tables { - n, err := s.snapshotTable(ctx, conn, tbl) + n, err := s.snapshotTableWithSink(ctx, conn, tbl, sink) if err != nil { return err } @@ -853,7 +930,82 @@ func (s *Source) snapshotExisting(ctx context.Context, adminDB *sql.DB, publicat return nil } -func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, tbl tableRef) (int, error) { +// snapshotCurrent establishes an exported logical-decoding snapshot for one +// subscriber. The temporary slot's consistent point is the exact WAL fence; +// unlike a SQL-only pg_current_wal_lsn query, it cannot race a concurrent +// commit between snapshot acquisition and fence capture. +func (s *Source) snapshotCurrentTo(ctx context.Context, sub *sourceSubscription) (pglogrepl.LSN, error) { + replConn, err := pgconn.Connect(ctx, s.replDSN) + if err != nil { + return 0, fmt.Errorf("snapshot replication connection: %w", err) + } + defer func() { _ = replConn.Close(context.Background()) }() + if _, err := pglogrepl.IdentifySystem(ctx, replConn); err != nil { + return 0, fmt.Errorf("identify snapshot replication system: %w", err) + } + + snapshotSlot := subscriberSnapshotSlot(s.slot, sub.id) + snapshotSlotID, err := quoteReplicationSlotName(snapshotSlot) + if err != nil { + return 0, err + } + result, err := pglogrepl.CreateReplicationSlot(ctx, replConn, snapshotSlotID, config.OutputPlugin, + pglogrepl.CreateReplicationSlotOptions{Temporary: true, SnapshotAction: "EXPORT_SNAPSHOT"}) + if err != nil { + return 0, fmt.Errorf("create subscriber snapshot slot: %w", err) + } + fence, err := pglogrepl.ParseLSN(result.ConsistentPoint) + if err != nil { + return 0, fmt.Errorf("parse subscriber snapshot fence %q: %w", result.ConsistentPoint, err) + } + if result.SnapshotName == "" { + return 0, errors.New("subscriber snapshot slot returned no exported snapshot") + } + if err := s.waitStreamPosition(ctx, fence); err != nil { + return 0, fmt.Errorf("wait for replication fence: %w", err) + } + + adminDB, err := sql.Open("postgres", s.adminDSN) + if err != nil { + return 0, fmt.Errorf("open snapshot connection: %w", err) + } + defer func() { _ = adminDB.Close() }() + adminDB.SetMaxOpenConns(1) + adminDB.SetMaxIdleConns(1) + if err := adminDB.PingContext(ctx); err != nil { + return 0, fmt.Errorf("ping snapshot connection: %w", err) + } + + err = s.snapshotWithSink(ctx, adminDB, s.publication, result.SnapshotName, func(rc RowChange) error { + if !sub.matchesSnapshot(config.Change{Table: rc.Table, Relation: rc.Relation()}) { + return nil + } + change := config.Change{ + Source: s.name, + Op: string(OpSnapshot), + Schema: rc.Schema, + Table: rc.Table, + Relation: rc.Relation(), + CommitLSN: fence.String(), + Before: rc.Before, + After: rc.After, + } + return sub.sendSnapshot(change, config.EstimateChangeBytes(change)) + }) + if err != nil { + return 0, fmt.Errorf("subscriber snapshot scan: %w", err) + } + return fence, nil +} + +func subscriberSnapshotSlot(slot string, id uint64) string { + digest := sha256.Sum256([]byte(slot)) + return fmt.Sprintf("wippy_snap_%x_%d", digest[:8], id) +} + +type snapshotSink func(RowChange) error + +func (s *Source) snapshotTableWithSink(ctx context.Context, conn *sql.Conn, tbl tableRef, sink snapshotSink) (int, error) { if _, err := conn.ExecContext(ctx, "DECLARE "+snapshotCursor+" NO SCROLL CURSOR FOR SELECT * FROM "+tbl.quoted()); err != nil { return 0, fmt.Errorf("declare cursor %s.%s: %w", tbl.schema, tbl.name, err) @@ -863,7 +1015,7 @@ func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, tbl tableRef fetchSQL := fmt.Sprintf("FETCH %d FROM %s", s.snapshotFetchSize, snapshotCursor) n := 0 for { - got, err := s.fetchSnapshotBatch(ctx, conn, tbl, fetchSQL) + got, err := s.fetchSnapshotBatch(ctx, conn, tbl, fetchSQL, sink) if err != nil { return n, err } @@ -874,7 +1026,7 @@ func (s *Source) snapshotTable(ctx context.Context, conn *sql.Conn, tbl tableRef } } -func (s *Source) fetchSnapshotBatch(ctx context.Context, conn *sql.Conn, tbl tableRef, fetchSQL string) (int, error) { +func (s *Source) fetchSnapshotBatch(ctx context.Context, conn *sql.Conn, tbl tableRef, fetchSQL string, sink snapshotSink) (int, error) { rows, err := conn.QueryContext(ctx, fetchSQL) if err != nil { return 0, fmt.Errorf("fetch %s.%s: %w", tbl.schema, tbl.name, err) @@ -905,7 +1057,9 @@ func (s *Source) fetchSnapshotBatch(ctx context.Context, conn *sql.Conn, tbl tab after[c] = nil } } - s.emitChange(ctx, RowChange{Op: OpSnapshot, Schema: tbl.schema, Table: tbl.name, After: after}) + if err := sink(RowChange{Op: OpSnapshot, Schema: tbl.schema, Table: tbl.name, After: after}); err != nil { + return got, err + } got++ } return got, rows.Err() diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 770bdf0ed..24ba3144a 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/jackc/pglogrepl" config "github.com/wippyai/runtime/api/service/cdc" ) @@ -20,23 +21,26 @@ const ( // that cannot keep up must not back-pressure the replication receive loop or // unrelated subscribers. var errSubscriberOverflow = errors.New("postgres cdc subscriber backlog overflow") +var errSnapshotNotActive = errors.New("postgres cdc snapshot is no longer active") type sourceSubscription struct { - err error - tables map[string]struct{} - source *Source - done chan struct{} - notify chan struct{} - relayDone chan struct{} - ops map[string]struct{} - out chan config.Change - queue []queuedChange - maxBytes int64 - maxChanges int - id uint64 - queuedBytes int64 - mu sync.Mutex - closed bool + err error + tables map[string]struct{} + source *Source + done chan struct{} + notify chan struct{} + relayDone chan struct{} + ops map[string]struct{} + out chan config.Change + queue []queuedChange + pending []queuedChange + maxBytes int64 + maxChanges int + id uint64 + queuedBytes int64 + mu sync.Mutex + closed bool + snapshotting bool } type queuedChange struct { @@ -53,7 +57,16 @@ func (s *Source) Subscribe(opts config.StreamOptions) config.Stream { if s.state != sourceNew && s.state != sourceStarting && s.state != sourceRunning { return nil } - return s.newSubscription(opts) + effectiveSnapshot := opts.Snapshot || s.snapshot + if effectiveSnapshot && s.state != sourceRunning { + return nil + } + opts.Snapshot = effectiveSnapshot + sub := s.newSubscription(opts) + if effectiveSnapshot { + s.startSnapshot(context.Background(), sub) + } + return sub } // subscribe is the driver-facing subscription path. It holds the source @@ -74,10 +87,19 @@ func (s *Source) subscribe(ctx context.Context, opts config.StreamOptions) (conf s.permanentlyClosed || s.sourceErr != nil { return nil, config.ErrSourceNotReady } - return s.newSubscription(opts), nil + effectiveSnapshot := opts.Snapshot || s.snapshot + if effectiveSnapshot && s.state != sourceRunning { + return nil, config.ErrSourceNotReady + } + opts.Snapshot = effectiveSnapshot + sub := s.newSubscription(opts) + if effectiveSnapshot { + s.startSnapshot(ctx, sub) + } + return sub, nil } -func (s *Source) newSubscription(opts config.StreamOptions) config.Stream { +func (s *Source) newSubscription(opts config.StreamOptions) *sourceSubscription { buffer := opts.Buffer if buffer <= 0 { buffer = defaultStreamBuffer @@ -94,14 +116,15 @@ func (s *Source) newSubscription(opts config.StreamOptions) config.Stream { // queue is the sole driver-owned backlog. out is an unbuffered // delivery handoff, so bytes are released exactly after a consumer // receives the change rather than when it is merely enqueued. - out: make(chan config.Change), - done: make(chan struct{}), - notify: make(chan struct{}, 1), - maxChanges: buffer, - maxBytes: opts.EffectiveMaxBytes(), - relayDone: make(chan struct{}), - tables: filterSet(opts.Tables), - ops: filterSet(opts.Ops), + out: make(chan config.Change), + done: make(chan struct{}), + notify: make(chan struct{}, 1), + maxChanges: buffer, + maxBytes: opts.EffectiveMaxBytes(), + snapshotting: opts.Snapshot, + relayDone: make(chan struct{}), + tables: filterSet(opts.Tables), + ops: filterSet(opts.Ops), } s.subs[sub.id] = sub s.subMu.Unlock() @@ -194,6 +217,7 @@ func (s *sourceSubscription) closeLocked(err error) (*Source, uint64) { s.closed = true s.err = err s.queue = nil + s.pending = nil s.queuedBytes = 0 close(s.done) return s.source, s.id @@ -243,7 +267,7 @@ func (s *sourceSubscription) send(_ context.Context, change config.Change, bytes s.mu.Unlock() return } - if len(s.queue) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { + if len(s.queue)+len(s.pending) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { parent, id := s.closeLocked(errSubscriberOverflow) s.mu.Unlock() if parent != nil { @@ -251,6 +275,42 @@ func (s *sourceSubscription) send(_ context.Context, change config.Change, bytes } return } + item := queuedChange{change: change, bytes: bytes} + if s.snapshotting { + s.pending = append(s.pending, item) + } else { + s.queue = append(s.queue, item) + } + s.queuedBytes += bytes + s.mu.Unlock() + select { + case s.notify <- struct{}{}: + default: + } +} + +func (s *sourceSubscription) sendSnapshot(change config.Change, bytes int64) error { + s.mu.Lock() + if s.closed { + err := s.err + if err == nil { + err = context.Canceled + } + s.mu.Unlock() + return err + } + if !s.snapshotting { + s.mu.Unlock() + return errSnapshotNotActive + } + if len(s.queue)+len(s.pending) >= s.maxChanges || bytes > s.maxBytes-s.queuedBytes { + parent, id := s.closeLocked(errSubscriberOverflow) + s.mu.Unlock() + if parent != nil { + parent.removeSubscription(id) + } + return errSubscriberOverflow + } s.queue = append(s.queue, queuedChange{change: change, bytes: bytes}) s.queuedBytes += bytes s.mu.Unlock() @@ -258,6 +318,46 @@ func (s *sourceSubscription) send(_ context.Context, change config.Change, bytes case s.notify <- struct{}{}: default: } + return nil +} + +func (s *sourceSubscription) finishSnapshot(fence pglogrepl.LSN, err error) { + if err != nil { + s.closeWithError(err) + return + } + s.mu.Lock() + if s.closed || !s.snapshotting { + s.mu.Unlock() + return + } + for _, item := range s.pending { + if !changeAfterSnapshotFence(item.change, fence) { + s.queuedBytes -= item.bytes + continue + } + s.queue = append(s.queue, item) + } + s.pending = nil + s.snapshotting = false + s.mu.Unlock() + select { + case s.notify <- struct{}{}: + default: + } +} + +func changeAfterSnapshotFence(change config.Change, fence pglogrepl.LSN) bool { + if change.CommitLSN == "" { + return true + } + commit, err := pglogrepl.ParseLSN(change.CommitLSN) + if err != nil { + // A malformed cursor cannot be proven to be represented by the + // snapshot. Retain it rather than silently dropping a change. + return true + } + return commit > fence } func (s *sourceSubscription) matches(change config.Change) bool { @@ -278,6 +378,17 @@ func (s *sourceSubscription) matches(change config.Change) bool { return true } +func (s *sourceSubscription) matchesSnapshot(change config.Change) bool { + if len(s.tables) == 0 { + return true + } + if _, ok := s.tables[strings.ToLower(change.Relation)]; ok { + return true + } + _, ok := s.tables[strings.ToLower(change.Table)] + return ok +} + func filterSet(values []string) map[string]struct{} { if len(values) == 0 { return nil diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 8aa80de68..86ca612f2 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/jackc/pglogrepl" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" cdcapi "github.com/wippyai/runtime/api/service/cdc" @@ -41,7 +42,7 @@ func TestSourceSubscribePublishesMatchingChanges(t *testing.T) { } } -func TestSourceSubscribePreparesStartupSnapshotBeforeStart(t *testing.T) { +func TestSourceSubscribeAllowsOrdinaryPreStartStream(t *testing.T) { src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) stream, err := src.subscribe(context.Background(), cdcapi.StreamOptions{Buffer: 1}) require.NoError(t, err) @@ -49,7 +50,7 @@ func TestSourceSubscribePreparesStartupSnapshotBeforeStart(t *testing.T) { defer stream.Close() src.publishChange(context.Background(), cdcapi.Change{ - Op: "snapshot", + Op: "insert", Table: "accounts", After: map[string]any{"id": int64(1)}, Source: "test:cdc", @@ -57,10 +58,45 @@ func TestSourceSubscribePreparesStartupSnapshotBeforeStart(t *testing.T) { select { case got := <-stream.Changes(): - require.Equal(t, "snapshot", got.Op) + require.Equal(t, "insert", got.Op) require.Equal(t, "accounts", got.Table) case <-time.After(time.Second): - t.Fatal("pre-start subscription did not retain startup snapshot") + t.Fatal("pre-start subscription did not retain the ordinary stream") + } +} + +func TestSourceSnapshotDefaultRequiresRunningGeneration(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a", Snapshot: true}) + stream, err := src.subscribe(context.Background(), cdcapi.StreamOptions{}) + assert.ErrorIs(t, err, cdcapi.ErrSourceNotReady) + assert.Nil(t, stream) + assert.Nil(t, src.Subscribe(cdcapi.StreamOptions{})) +} + +func TestSnapshotHandoffWaitsForReplicationFence(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + done := make(chan struct{}) + src.mu.Lock() + src.done = done + src.streamPosition = 0 + src.mu.Unlock() + + fence, err := pglogrepl.ParseLSN("0/20") + require.NoError(t, err) + waited := make(chan error, 1) + go func() { waited <- src.waitStreamPosition(context.Background(), fence) }() + select { + case err := <-waited: + t.Fatalf("handoff released before replication reached fence: %v", err) + case <-time.After(20 * time.Millisecond): + } + + src.advanceStreamPosition(done, fence) + select { + case err := <-waited: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("handoff did not observe the replication fence") } } @@ -210,7 +246,7 @@ func TestSourceSubscriptionMaxBytesReleasesOnlyAfterDelivery(t *testing.T) { } changeBytes := cdcapi.EstimateChangeBytes(change) stream := src.newSubscription(cdcapi.StreamOptions{Buffer: 2, MaxBytes: changeBytes + 1}) - sub := stream.(*sourceSubscription) + sub := stream defer sub.Close() sub.send(context.Background(), change, cdcapi.EstimateChangeBytes(change)) @@ -266,3 +302,95 @@ func TestSourceSubscriptionMaxBytesOverflowIsIsolated(t *testing.T) { t.Fatal("independent source did not receive change") } } + +func TestSnapshotSubscriptionHandoffIsCommitLSNFenced(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true, Buffer: 8}) + defer sub.Close() + + before := cdcapi.Change{Op: "insert", Table: "users", CommitLSN: "0/10"} + after := cdcapi.Change{Op: "insert", Table: "users", CommitLSN: "0/30"} + sub.send(context.Background(), before, cdcapi.EstimateChangeBytes(before)) + sub.send(context.Background(), after, cdcapi.EstimateChangeBytes(after)) + select { + case got := <-sub.Changes(): + t.Fatalf("live change escaped before snapshot completion: %#v", got) + case <-time.After(20 * time.Millisecond): + } + + fence, err := pglogrepl.ParseLSN("0/20") + require.NoError(t, err) + snapshot := cdcapi.Change{ + Op: "snapshot", + Table: "users", + CommitLSN: fence.String(), + After: map[string]any{"id": int64(1)}, + } + require.NoError(t, sub.sendSnapshot(snapshot, cdcapi.EstimateChangeBytes(snapshot))) + sub.finishSnapshot(fence, nil) + + select { + case got := <-sub.Changes(): + require.Equal(t, "snapshot", got.Op) + case <-time.After(time.Second): + t.Fatal("snapshot row was not delivered") + } + select { + case got := <-sub.Changes(): + require.Equal(t, after.CommitLSN, got.CommitLSN) + case <-time.After(time.Second): + t.Fatal("post-fence live row was not delivered") + } +} + +func TestSnapshotSubscriptionBoundsPendingLiveChanges(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + change := cdcapi.Change{ + Op: "insert", + Table: "users", + After: map[string]any{"payload": []byte("large")}, + } + bytes := cdcapi.EstimateChangeBytes(change) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true, MaxBytes: bytes}) + defer sub.Close() + sub.send(context.Background(), change, bytes) + sub.send(context.Background(), change, bytes) + assert.ErrorIs(t, sub.Err(), errSubscriberOverflow) + select { + case _, ok := <-sub.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("overflowed snapshot stream did not close") + } +} + +func TestSnapshotSubscriptionsDoNotSharePendingState(t *testing.T) { + firstSource := NewSource(SourceOptions{Name: "db-one", Slot: "slot_one"}) + secondSource := NewSource(SourceOptions{Name: "db-two", Slot: "slot_two"}) + first := firstSource.newSubscription(cdcapi.StreamOptions{Snapshot: true, Buffer: 4}) + second := secondSource.newSubscription(cdcapi.StreamOptions{Snapshot: true, Buffer: 4}) + defer first.Close() + defer second.Close() + + firstChange := cdcapi.Change{Op: "insert", Table: "first", CommitLSN: "0/30"} + secondChange := cdcapi.Change{Op: "insert", Table: "second", CommitLSN: "0/30"} + first.send(context.Background(), firstChange, cdcapi.EstimateChangeBytes(firstChange)) + second.send(context.Background(), secondChange, cdcapi.EstimateChangeBytes(secondChange)) + fence, err := pglogrepl.ParseLSN("0/20") + require.NoError(t, err) + first.finishSnapshot(fence, nil) + second.finishSnapshot(fence, nil) + + select { + case got := <-first.Changes(): + require.Equal(t, "first", got.Table) + case <-time.After(time.Second): + t.Fatal("first source did not deliver its pending change") + } + select { + case got := <-second.Changes(): + require.Equal(t, "second", got.Table) + case <-time.After(time.Second): + t.Fatal("second source did not deliver its pending change") + } +} From 0aac94507689da3d03c46617e7d79f86d27ba85c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 04:43:40 -0400 Subject: [PATCH 46/47] fix(cdc/postgres): join snapshot workers safely --- service/cdc/postgres/service.go | 70 ++++++++++++++---- service/cdc/postgres/stream.go | 75 ++++++++++++++----- service/cdc/postgres/stream_test.go | 108 ++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 31 deletions(-) diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index a3c8107d9..35748167b 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -83,6 +83,7 @@ type Source struct { done chan struct{} subs map[uint64]*sourceSubscription streamNotify chan struct{} + snapshotGate chan struct{} // one temporary logical snapshot per source replDSN string adminDSN string name string @@ -187,6 +188,7 @@ func NewSource(opts SourceOptions) *Source { maxInflightChanges: limits.maxInflightChanges, maxInflightBytes: limits.maxInflightBytes, streamNotify: make(chan struct{}), + snapshotGate: make(chan struct{}, 1), } } @@ -350,9 +352,6 @@ func (s *Source) Stop(ctx context.Context) error { if s.state == sourceStopped { drop := s.dropSlot.Load() s.mu.Unlock() - if err := s.waitSnapshots(ctx); err != nil { - return err - } if drop { return s.dropSlotAndCheckpoint(ctx) } @@ -360,13 +359,21 @@ func (s *Source) Stop(ctx context.Context) error { } if s.state == sourceNew || s.state == sourceFailed { if s.state == sourceNew { - s.state = sourceStopped + // Keep the generation stopping until all snapshot workers have + // joined. This prevents a concurrent Start from resetting state + // while a worker can still call WaitGroup.Done. + s.state = sourceStopping s.cancel = nil s.mu.Unlock() s.closeSubscriptions() if err := s.waitSnapshots(ctx); err != nil { return err } + s.mu.Lock() + if s.state == sourceStopping { + s.state = sourceStopped + } + s.mu.Unlock() if s.dropSlot.Load() { return s.dropSlotAndCheckpoint(ctx) } @@ -392,10 +399,6 @@ func (s *Source) Stop(ctx context.Context) error { if done != nil { select { case <-done: - s.mu.Lock() - s.state = sourceStopped - s.cancel = nil - s.mu.Unlock() case <-ctx.Done(): return ctx.Err() } @@ -410,6 +413,12 @@ func (s *Source) Stop(ctx context.Context) error { if err := s.waitSnapshots(ctx); err != nil { return err } + s.mu.Lock() + if s.state == sourceStopping { + s.state = sourceStopped + s.cancel = nil + } + s.mu.Unlock() if s.dropSlot.Load() { return s.dropSlotAndCheckpoint(ctx) @@ -420,8 +429,15 @@ func (s *Source) Stop(ctx context.Context) error { func (s *Source) startSnapshot(ctx context.Context, sub *sourceSubscription) { snapshotCtx, cancel := context.WithCancel(ctx) s.snapshotWG.Add(1) + snapshotDone := make(chan struct{}) + if !sub.registerSnapshot(cancel, snapshotDone) { + s.snapshotWG.Done() + cancel() + return + } go func() { defer s.snapshotWG.Done() + defer sub.finishSnapshotWorker() watchDone := make(chan struct{}) go func() { select { @@ -433,6 +449,12 @@ func (s *Source) startSnapshot(ctx context.Context, sub *sourceSubscription) { }() defer close(watchDone) + if err := s.acquireSnapshot(snapshotCtx); err != nil { + sub.finishSnapshot(0, err) + return + } + defer s.releaseSnapshot() + fence, err := s.snapshotCurrentTo(snapshotCtx, sub) if err != nil { sub.finishSnapshot(0, err) @@ -442,6 +464,19 @@ func (s *Source) startSnapshot(ctx context.Context, sub *sourceSubscription) { }() } +func (s *Source) acquireSnapshot(ctx context.Context) error { + select { + case s.snapshotGate <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *Source) releaseSnapshot() { + <-s.snapshotGate +} + func (s *Source) waitSnapshots(ctx context.Context) error { done := make(chan struct{}) go func() { @@ -457,10 +492,11 @@ func (s *Source) waitSnapshots(ctx context.Context) error { } // advanceStreamPosition publishes the replication receive watermark after a -// complete XLogData message has been decoded and emitted. Snapshot handoff -// waits for this watermark before releasing its pending live queue, so a -// change at or before the exported snapshot fence cannot arrive late and be -// duplicated after the handoff. +// complete XLogData message has been decoded and emitted, or after a server +// keepalive reports its WAL end. Snapshot handoff waits for this watermark +// before releasing its pending live queue, so a change at or before the +// exported snapshot fence cannot arrive late and be duplicated after the +// handoff. It never updates the transaction-safe checkpoint. func (s *Source) advanceStreamPosition(done chan struct{}, position pglogrepl.LSN) { s.mu.Lock() defer s.mu.Unlock() @@ -472,6 +508,10 @@ func (s *Source) advanceStreamPosition(done chan struct{}, position pglogrepl.LS s.streamNotify = make(chan struct{}) } +func (s *Source) observeKeepalive(done chan struct{}, keepalive pglogrepl.PrimaryKeepaliveMessage) { + s.advanceStreamPosition(done, keepalive.ServerWALEnd) +} + func (s *Source) waitStreamPosition(ctx context.Context, fence pglogrepl.LSN) error { for { s.mu.Lock() @@ -506,8 +546,6 @@ func (s *Source) run( current := s.done == done if current { switch s.state { - case sourceStopping: - s.state = sourceStopped case sourceRunning, sourceStarting: s.state = sourceFailed } @@ -665,6 +703,10 @@ func (s *Source) run( s.fail(ctx, status, kaErr) return } + // ServerWALEnd is the receive watermark used by an in-flight + // snapshot handoff. It is deliberately independent from safePos: + // keepalives must not advance the transaction-safe checkpoint. + s.observeKeepalive(done, ka) if ka.ReplyRequested { if err := saveSafe(); err != nil { s.fail(ctx, status, err) diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 24ba3144a..2f7c4c334 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -24,23 +24,25 @@ var errSubscriberOverflow = errors.New("postgres cdc subscriber backlog overflow var errSnapshotNotActive = errors.New("postgres cdc snapshot is no longer active") type sourceSubscription struct { - err error - tables map[string]struct{} - source *Source - done chan struct{} - notify chan struct{} - relayDone chan struct{} - ops map[string]struct{} - out chan config.Change - queue []queuedChange - pending []queuedChange - maxBytes int64 - maxChanges int - id uint64 - queuedBytes int64 - mu sync.Mutex - closed bool - snapshotting bool + err error + tables map[string]struct{} + source *Source + done chan struct{} + notify chan struct{} + relayDone chan struct{} + snapshotDone chan struct{} + snapshotCancel context.CancelFunc + ops map[string]struct{} + out chan config.Change + queue []queuedChange + pending []queuedChange + maxBytes int64 + maxChanges int + id uint64 + queuedBytes int64 + mu sync.Mutex + closed bool + snapshotting bool } type queuedChange struct { @@ -176,6 +178,9 @@ func (s *Source) closeSubscriptionsWithError(err error) { for _, sub := range subs { sub.closeWithError(err) } + for _, sub := range subs { + sub.waitSnapshot() + } for _, sub := range subs { sub.waitRelay() } @@ -187,6 +192,7 @@ func (s *sourceSubscription) Changes() <-chan config.Change { func (s *sourceSubscription) Close() { s.closeWithError(nil) + s.waitSnapshot() s.waitRelay() } @@ -200,12 +206,47 @@ func (s *sourceSubscription) Err() error { func (s *sourceSubscription) closeWithError(err error) { s.mu.Lock() parent, id := s.closeLocked(err) + cancel := s.snapshotCancel s.mu.Unlock() + if cancel != nil { + cancel() + } if parent != nil { parent.removeSubscription(id) } } +func (s *sourceSubscription) registerSnapshot(cancel context.CancelFunc, done chan struct{}) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return false + } + s.snapshotCancel = cancel + s.snapshotDone = done + return true +} + +func (s *sourceSubscription) finishSnapshotWorker() { + s.mu.Lock() + done := s.snapshotDone + s.snapshotDone = nil + s.snapshotCancel = nil + s.mu.Unlock() + if done != nil { + close(done) + } +} + +func (s *sourceSubscription) waitSnapshot() { + s.mu.Lock() + done := s.snapshotDone + s.mu.Unlock() + if done != nil { + <-done + } +} + func (s *sourceSubscription) waitRelay() { <-s.relayDone } diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 86ca612f2..18f23c160 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -100,6 +100,114 @@ func TestSnapshotHandoffWaitsForReplicationFence(t *testing.T) { } } +func TestIdleKeepaliveAdvancesSnapshotWatermark(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + done := make(chan struct{}) + src.mu.Lock() + src.done = done + src.streamPosition = 0 + src.mu.Unlock() + + fence, err := pglogrepl.ParseLSN("0/40") + require.NoError(t, err) + waited := make(chan error, 1) + go func() { waited <- src.waitStreamPosition(context.Background(), fence) }() + select { + case err := <-waited: + t.Fatalf("idle snapshot released before keepalive: %v", err) + case <-time.After(20 * time.Millisecond): + } + + src.observeKeepalive(done, pglogrepl.PrimaryKeepaliveMessage{ServerWALEnd: fence}) + select { + case err := <-waited: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("idle keepalive did not advance snapshot watermark") + } +} + +func TestSnapshotGateSerializesPerSourceAndIsolatesSources(t *testing.T) { + first := NewSource(SourceOptions{Name: "db-one", Slot: "slot_one"}) + second := NewSource(SourceOptions{Name: "db-two", Slot: "slot_two"}) + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, first.acquireSnapshot(ctx)) + + waiting := make(chan error, 1) + go func() { waiting <- first.acquireSnapshot(ctx) }() + select { + case err := <-waiting: + t.Fatalf("same-source snapshot gate was not serialized: %v", err) + case <-time.After(20 * time.Millisecond): + } + require.NoError(t, second.acquireSnapshot(context.Background())) + second.releaseSnapshot() + cancel() + select { + case err := <-waiting: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("cancelled snapshot did not leave the per-source gate") + } + first.releaseSnapshot() +} + +func TestSubscriptionCloseWaitsForSnapshotWorker(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true}) + cancelled := make(chan struct{}) + workerDone := make(chan struct{}) + require.True(t, sub.registerSnapshot(func() { close(cancelled) }, workerDone)) + go func() { + <-cancelled + sub.finishSnapshotWorker() + }() + + sub.Close() + select { + case <-workerDone: + default: + t.Fatal("subscription Close returned before its snapshot worker joined") + } +} + +func TestStopJoinsSnapshotWorkerBeforeGenerationReset(t *testing.T) { + src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) + sub := src.newSubscription(cdcapi.StreamOptions{Snapshot: true}) + cancelled := make(chan struct{}) + release := make(chan struct{}) + workerDone := make(chan struct{}) + require.True(t, sub.registerSnapshot(func() { close(cancelled) }, workerDone)) + src.snapshotWG.Add(1) + go func() { + <-cancelled + <-release + sub.finishSnapshotWorker() + src.snapshotWG.Done() + }() + + stopped := make(chan error, 1) + go func() { stopped <- src.Stop(context.Background()) }() + select { + case err := <-stopped: + t.Fatalf("Stop returned before snapshot worker joined: %v", err) + case <-time.After(20 * time.Millisecond): + } + src.mu.Lock() + assert.Equal(t, sourceStopping, src.state) + src.mu.Unlock() + close(release) + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("Stop did not complete after snapshot worker release") + } + src.mu.Lock() + assert.Equal(t, sourceStopped, src.state) + src.mu.Unlock() +} + func TestSourceSubscribeFiltersChanges(t *testing.T) { src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) stream := src.Subscribe(cdcapi.StreamOptions{ From 52d700ecca83667a280e4c7a3f5dbd4b0286ad6c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 9 Aug 2026 04:47:57 -0400 Subject: [PATCH 47/47] fix(cdc/postgres): isolate generation subscriber cleanup --- service/cdc/postgres/service.go | 30 ++++++++-------- service/cdc/postgres/stream.go | 23 ++++++++++++ service/cdc/postgres/stream_test.go | 56 +++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 14 deletions(-) diff --git a/service/cdc/postgres/service.go b/service/cdc/postgres/service.go index 35748167b..48b359fe3 100644 --- a/service/cdc/postgres/service.go +++ b/service/cdc/postgres/service.go @@ -529,6 +529,20 @@ func (s *Source) waitStreamPosition(ctx context.Context, fence pglogrepl.LSN) er } } +func (s *Source) finishRunGeneration(done chan struct{}) (bool, []*sourceSubscription) { + s.mu.Lock() + defer s.mu.Unlock() + if s.done != done { + return false, nil + } + switch s.state { + case sourceRunning, sourceStarting: + s.state = sourceFailed + } + s.cancel = nil + return true, s.detachSubscriptionsLocked() +} + func (s *Source) run( ctx context.Context, conn *pgconn.PgConn, @@ -542,21 +556,9 @@ func (s *Source) run( done chan struct{}, ) { defer func() { - s.mu.Lock() - current := s.done == done + current, subs := s.finishRunGeneration(done) if current { - switch s.state { - case sourceRunning, sourceStarting: - s.state = sourceFailed - } - s.cancel = nil - } - s.mu.Unlock() - // A failed generation can finish after a supervisor has already - // started its replacement. Only the active generation owns the - // subscription set; an old run must never prune new subscribers. - if current { - s.closeSubscriptions() + s.closeDetachedSubscriptions(subs, nil) } close(done) }() diff --git a/service/cdc/postgres/stream.go b/service/cdc/postgres/stream.go index 2f7c4c334..4dc938812 100644 --- a/service/cdc/postgres/stream.go +++ b/service/cdc/postgres/stream.go @@ -167,6 +167,26 @@ func (s *Source) closeSubscriptions() { } func (s *Source) closeSubscriptionsWithError(err error) { + subs := s.detachSubscriptions() + s.closeDetachedSubscriptions(subs, err) +} + +// detachSubscriptionsLocked must be called while s.mu is held. Subscribe +// takes s.mu before subMu, so this ordering makes generation cleanup atomic +// with the lifecycle transition and prevents an old run from taking a new +// generation's subscribers. +func (s *Source) detachSubscriptionsLocked() []*sourceSubscription { + s.subMu.Lock() + subs := make([]*sourceSubscription, 0, len(s.subs)) + for id, sub := range s.subs { + subs = append(subs, sub) + delete(s.subs, id) + } + s.subMu.Unlock() + return subs +} + +func (s *Source) detachSubscriptions() []*sourceSubscription { s.subMu.Lock() subs := make([]*sourceSubscription, 0, len(s.subs)) for id, sub := range s.subs { @@ -174,7 +194,10 @@ func (s *Source) closeSubscriptionsWithError(err error) { delete(s.subs, id) } s.subMu.Unlock() + return subs +} +func (s *Source) closeDetachedSubscriptions(subs []*sourceSubscription, err error) { for _, sub := range subs { sub.closeWithError(err) } diff --git a/service/cdc/postgres/stream_test.go b/service/cdc/postgres/stream_test.go index 18f23c160..5f512e8d9 100644 --- a/service/cdc/postgres/stream_test.go +++ b/service/cdc/postgres/stream_test.go @@ -208,6 +208,62 @@ func TestStopJoinsSnapshotWorkerBeforeGenerationReset(t *testing.T) { src.mu.Unlock() } +func TestOldGenerationCleanupDoesNotCloseReplacementSubscribers(t *testing.T) { + src := NewSource(SourceOptions{Name: "db-one", Slot: "slot_one"}) + oldDone := make(chan struct{}) + src.mu.Lock() + src.state = sourceRunning + src.done = oldDone + src.mu.Unlock() + oldSub := src.Subscribe(cdcapi.StreamOptions{Buffer: 2}) + require.NotNil(t, oldSub) + + current, detached := src.finishRunGeneration(oldDone) + require.True(t, current) + src.closeDetachedSubscriptions(detached, nil) + close(oldDone) + + newDone := make(chan struct{}) + src.mu.Lock() + src.state = sourceRunning + src.done = newDone + src.mu.Unlock() + newSub := src.Subscribe(cdcapi.StreamOptions{Buffer: 2}) + require.NotNil(t, newSub) + defer newSub.Close() + + select { + case _, ok := <-oldSub.Changes(): + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("old generation subscriber was not closed") + } + src.publishChange(context.Background(), cdcapi.Change{Op: "insert", Table: "users"}) + select { + case change := <-newSub.Changes(): + require.Equal(t, "insert", change.Op) + case <-time.After(time.Second): + t.Fatal("replacement subscriber was closed by old generation cleanup") + } + + other := NewSource(SourceOptions{Name: "db-two", Slot: "slot_two"}) + otherDone := make(chan struct{}) + other.mu.Lock() + other.state = sourceRunning + other.done = otherDone + other.mu.Unlock() + otherSub := other.Subscribe(cdcapi.StreamOptions{Buffer: 2}) + require.NotNil(t, otherSub) + defer otherSub.Close() + other.publishChange(context.Background(), cdcapi.Change{Op: "insert", Table: "isolated"}) + select { + case change := <-otherSub.Changes(): + require.Equal(t, "isolated", change.Table) + case <-time.After(time.Second): + t.Fatal("independent source subscriber was affected by generation cleanup") + } +} + func TestSourceSubscribeFiltersChanges(t *testing.T) { src := NewSource(SourceOptions{Name: "test:cdc", Slot: "slot_a"}) stream := src.Subscribe(cdcapi.StreamOptions{