port(phase 3): relay operations and database pressure from block/buzz

.env.example

@@ -38,6 +38,21 @@ REDIS_URL=redis://localhost:6379
 # READ_DATABASE_URL is set, reader (default 50).
 # BUZZ_DB_POOL_SIZE=50
 
+# Writer-session Postgres timeouts for buzz-db-backed pools and the relay audit
+# pool, all in milliseconds; 0 disables. The separately deployed push gateway
+# owns its own database and session policy and does not consume these knobs.
+# lock_timeout: fail a statement that waits this long on any lock instead of
+# parking behind a wedged holder (default 5000).
+# BUZZ_DB_LOCK_TIMEOUT_MS=5000
+# idle_in_transaction_session_timeout: reap sessions idle inside an open
+# transaction — bounds how long a wedged client can hold locks (default 60000).
+# BUZZ_DB_IDLE_TXN_TIMEOUT_MS=60000
+# statement_timeout: cap any single statement's runtime. Off by default —
+# startup migrations/backfills legitimately run long statements. Warning: a
+# pathologically low value (e.g. 1) also times out connection setup and can
+# prevent any DB connection from establishing.
+# BUZZ_DB_STATEMENT_TIMEOUT_MS=0
+
 # -----------------------------------------------------------------------------
 # Typesense (search)
 # -----------------------------------------------------------------------------

.github/workflows/ci.yml

@@ -83,7 +83,7 @@ jobs:
               # schema and the durable relay key e2e_interrupts requires.
               - 'scripts/start-relay-for-tests.sh'
               - 'scripts/create-required-extensions.sql'
-              - 'scripts/attach-schema-partitions.sql'
+              - 'scripts/reconcile-schema-after-pgschema.sql'
               - 'justfile'
             # Never put a negated pattern (`!foo/**`) in one of these lists.
             # dorny/paths-filter ORs the patterns in a rule, and picomatch
@@ -179,7 +179,7 @@ jobs:
               - 'pnpm-lock.yaml'
               - 'scripts/start-relay-for-tests.sh'
               - 'scripts/create-required-extensions.sql'
-              - 'scripts/attach-schema-partitions.sql'
+              - 'scripts/reconcile-schema-after-pgschema.sql'
               - '.github/workflows/ci.yml'
               - 'scripts/ci-activate-hermit.sh'
               - 'scripts/ci-pnpm-store-path.sh'
@@ -682,8 +682,12 @@ jobs:
             --cargo-profile ci \
             -p buzz-db \
             -p buzz-relay \
+            -p buzz-search \
             -p buzz-test-client \
             --lib \
+            --bin buzz-relay \
+            --test boot_lifecycle \
+            --test fts_integration \
             --test e2e_event_reminder \
             --test interrupt_gate \
             --test interrupt_runtime \
@@ -840,14 +844,14 @@ jobs:
           PGSCHEMA_PLAN_USER: buzz
           PGSCHEMA_PLAN_PASSWORD: buzz_dev
         run: |
-          ./bin/pgschema apply --file schema/schema.sql --auto-approve
           # pgschema does not manage extensions, so schema.sql's CREATE
           # EXTENSION line is silently ignored and the relay's digest()-based
           # queries fail. See scripts/create-required-extensions.sql.
+          ./bin/pgschema apply --file schema/schema.sql --auto-approve
           docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \
             psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/create-required-extensions.sql
           docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \
-            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql
+            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql
           docker exec -e PGPASSWORD=buzz_dev buzz-postgres \
             psql -U buzz -d buzz -qtA -c "
           INSERT INTO communities (id, host)
@@ -1118,6 +1122,14 @@ jobs:
         with:
           name: desktop-e2e-relay
           path: target/ci
+      - name: Restore the executable bit on the downloaded binaries
+        # actions/upload-artifact does not preserve file modes, so both binaries
+        # arrive 0644. Every "Start relay" step in this workflow already chmods
+        # for that reason, but it runs near the end of the job: the nextest
+        # steps below come first, and `boot_lifecycle` spawns the relay through
+        # `env!("CARGO_BIN_EXE_buzz-relay")`, which resolves to this exact file.
+        # Without this it fails with EACCES (os error 13) before any assertion.
+        run: chmod +x ./target/ci/buzz-relay ./target/ci/git-credential-nostr
       - name: Prefetch pgschema
         # The bin/pgschema stub downloads on first use, so a 504 from GitHub
         # Releases fails the schema step instead of reporting a fetch problem.
@@ -1148,14 +1160,14 @@ jobs:
           PGSCHEMA_PLAN_USER: buzz
           PGSCHEMA_PLAN_PASSWORD: buzz_dev
         run: |
-          ./bin/pgschema apply --file schema/schema.sql --auto-approve
           # pgschema does not manage extensions, so schema.sql's CREATE
           # EXTENSION line is silently ignored and the relay's digest()-based
           # queries fail. See scripts/create-required-extensions.sql.
+          ./bin/pgschema apply --file schema/schema.sql --auto-approve
           docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \
             psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/create-required-extensions.sql
           docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \
-            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql
+            psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql
           docker exec -e PGPASSWORD=buzz_dev buzz-postgres \
             psql -U buzz -d buzz -qtA -c "
           INSERT INTO communities (id, host)
@@ -1175,6 +1187,74 @@ jobs:
         env:
           DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
           TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
+      - name: Database pressure observability PostgreSQL tests
+        # Explicit pool acquisition, advisory-lock and operation-label metrics
+        # require real Postgres and are ignored by the infrastructure-free
+        # unit-test job.
+        #
+        # Match the whole `postgres_tests` module rather than naming tests one
+        # by one. #7195 moved these under that module and added three more, so
+        # the old two-name filter matched nothing: nextest reported "Starting 0
+        # tests" and exited 4, which is what turned this step red. A module
+        # match cannot go stale the same way when a test is added or renamed.
+        run: |
+          filter='package(buzz-db) and test(/runtime::observability::tests::postgres_tests::/)'
+          cargo nextest run \
+            --archive-file target/ci/backend-integration-tests.tar.zst \
+            -E "${filter}" \
+            --run-ignored ignored-only
+        env:
+          DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
+          TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
+          # This job provisions Postgres from schema/schema.sql via pgschema, not
+          # by running migrations, so `serving_write_gate_...` must skip its own
+          # `db.migrate()`. Without this it fails on 42710, `type "channel_type"
+          # already exists`, from migration 0001 re-creating desired-state types.
+          BUZZ_TEST_SCHEMA_MODE: desired
+      - name: Full-text search policy
+        # The FTS suite is the only place the brownfield search-policy
+        # migrations are executed against a real database; nothing else in CI
+        # builds buzz-search's integration tests.
+        run: |
+          cargo nextest run \
+            --archive-file target/ci/backend-integration-tests.tar.zst \
+            -E 'package(buzz-search) and binary(fts_integration)' \
+            --run-ignored ignored-only
+        env:
+          BUZZ_TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
+      - name: Startup lifecycle evidence
+        # Spawns the relay binary against a dead TCP listener standing in for
+        # Postgres, so it needs no database; it proves the pre-runtime boot
+        # phases emit exactly one bounded terminal each.
+        run: |
+          cargo nextest run \
+            --archive-file target/ci/backend-integration-tests.tar.zst \
+            -E 'package(buzz-relay) and binary(boot_lifecycle)'
+      - name: Writer session timeout guardrails
+        run: |
+          cargo nextest run \
+            --archive-file target/ci/backend-integration-tests.tar.zst \
+            -E 'package(buzz-db) and test(session_timeouts_install_through_db_new_and_bound_lock_waits)' \
+            --run-ignored ignored-only
+        env:
+          DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
+          TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
+      - name: Audit writer session timeout guardrails
+        run: |
+          cargo nextest run \
+            --archive-file target/ci/backend-integration-tests.tar.zst \
+            -E 'package(buzz-relay) and test(audit_writer_pool_installs_timeouts_and_bounds_advisory_lock_waits)' \
+            --run-ignored ignored-only
+        env:
+          DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
+      - name: Audit worker lock-timeout recovery
+        run: |
+          cargo nextest run \
+            --archive-file target/ci/backend-integration-tests.tar.zst \
+            -E 'package(buzz-relay) and test(audit_worker_retries_lock_timeout_until_original_entry_is_appended_once)' \
+            --run-ignored ignored-only
+        env:
+          DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz
       - name: Start relay
         run: |
           chmod +x ./target/ci/buzz-relay

TESTING.md

@@ -507,9 +507,49 @@ buzz messages thread --channel "$CHANNEL" --event "$EVENT_ID" | jq .
 
 A successful run prints `{"event_id":"…","accepted":true,"message":""}` for
 the send, and the message body in the `get` output. `thread` returns `[]`
-for a leaf message — populated only after a reply comes in (see §5).
+for a leaf message — populated only after a reply comes in (see §6).
 
-### 5. Going deeper
+### 5. Verify a roster beyond 1,000 members
+
+Use the focused live-relay script when changing channel membership, discovery,
+or reconciliation. It proves the three boundaries that DB-only tests cannot:
+the relay-served kind 39002 includes a member at roster position 1,501, that
+identity can publish a channel message, and targeted reconciliation preserves
+its discoverability.
+
+Run this only against an isolated local database. The script inserts fixture
+members directly, then drives discovery and messaging through the release CLI
+and relay. Keep the release relay from step 3 running and use its configured
+relay key for authoritative replacement:
+
+```bash
+export PATH="$PWD/target/release:$PATH"
+export DATABASE_URL="postgres://buzz:buzz_dev@localhost:5432/buzz_roster_e2e"
+export BUZZ_RELAY_URL="http://localhost:3030"  # match the relay from step 3
+export RELAY_URL="ws://localhost:3030"
+export BUZZ_RELAY_PRIVATE_KEY="<same key used by buzz-relay>"
+
+scripts/e2e-large-channel-roster.sh
+```
+
+Success is directly observable as four `PASS` lines. The first and fourth
+include a member count greater than 1,000 and the same late-member pubkey; the
+second includes the accepted kind 9 event ID, and the third proves targeted
+repair left kind 39000/39001 IDs and tags unchanged:
+
+```text
+PASS discovery-before-republish channel=<uuid> members=1502 late_pubkey=<hex>
+PASS late-member-action event_id=<hex>
+PASS targeted-repair-preserves-metadata-and-admin-events channel=<uuid>
+PASS discovery-after-republish channel=<uuid> members=1502 late_pubkey=<hex>
+```
+
+The script refuses debug binaries and refuses a `buzz` or `buzz-admin` resolved
+outside this checkout's `target/release`. It also requires the targeted admin
+operation to use `BUZZ_RELAY_PRIVATE_KEY`; never substitute an ephemeral signer
+for an authoritative replacement.
+
+### 6. Going deeper
 
 For full coverage of every CLI command (54 subcommands across 12 groups),
 follow [`crates/buzz-cli/TESTING.md`](crates/buzz-cli/TESTING.md).
@@ -652,7 +692,7 @@ CLI-side, only two matter for testing:
 | Symptom | Cause | Fix |
 |---------|-------|-----|
 | `relay error 500` or `400: restricted: not a channel member` after a code change | Stale binary | Rebuild and re-export `PATH`; or `cargo run` directly |
-| `Address already in use` on relay start (os error 48 on macOS, 98 on Linux) | Another relay (or stale process) holding `:3000` / `:8080` / `:9102` (or your override ports) | The panic line names the failing port — read it first. Then `lsof -iTCP:3000,8080,9102 -sTCP:LISTEN` (or your override equivalents). Kill the offender (`pkill -f buzz-relay`) or use the port-override block in step 3. If you already overrode and *still* collide, a prior reviewer left a relay running on the same alt ports — kill it or pick fresh ports |
+| `Address already in use` on relay start (os error 48 on macOS, 98 on Linux) | Another relay (or stale process) holding `:3000` / `:8080` / `:9102` (or your override ports) | Metrics-listener failures emit a `metrics_bind` lifecycle terminal with reason `bind`. Check the configured ports with `lsof -iTCP:3000,8080,9102 -sTCP:LISTEN` (or your override equivalents). Kill the offender (`pkill -f buzz-relay`) or use the port-override block in step 3. If you already overrode and *still* collide, a prior reviewer left a relay running on the same alt ports — kill it or pick fresh ports |
 | `auth_error: BUZZ_PRIVATE_KEY is required` | Env not exported into the CLI's shell | `export BUZZ_PRIVATE_KEY=...` (or pass `--private-key`) |
 | `auth_error: BUZZ_AUTH_TAG verification failed … signature verification failed` | A stale `BUZZ_AUTH_TAG` inherited from a parent shell. The local dev relay rejects it. | `unset BUZZ_AUTH_TAG` (see the scrub block in step 1) |
 | `auth-required: verification failed` on a closed relay | NIP-OA attestation needed | Set `BUZZ_AUTH_TAG` to the owner-issued JSON, or relax `BUZZ_REQUIRE_RELAY_MEMBERSHIP` |