diff --git a/.env.example b/.env.example index e636b0c34..f79ad45ed 100644 --- a/.env.example +++ b/.env.example @@ -38,6 +38,21 @@ REDIS_URL=redis://localhost:6379 # READ_DATABASE_URL is set, reader (default 50). # BUZZ_DB_POOL_SIZE=50 +# Writer-session Postgres timeouts for buzz-db-backed pools and the relay audit +# pool, all in milliseconds; 0 disables. The separately deployed push gateway +# owns its own database and session policy and does not consume these knobs. +# lock_timeout: fail a statement that waits this long on any lock instead of +# parking behind a wedged holder (default 5000). +# BUZZ_DB_LOCK_TIMEOUT_MS=5000 +# idle_in_transaction_session_timeout: reap sessions idle inside an open +# transaction — bounds how long a wedged client can hold locks (default 60000). +# BUZZ_DB_IDLE_TXN_TIMEOUT_MS=60000 +# statement_timeout: cap any single statement's runtime. Off by default — +# startup migrations/backfills legitimately run long statements. Warning: a +# pathologically low value (e.g. 1) also times out connection setup and can +# prevent any DB connection from establishing. +# BUZZ_DB_STATEMENT_TIMEOUT_MS=0 + # ----------------------------------------------------------------------------- # Typesense (search) # ----------------------------------------------------------------------------- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bf8cec7d..1ef9eed5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: # schema and the durable relay key e2e_interrupts requires. - 'scripts/start-relay-for-tests.sh' - 'scripts/create-required-extensions.sql' - - 'scripts/attach-schema-partitions.sql' + - 'scripts/reconcile-schema-after-pgschema.sql' - 'justfile' # Never put a negated pattern (`!foo/**`) in one of these lists. # dorny/paths-filter ORs the patterns in a rule, and picomatch @@ -179,7 +179,7 @@ jobs: - 'pnpm-lock.yaml' - 'scripts/start-relay-for-tests.sh' - 'scripts/create-required-extensions.sql' - - 'scripts/attach-schema-partitions.sql' + - 'scripts/reconcile-schema-after-pgschema.sql' - '.github/workflows/ci.yml' - 'scripts/ci-activate-hermit.sh' - 'scripts/ci-pnpm-store-path.sh' @@ -682,8 +682,12 @@ jobs: --cargo-profile ci \ -p buzz-db \ -p buzz-relay \ + -p buzz-search \ -p buzz-test-client \ --lib \ + --bin buzz-relay \ + --test boot_lifecycle \ + --test fts_integration \ --test e2e_event_reminder \ --test interrupt_gate \ --test interrupt_runtime \ @@ -840,14 +844,14 @@ jobs: PGSCHEMA_PLAN_USER: buzz PGSCHEMA_PLAN_PASSWORD: buzz_dev run: | - ./bin/pgschema apply --file schema/schema.sql --auto-approve # pgschema does not manage extensions, so schema.sql's CREATE # EXTENSION line is silently ignored and the relay's digest()-based # queries fail. See scripts/create-required-extensions.sql. + ./bin/pgschema apply --file schema/schema.sql --auto-approve docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \ psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/create-required-extensions.sql docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \ - psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql + psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql docker exec -e PGPASSWORD=buzz_dev buzz-postgres \ psql -U buzz -d buzz -qtA -c " INSERT INTO communities (id, host) @@ -1118,6 +1122,14 @@ jobs: with: name: desktop-e2e-relay path: target/ci + - name: Restore the executable bit on the downloaded binaries + # actions/upload-artifact does not preserve file modes, so both binaries + # arrive 0644. Every "Start relay" step in this workflow already chmods + # for that reason, but it runs near the end of the job: the nextest + # steps below come first, and `boot_lifecycle` spawns the relay through + # `env!("CARGO_BIN_EXE_buzz-relay")`, which resolves to this exact file. + # Without this it fails with EACCES (os error 13) before any assertion. + run: chmod +x ./target/ci/buzz-relay ./target/ci/git-credential-nostr - name: Prefetch pgschema # The bin/pgschema stub downloads on first use, so a 504 from GitHub # Releases fails the schema step instead of reporting a fetch problem. @@ -1148,14 +1160,14 @@ jobs: PGSCHEMA_PLAN_USER: buzz PGSCHEMA_PLAN_PASSWORD: buzz_dev run: | - ./bin/pgschema apply --file schema/schema.sql --auto-approve # pgschema does not manage extensions, so schema.sql's CREATE # EXTENSION line is silently ignored and the relay's digest()-based # queries fail. See scripts/create-required-extensions.sql. + ./bin/pgschema apply --file schema/schema.sql --auto-approve docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \ psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/create-required-extensions.sql docker exec -i -e PGPASSWORD=buzz_dev buzz-postgres \ - psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql + psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql docker exec -e PGPASSWORD=buzz_dev buzz-postgres \ psql -U buzz -d buzz -qtA -c " INSERT INTO communities (id, host) @@ -1175,6 +1187,74 @@ jobs: env: DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + - name: Database pressure observability PostgreSQL tests + # Explicit pool acquisition, advisory-lock and operation-label metrics + # require real Postgres and are ignored by the infrastructure-free + # unit-test job. + # + # Match the whole `postgres_tests` module rather than naming tests one + # by one. #7195 moved these under that module and added three more, so + # the old two-name filter matched nothing: nextest reported "Starting 0 + # tests" and exited 4, which is what turned this step red. A module + # match cannot go stale the same way when a test is added or renamed. + run: | + filter='package(buzz-db) and test(/runtime::observability::tests::postgres_tests::/)' + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E "${filter}" \ + --run-ignored ignored-only + env: + DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + # This job provisions Postgres from schema/schema.sql via pgschema, not + # by running migrations, so `serving_write_gate_...` must skip its own + # `db.migrate()`. Without this it fails on 42710, `type "channel_type" + # already exists`, from migration 0001 re-creating desired-state types. + BUZZ_TEST_SCHEMA_MODE: desired + - name: Full-text search policy + # The FTS suite is the only place the brownfield search-policy + # migrations are executed against a real database; nothing else in CI + # builds buzz-search's integration tests. + run: | + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E 'package(buzz-search) and binary(fts_integration)' \ + --run-ignored ignored-only + env: + BUZZ_TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + - name: Startup lifecycle evidence + # Spawns the relay binary against a dead TCP listener standing in for + # Postgres, so it needs no database; it proves the pre-runtime boot + # phases emit exactly one bounded terminal each. + run: | + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E 'package(buzz-relay) and binary(boot_lifecycle)' + - name: Writer session timeout guardrails + run: | + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E 'package(buzz-db) and test(session_timeouts_install_through_db_new_and_bound_lock_waits)' \ + --run-ignored ignored-only + env: + DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + TEST_DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + - name: Audit writer session timeout guardrails + run: | + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E 'package(buzz-relay) and test(audit_writer_pool_installs_timeouts_and_bounds_advisory_lock_waits)' \ + --run-ignored ignored-only + env: + DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + - name: Audit worker lock-timeout recovery + run: | + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E 'package(buzz-relay) and test(audit_worker_retries_lock_timeout_until_original_entry_is_appended_once)' \ + --run-ignored ignored-only + env: + DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz - name: Start relay run: | chmod +x ./target/ci/buzz-relay diff --git a/Cargo.lock b/Cargo.lock index b51763b70..5f0ac49f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -942,6 +942,7 @@ dependencies = [ "chrono", "futures-util", "hex", + "metrics", "serde", "serde_json", "sha2 0.11.0", @@ -1087,6 +1088,8 @@ dependencies = [ name = "buzz-datastore-tracing" version = "0.1.0" dependencies = [ + "metrics", + "metrics-util", "opentelemetry 0.32.0", "opentelemetry_sdk 0.32.1", "proc-macro2", @@ -1458,6 +1461,7 @@ version = "0.1.0" dependencies = [ "buzz-core", "buzz-datastore-tracing", + "metrics", "sqlx", "thiserror 2.0.18", "tokio", diff --git a/TESTING.md b/TESTING.md index 8d960e95a..b650c05fb 100644 --- a/TESTING.md +++ b/TESTING.md @@ -507,9 +507,49 @@ buzz messages thread --channel "$CHANNEL" --event "$EVENT_ID" | jq . A successful run prints `{"event_id":"…","accepted":true,"message":""}` for the send, and the message body in the `get` output. `thread` returns `[]` -for a leaf message — populated only after a reply comes in (see §5). +for a leaf message — populated only after a reply comes in (see §6). -### 5. Going deeper +### 5. Verify a roster beyond 1,000 members + +Use the focused live-relay script when changing channel membership, discovery, +or reconciliation. It proves the three boundaries that DB-only tests cannot: +the relay-served kind 39002 includes a member at roster position 1,501, that +identity can publish a channel message, and targeted reconciliation preserves +its discoverability. + +Run this only against an isolated local database. The script inserts fixture +members directly, then drives discovery and messaging through the release CLI +and relay. Keep the release relay from step 3 running and use its configured +relay key for authoritative replacement: + +```bash +export PATH="$PWD/target/release:$PATH" +export DATABASE_URL="postgres://buzz:buzz_dev@localhost:5432/buzz_roster_e2e" +export BUZZ_RELAY_URL="http://localhost:3030" # match the relay from step 3 +export RELAY_URL="ws://localhost:3030" +export BUZZ_RELAY_PRIVATE_KEY="" + +scripts/e2e-large-channel-roster.sh +``` + +Success is directly observable as four `PASS` lines. The first and fourth +include a member count greater than 1,000 and the same late-member pubkey; the +second includes the accepted kind 9 event ID, and the third proves targeted +repair left kind 39000/39001 IDs and tags unchanged: + +```text +PASS discovery-before-republish channel= members=1502 late_pubkey= +PASS late-member-action event_id= +PASS targeted-repair-preserves-metadata-and-admin-events channel= +PASS discovery-after-republish channel= members=1502 late_pubkey= +``` + +The script refuses debug binaries and refuses a `buzz` or `buzz-admin` resolved +outside this checkout's `target/release`. It also requires the targeted admin +operation to use `BUZZ_RELAY_PRIVATE_KEY`; never substitute an ephemeral signer +for an authoritative replacement. + +### 6. Going deeper For full coverage of every CLI command (54 subcommands across 12 groups), follow [`crates/buzz-cli/TESTING.md`](crates/buzz-cli/TESTING.md). @@ -652,7 +692,7 @@ CLI-side, only two matter for testing: | Symptom | Cause | Fix | |---------|-------|-----| | `relay error 500` or `400: restricted: not a channel member` after a code change | Stale binary | Rebuild and re-export `PATH`; or `cargo run` directly | -| `Address already in use` on relay start (os error 48 on macOS, 98 on Linux) | Another relay (or stale process) holding `:3000` / `:8080` / `:9102` (or your override ports) | The panic line names the failing port — read it first. Then `lsof -iTCP:3000,8080,9102 -sTCP:LISTEN` (or your override equivalents). Kill the offender (`pkill -f buzz-relay`) or use the port-override block in step 3. If you already overrode and *still* collide, a prior reviewer left a relay running on the same alt ports — kill it or pick fresh ports | +| `Address already in use` on relay start (os error 48 on macOS, 98 on Linux) | Another relay (or stale process) holding `:3000` / `:8080` / `:9102` (or your override ports) | Metrics-listener failures emit a `metrics_bind` lifecycle terminal with reason `bind`. Check the configured ports with `lsof -iTCP:3000,8080,9102 -sTCP:LISTEN` (or your override equivalents). Kill the offender (`pkill -f buzz-relay`) or use the port-override block in step 3. If you already overrode and *still* collide, a prior reviewer left a relay running on the same alt ports — kill it or pick fresh ports | | `auth_error: BUZZ_PRIVATE_KEY is required` | Env not exported into the CLI's shell | `export BUZZ_PRIVATE_KEY=...` (or pass `--private-key`) | | `auth_error: BUZZ_AUTH_TAG verification failed … signature verification failed` | A stale `BUZZ_AUTH_TAG` inherited from a parent shell. The local dev relay rejects it. | `unset BUZZ_AUTH_TAG` (see the scrub block in step 1) | | `auth-required: verification failed` on a closed relay | NIP-OA attestation needed | Set `BUZZ_AUTH_TAG` to the owner-issued JSON, or relax `BUZZ_REQUIRE_RELAY_MEMBERSHIP` | diff --git a/crates/buzz-acp/src/relay.rs b/crates/buzz-acp/src/relay.rs index 073cfaf2e..b008cd4f1 100644 --- a/crates/buzz-acp/src/relay.rs +++ b/crates/buzz-acp/src/relay.rs @@ -1345,6 +1345,40 @@ impl BgState { while let Some(event) = self.observer_in_flight.pop_back() { self.gated_observer_pending.push_front(event); } + self.trim_gated_observer_pending(); + } + + /// Re-park a frame the relay explicitly refused, ahead of frames parked + /// after the gate armed. + /// + /// An `OK(id, false, …)` names the refused frame, so only that frame is + /// retried — frames still awaiting their own verdict stay in the + /// acknowledgment window. This is the correlated counterpart to + /// [`Self::requeue_observer_in_flight`], which must retry everything + /// because a NOTICE identifies nothing. + fn requeue_rejected_observer_frame(&mut self, event_id: &str) { + let Some(index) = self + .observer_in_flight + .iter() + .position(|event| event.id.to_hex() == event_id) + else { + return; + }; + if let Some(event) = self.observer_in_flight.remove(index) { + if self.gated_observer_pending.len() >= GATED_OBSERVER_QUEUE_CAP { + self.gated_observer_pending.pop_front(); + self.gated_observer_dropped += 1; + warn!( + dropped_total = self.gated_observer_dropped, + "gated observer queue full — dropped oldest parked frame for refused retry" + ); + } + self.gated_observer_pending.push_front(event); + } + } + + /// Enforce the parked-queue bound, counting evictions so loss stays visible. + fn trim_gated_observer_pending(&mut self) { while self.gated_observer_pending.len() > GATED_OBSERVER_QUEUE_CAP { self.gated_observer_pending.pop_front(); self.gated_observer_dropped += 1; @@ -2456,7 +2490,10 @@ async fn handle_ws_message( RelayMessage::Notice { message } => { // Fix 4: NOTICE at warn level. tracing::warn!("relay NOTICE: {message}"); - // The relay sends NOTICE for rate-limited EVENT/COUNT frames. + // NOTICE now carries only connection-scoped refusals: an + // EVENT is refused via OK and a REQ/COUNT via CLOSED. A + // NOTICE names nothing, so every unacknowledged observer + // write must be retried. if message.starts_with("rate-limited:") { let secs = parse_rate_limit_retry_secs(&message).unwrap_or(0); let deadline = state.set_rate_limit_gate(secs); @@ -2641,6 +2678,25 @@ async fn handle_ws_message( warn!("mid-session AUTH rejected (event {event_id}): {message} — triggering reconnect"); return false; } + // A refused EVENT is acknowledged on its own channel, so the + // backoff must arm here — not only in the NOTICE arm. Without + // this the harness would publish straight back into the same + // quota it was just refused on. + if !accepted && message.starts_with("rate-limited:") { + let secs = parse_rate_limit_retry_secs(&message).unwrap_or(0); + let deadline = state.set_rate_limit_gate(secs); + // The OK names the refused frame, so re-park only that + // one rather than every unacknowledged frame. + state.requeue_rejected_observer_frame(&event_id); + warn!( + "rate-limit gate armed via OK for event {event_id} until ~{:.1}s from now", + deadline + .checked_duration_since(tokio::time::Instant::now()) + .unwrap_or_default() + .as_secs_f64() + ); + return true; + } state.acknowledge_observer_frame(&event_id); debug!("OK for event {event_id}: accepted={accepted} message={message}"); } @@ -6543,6 +6599,161 @@ mod tests { ); } + /// A rate-limited `OK(id, false, …)` must arm the backoff gate and re-park + /// the refused frame, driven through the real frame dispatcher. + /// + /// This is the buzz-acp side of the relay's rejection-correlation change: + /// a refused EVENT is now acknowledged on its own channel instead of via + /// NOTICE. Reverting either the gate arming or the requeue in the `Ok` arm + /// must fail this test. + #[tokio::test] + async fn rate_limited_ok_arms_gate_and_reparks_refused_observer_frame() { + let (mut client, _server) = test_ws_pair().await; + let (event_tx, _event_rx) = mpsc::channel::>(4); + let (observer_control_tx, _observer_control_rx) = mpsc::channel::(4); + let keys = Keys::generate(); + let mut state = BgState::new(); + + let refused = make_observer_frame(&keys); + let still_pending = make_observer_frame(&keys); + state.track_observer_in_flight(Box::new(refused.clone())); + state.track_observer_in_flight(Box::new(still_pending.clone())); + assert!( + state.check_rate_gate().is_none(), + "gate must start disarmed" + ); + + let frame = json!([ + "OK", + refused.id.to_hex(), + false, + "rate-limited: retry in 5s" + ]); + // Colony routes asks through their own queue and pins the relay; the + // upstream test predates both, so give it throwaways. + let (ask_tx, _ask_rx) = mpsc::channel::(4); + let relay_pin = RelayPin::new("wss://relay.test").expect("pinnable test relay"); + let should_continue = handle_ws_message( + Message::Text(frame.to_string().into()), + &mut client, + &event_tx, + &ask_tx, + &observer_control_tx, + &mut state, + &keys, + &relay_pin, + "agent-pubkey", + None, + ) + .await; + + assert!(should_continue, "a rate-limited OK must keep the socket"); + assert!( + state.check_rate_gate().is_some(), + "a rate-limited OK must arm the backoff gate, or the harness \ + republishes straight into the same quota" + ); + let parked: Vec<_> = state + .gated_observer_pending + .iter() + .map(|event| event.id) + .collect(); + assert_eq!( + parked, + [refused.id], + "the refused frame must be re-parked for redelivery, not dropped" + ); + let in_flight: Vec<_> = state + .observer_in_flight + .iter() + .map(|event| event.id) + .collect(); + assert_eq!( + in_flight, + [still_pending.id], + "frames still awaiting their own verdict must stay in flight" + ); + } + + #[test] + fn rejected_observer_frame_displaces_oldest_parked_frame_at_capacity() { + let mut state = BgState::new(); + let keys = Keys::generate(); + let refused = make_observer_frame(&keys); + state.track_observer_in_flight(Box::new(refused.clone())); + + let oldest = make_observer_frame(&keys); + state.park_gated_observer_frame(Box::new(oldest.clone())); + let mut survivors = Vec::with_capacity(GATED_OBSERVER_QUEUE_CAP - 1); + for _ in 1..GATED_OBSERVER_QUEUE_CAP { + let event = make_observer_frame(&keys); + survivors.push(event.id); + state.park_gated_observer_frame(Box::new(event)); + } + + state.requeue_rejected_observer_frame(&refused.id.to_hex()); + + let parked: Vec<_> = state + .gated_observer_pending + .iter() + .map(|event| event.id) + .collect(); + assert_eq!(parked.len(), GATED_OBSERVER_QUEUE_CAP); + assert_eq!(parked.first(), Some(&refused.id)); + assert_eq!(&parked[1..], survivors.as_slice()); + assert!(!parked.contains(&oldest.id)); + assert_eq!(state.gated_observer_dropped, 1); + assert!(state.observer_in_flight.is_empty()); + } + + /// A non-rate-limit refusal is terminal: retrying would be refused + /// identically, so the frame is retired rather than re-parked, and the + /// backoff gate stays disarmed. + #[tokio::test] + async fn non_rate_limited_ok_rejection_retires_frame_without_arming_gate() { + let (mut client, _server) = test_ws_pair().await; + let (event_tx, _event_rx) = mpsc::channel::>(4); + let (observer_control_tx, _observer_control_rx) = mpsc::channel::(4); + let keys = Keys::generate(); + let mut state = BgState::new(); + + let refused = make_observer_frame(&keys); + state.track_observer_in_flight(Box::new(refused.clone())); + + let frame = json!(["OK", refused.id.to_hex(), false, "invalid: bad signature"]); + // Colony routes asks through their own queue and pins the relay; the + // upstream test predates both, so give it throwaways. + let (ask_tx, _ask_rx) = mpsc::channel::(4); + let relay_pin = RelayPin::new("wss://relay.test").expect("pinnable test relay"); + let should_continue = handle_ws_message( + Message::Text(frame.to_string().into()), + &mut client, + &event_tx, + &ask_tx, + &observer_control_tx, + &mut state, + &keys, + &relay_pin, + "agent-pubkey", + None, + ) + .await; + + assert!(should_continue, "a rejected event must not drop the socket"); + assert!( + state.check_rate_gate().is_none(), + "only a rate-limit refusal arms the backoff gate" + ); + assert!( + state.gated_observer_pending.is_empty(), + "a permanently refused frame must not be requeued into a retry loop" + ); + assert!( + state.observer_in_flight.is_empty(), + "a permanently refused frame must be retired from the window" + ); + } + /// Build a signed observer telemetry frame (kind 24200) for gate tests. fn make_observer_frame(keys: &Keys) -> Event { let recipient = Keys::generate(); diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 3a58602f3..5b325793d 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -92,12 +92,17 @@ enum Command { #[command(subcommand)] command: operator_analytics::Command, }, - /// Emit kind:39000/39002 events for channels missing them. + /// Emit missing kind:39000/39001/39002 channel discovery events, or + /// republish only a targeted channel's kind:39002 roster. /// - /// Channels created via direct SQL (seed scripts, pre-migration data) won't - /// have Nostr discovery events. This command creates them so pure-nostr - /// clients can see those channels. Idempotent — safe to run multiple times. + /// Without `--channel`, only channels missing discovery metadata are + /// reconciled. With `--channel`, only that channel's member snapshot is + /// replaced; canonical metadata and admin events remain untouched. ReconcileChannels { + /// Optional channel UUID to force-republish. + #[arg(long)] + channel: Option, + /// Relay private key (hex) for signing events. Falls back to /// BUZZ_RELAY_PRIVATE_KEY env var. If neither is set, generates /// an ephemeral key (events will be unverifiable after restart). @@ -206,8 +211,8 @@ async fn run(cli: Cli) -> Result { command: ProductFeedbackCommand::List { limit }, } => cmd_list_product_feedback(limit).await, Command::OperatorAnalytics { command } => operator_analytics::run(command).await, - Command::ReconcileChannels { relay_key } => { - reconcile_channels(relay_key).await?; + Command::ReconcileChannels { channel, relay_key } => { + reconcile_channels(channel, relay_key).await?; Ok(0) } Command::Credits { command } => cmd_credits(command).await, @@ -479,10 +484,13 @@ async fn connect_member_services() -> Result<(Db, Arc, Keys)> { async fn connect_db() -> Result { let db_url = std::env::var("DATABASE_URL") .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); - let db = Db::new(&DbConfig { - database_url: db_url, - ..DbConfig::default() - }) + let db = Db::new( + &DbConfig { + database_url: db_url, + ..DbConfig::default() + } + .with_session_timeouts_from_env(), + ) .await?; Ok(db) } @@ -517,14 +525,26 @@ async fn resolve_admin_tenant(db: &Db) -> Result { Ok(TenantContext::resolved(record.id, record.host)) } -async fn reconcile_channels(relay_key_arg: Option) -> Result<()> { +async fn reconcile_channels( + channel_arg: Option, + relay_key_arg: Option, +) -> Result<()> { use buzz_core::kind::KIND_NIP29_GROUP_ADMINS; use buzz_db::event::EventQuery; let db = connect_db().await?; - // Resolve relay signing key: arg > env > ephemeral - let relay_keys = match relay_key_arg.or_else(|| std::env::var("BUZZ_RELAY_PRIVATE_KEY").ok()) { + // Resolve relay signing key: arg > env > ephemeral. Force-republish must + // never use an ephemeral key because it replaces an existing authoritative + // snapshot. + let configured_relay_key = + relay_key_arg.or_else(|| std::env::var("BUZZ_RELAY_PRIVATE_KEY").ok()); + if channel_arg.is_some() && configured_relay_key.is_none() { + return Err(anyhow::anyhow!( + "--channel requires --relay-key or BUZZ_RELAY_PRIVATE_KEY" + )); + } + let relay_keys = match configured_relay_key { Some(key_hex) => { Keys::parse(&key_hex).map_err(|e| anyhow::anyhow!("invalid relay key: {e}"))? } @@ -541,7 +561,21 @@ async fn reconcile_channels(relay_key_arg: Option) -> Result<()> { }; let tenant = resolve_admin_tenant(&db).await?; - let channels = db.list_channels(tenant.community(), None).await?; + let target_channel = channel_arg + .as_deref() + .map(uuid::Uuid::parse_str) + .transpose() + .map_err(|e| anyhow::anyhow!("invalid --channel UUID: {e}"))?; + let channels = if let Some(target) = target_channel { + vec![db + .get_channel(tenant.community(), target) + .await + .map_err(|_| { + anyhow::anyhow!("channel {target} not found in community {}", tenant.host()) + })?] + } else { + db.list_channels(tenant.community(), None).await? + }; if channels.is_empty() { println!("No channels in database."); return Ok(()); @@ -564,57 +598,64 @@ async fn reconcile_channels(relay_key_arg: Option) -> Result<()> { .await .unwrap_or_default(); - if !existing.is_empty() { + if !existing.is_empty() && target_channel.is_none() { skipped += 1; continue; } let members = db.get_members(tenant.community(), channel.id).await?; - // kind:39000 — channel metadata - { - let mut tags: Vec = vec![Tag::parse(["d", &channel_id_str])?]; - tags.push(Tag::parse(["name", &channel.name])?); - if let Some(ref desc) = channel.description { - if !desc.is_empty() { - tags.push(Tag::parse(["about", desc])?); + // A targeted repair is deliberately roster-only. kind:39000 metadata + // is richer than this legacy backfill builder, and kind:39001 is not + // part of the stale-roster incident; replacing either can destroy + // canonical state. Full backfill still creates all three event kinds + // for channels with no discovery metadata. + if target_channel.is_none() { + // kind:39000 — channel metadata + { + let mut tags: Vec = vec![Tag::parse(["d", &channel_id_str])?]; + tags.push(Tag::parse(["name", &channel.name])?); + if let Some(ref desc) = channel.description { + if !desc.is_empty() { + tags.push(Tag::parse(["about", desc])?); + } } + if channel.visibility == "private" { + tags.push(Tag::parse(["private"])?); + } else { + tags.push(Tag::parse(["public"])?); + } + if channel.channel_type == "dm" { + tags.push(Tag::parse(["hidden"])?); + } + tags.push(Tag::parse(["closed"])?); + tags.push(Tag::parse(["t", &channel.channel_type])?); + + let event = EventBuilder::new(Kind::Custom(39000), "") + .tags(tags) + .sign_with_keys(&relay_keys) + .map_err(|e| anyhow::anyhow!("sign kind:39000: {e}"))?; + db.replace_addressable_event(tenant.community(), &event, Some(channel.id)) + .await?; } - if channel.visibility == "private" { - tags.push(Tag::parse(["private"])?); - } else { - tags.push(Tag::parse(["public"])?); - } - if channel.channel_type == "dm" { - tags.push(Tag::parse(["hidden"])?); - } - tags.push(Tag::parse(["closed"])?); - tags.push(Tag::parse(["t", &channel.channel_type])?); - let event = EventBuilder::new(Kind::Custom(39000), "") - .tags(tags) - .sign_with_keys(&relay_keys) - .map_err(|e| anyhow::anyhow!("sign kind:39000: {e}"))?; - db.replace_addressable_event(tenant.community(), &event, Some(channel.id)) - .await?; - } - - // kind:39001 — admins - { - let mut tags: Vec = vec![Tag::parse(["d", &channel_id_str])?]; - for m in members - .iter() - .filter(|m| m.role == "owner" || m.role == "admin") + // kind:39001 — admins { - let pk = hex::encode(&m.pubkey); - tags.push(Tag::parse(["p", &pk, &m.role])?); + let mut tags: Vec = vec![Tag::parse(["d", &channel_id_str])?]; + for m in members + .iter() + .filter(|m| m.role == "owner" || m.role == "admin") + { + let pk = hex::encode(&m.pubkey); + tags.push(Tag::parse(["p", &pk, &m.role])?); + } + let event = EventBuilder::new(Kind::Custom(KIND_NIP29_GROUP_ADMINS as u16), "") + .tags(tags) + .sign_with_keys(&relay_keys) + .map_err(|e| anyhow::anyhow!("sign kind:39001: {e}"))?; + db.replace_addressable_event(tenant.community(), &event, Some(channel.id)) + .await?; } - let event = EventBuilder::new(Kind::Custom(KIND_NIP29_GROUP_ADMINS as u16), "") - .tags(tags) - .sign_with_keys(&relay_keys) - .map_err(|e| anyhow::anyhow!("sign kind:39001: {e}"))?; - db.replace_addressable_event(tenant.community(), &event, Some(channel.id)) - .await?; } // kind:39002 — members diff --git a/crates/buzz-audit/Cargo.toml b/crates/buzz-audit/Cargo.toml index dfa73353d..766ade650 100644 --- a/crates/buzz-audit/Cargo.toml +++ b/crates/buzz-audit/Cargo.toml @@ -17,6 +17,7 @@ serde_json = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } tracing = { workspace = true } +metrics = { workspace = true } thiserror = { workspace = true } sha2 = { workspace = true } hex = { workspace = true } diff --git a/crates/buzz-datastore-tracing/Cargo.toml b/crates/buzz-datastore-tracing/Cargo.toml index e93900c54..fb7ba6f37 100644 --- a/crates/buzz-datastore-tracing/Cargo.toml +++ b/crates/buzz-datastore-tracing/Cargo.toml @@ -16,6 +16,8 @@ quote = "1" syn = { version = "2", features = ["full"] } [dev-dependencies] +metrics = { workspace = true } +metrics-util = { workspace = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } tokio = { workspace = true } diff --git a/crates/buzz-datastore-tracing/src/lib.rs b/crates/buzz-datastore-tracing/src/lib.rs index f2645cb8f..217f2335d 100644 --- a/crates/buzz-datastore-tracing/src/lib.rs +++ b/crates/buzz-datastore-tracing/src/lib.rs @@ -70,6 +70,9 @@ impl Parse for DatastoreArgs { /// PostgreSQL spans always omit function arguments, use the `buzz_datastore` /// target, and expose only canonical semantic fields plus explicitly supplied /// safe fields. An `Err` sets `otel.status_code` without inspecting the error. +/// The literal `name` also labels a logical-operation duration histogram. Slow +/// completions are sampled and logged with only that name, outcome, and elapsed +/// time; arguments, error values, and return values are never formatted. #[proc_macro_attribute] pub fn datastore_span(args: TokenStream, item: TokenStream) -> TokenStream { let args = parse_macro_input!(args as DatastoreArgs); @@ -129,9 +132,46 @@ pub fn datastore_span(args: TokenStream, item: TokenStream) -> TokenStream { } } }); + let outcome = if returns_result { + quote! { + if #result.is_err() { "error" } else { "success" } + } + } else { + quote!("success") + }; function.block = Box::new(syn::parse_quote!({ + let __buzz_datastore_started_7f3a9c = ::std::time::Instant::now(); let #result: #return_type = (async #original_body).await; #record_error + let __buzz_datastore_outcome_7f3a9c = #outcome; + let __buzz_datastore_elapsed_7f3a9c = __buzz_datastore_started_7f3a9c.elapsed(); + ::metrics::histogram!( + "buzz_db_operation_duration_seconds", + "operation" => #name, + "outcome" => __buzz_datastore_outcome_7f3a9c, + ) + .record(__buzz_datastore_elapsed_7f3a9c.as_secs_f64()); + if __buzz_datastore_elapsed_7f3a9c >= ::std::time::Duration::from_millis(500) { + static __BUZZ_DATASTORE_SLOW_SAMPLE_7F3A9C: + ::std::sync::atomic::AtomicU64 = ::std::sync::atomic::AtomicU64::new(0); + if __BUZZ_DATASTORE_SLOW_SAMPLE_7F3A9C.fetch_add( + 1, + ::std::sync::atomic::Ordering::Relaxed, + ) % 100 == 0 { + let __buzz_datastore_elapsed_ms_7f3a9c = + __buzz_datastore_elapsed_7f3a9c + .as_millis() + .min(::std::primitive::u64::MAX as u128) as u64; + ::tracing::warn!( + target: "buzz_datastore", + parent: None, + operation = #name, + outcome = __buzz_datastore_outcome_7f3a9c, + elapsed_ms = __buzz_datastore_elapsed_ms_7f3a9c, + "slow datastore operation" + ); + } + } #result })); diff --git a/crates/buzz-datastore-tracing/tests/runtime.rs b/crates/buzz-datastore-tracing/tests/runtime.rs index b58dca871..335519095 100644 --- a/crates/buzz-datastore-tracing/tests/runtime.rs +++ b/crates/buzz-datastore-tracing/tests/runtime.rs @@ -1,6 +1,12 @@ use buzz_datastore_tracing::datastore_span; +use metrics_util::debugging::{DebugValue, DebuggingRecorder}; use opentelemetry::trace::{SpanKind, Status, TracerProvider as _}; use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider}; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use tracing::field::{Field, Visit}; +use tracing::{Event, Subscriber}; +use tracing_subscriber::layer::{Context, Layer}; use tracing_subscriber::prelude::*; const DIRECT_ERROR: &str = "raw-secret-direct-error"; @@ -27,8 +33,48 @@ async fn operation( Ok(limit) } +#[datastore_span(name = "slow_test_operation", system = "postgresql")] +async fn slow_operation(delay: std::time::Duration) -> Result<(), &'static str> { + tokio::time::sleep(delay).await; + Err(DIRECT_ERROR) +} + +#[derive(Default)] +struct EventFields(BTreeMap); + +impl Visit for EventFields { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0.insert(field.name().to_owned(), format!("{value:?}")); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.0.insert(field.name().to_owned(), value.to_owned()); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.0.insert(field.name().to_owned(), value.to_string()); + } +} + +#[derive(Clone, Default)] +struct EventCapture(Arc>>); + +impl Layer for EventCapture +where + S: Subscriber, +{ + fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) { + let mut fields = EventFields::default(); + event.record(&mut fields); + self.0.lock().expect("capture lock").push(fields); + } +} + #[tokio::test(flavor = "current_thread")] async fn exports_policy_fields_without_error_or_argument_data() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _metrics_guard = metrics::set_default_local_recorder(&recorder); let exporter = InMemorySpanExporter::default(); let provider = SdkTracerProvider::builder() .with_simple_exporter(exporter.clone()) @@ -41,6 +87,37 @@ async fn exports_policy_fields_without_error_or_argument_data() { assert_eq!(operation(8, true, false).await, Err(DIRECT_ERROR)); assert_eq!(operation(9, false, true).await, Err(QUESTION_ERROR)); + let operation_samples = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(key, ..)| key.key().name() == "buzz_db_operation_duration_seconds") + .map(|(key, _, _, value)| { + let DebugValue::Histogram(samples) = value else { + panic!("operation duration must be a histogram"); + }; + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + (labels, samples) + }) + .collect::>(); + assert_eq!(operation_samples.len(), 2); + for (labels, samples) in operation_samples { + assert_eq!( + labels.get("operation").map(String::as_str), + Some("test_operation") + ); + assert!(matches!( + labels.get("outcome").map(String::as_str), + Some("success" | "error") + )); + assert!(!samples.is_empty()); + assert!(samples.iter().all(|sample| sample.into_inner() >= 0.0)); + } + provider.force_flush().expect("spans flush"); let spans = exporter.get_finished_spans().expect("exported spans"); assert_eq!(spans.len(), 3); @@ -78,3 +155,55 @@ async fn exports_policy_fields_without_error_or_argument_data() { } } } + +#[tokio::test(flavor = "current_thread")] +async fn slow_operation_logging_is_guarded_sampled_and_redacted() { + let capture = EventCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _subscriber_guard = tracing::subscriber::set_default(subscriber); + + assert_eq!( + slow_operation(std::time::Duration::from_millis(1)).await, + Err(DIRECT_ERROR) + ); + assert_eq!( + slow_operation(std::time::Duration::from_millis(510)).await, + Err(DIRECT_ERROR) + ); + assert_eq!( + slow_operation(std::time::Duration::from_millis(510)).await, + Err(DIRECT_ERROR) + ); + + let events = capture.0.lock().expect("capture lock"); + let slow = events + .iter() + .filter(|event| { + event + .0 + .get("message") + .is_some_and(|message| message.contains("slow datastore operation")) + }) + .collect::>(); + assert_eq!( + slow.len(), + 1, + "first slow call is logged, next 99 are sampled out" + ); + let fields = &slow[0].0; + assert_eq!( + fields.get("operation").map(String::as_str), + Some("slow_test_operation") + ); + assert_eq!(fields.get("outcome").map(String::as_str), Some("error")); + assert!(fields + .get("elapsed_ms") + .and_then(|value| value.parse::().ok()) + .is_some_and(|elapsed| elapsed >= 500)); + assert_eq!( + fields.len(), + 4, + "only message and fixed safe fields are logged" + ); + assert!(!format!("{fields:?}").contains(DIRECT_ERROR)); +} diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 627324da4..2f2d35b13 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -27,16 +27,27 @@ mod runtime; mod store; +#[cfg(test)] +mod test_support; /// Database error types. pub mod error; pub use runtime::{ insert_mentions, insert_mentions_tx, migration, replica_fence, Db, DbConfig, DbPoolStats, - ReadSession, + DbReadinessOutcome, ReadSession, }; + +/// Valid low-cardinality `(pool_role, operation)` pairs for pool-acquisition telemetry. +pub const DB_POOL_ACQUIRE_VALID_PAIRS: [(&str, &str); 11] = + runtime::observability::POOL_ACQUIRE_VALID_PAIRS; + +/// Raw Prometheus series ceiling per relay pod for the operation-aware contract. +pub const DB_POOL_ACQUIRE_RAW_SERIES_PER_POD: usize = + runtime::observability::POOL_ACQUIRE_RAW_SERIES_PER_POD; pub(crate) use runtime::{ - insert_mentions_in_transaction, route_proof, ReadSessionInner, RouteDecision, RoutePredicate, + insert_mentions_in_transaction, observability, route_proof, ReadSessionInner, RouteDecision, + RoutePredicate, }; pub use store::{ diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 5abbdbecc..a920a1c51 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -30,6 +30,22 @@ pub async fn run_migrations(pool: &PgPool) -> Result<()> { .await } +/// Run migrations only up to `target`, for tests that need a database frozen +/// at a specific schema version. +#[cfg(test)] +pub(crate) async fn run_migrations_through(pool: &PgPool, target: i64) -> Result<()> { + with_exclusive_schema_destruction_lock(pool, |lock_conn| async move { + let outcome = async { + reject_legacy_nip_rs_cardinality_ambiguity(pool).await?; + MIGRATOR.run_to(target, pool).await?; + Ok(()) + } + .await; + (lock_conn, outcome) + }) + .await +} + async fn run_migrations_locked(pool: &PgPool) -> Result<()> { reject_legacy_nip_rs_cardinality_ambiguity(pool).await?; MIGRATOR.run(pool).await?; @@ -42,6 +58,10 @@ async fn run_migrations_locked(pool: &PgPool) -> Result<()> { // guard, so migration fails closed if any is missing. (The fence probe // re-runs this same check at startup on non-migrating relays.) crate::replica_fence::verify_floor_guard_catalog(pool).await?; + // Migration 0073's roster fence is the mixed-version guard that keeps an + // old pod from overwriting a newer canonical kind:39002 snapshot. Same + // fail-closed rule as the floor guard above. + crate::channel_members::verify_channel_roster_fence_catalog(pool).await?; Ok(()) } @@ -62,11 +82,28 @@ where F: FnOnce(PgConnection) -> Fut, Fut: Future)>, { - let mut lock_conn = pool.acquire().await?.detach(); - sqlx::query("SELECT pg_advisory_lock($1)") - .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + let mut lock_conn = crate::observability::acquire_writer_with_legacy_metrics( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await? + .detach(); + // This dedicated connection intentionally waits for the current migration + // or schema-destruction owner and may then run long DDL. Exempt those two + // phases from runtime lock/statement budgets. Keep the idle-in-transaction + // timeout: a client wedged idle mid-migration is still a lock holder that + // should be reaped. The detached connection is closed below and never + // returns these session settings to the pool. + sqlx::raw_sql("SET lock_timeout = 0; SET statement_timeout = 0") .execute(&mut lock_conn) .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::MigrationSchemaSafety, + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(&mut lock_conn), + ) + .await?; let (mut lock_conn, outcome) = op(lock_conn).await; let unlock = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(SCHEMA_DESTRUCTION_LOCK_KEY) @@ -257,7 +294,11 @@ async fn reject_legacy_nip_rs_cardinality_ambiguity(pool: &PgPool) -> Result<()> #[cfg(test)] mod tests { use super::*; - use std::collections::BTreeSet; + use std::{ + collections::BTreeSet, + fs, + path::{Path, PathBuf}, + }; const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; @@ -735,7 +776,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 71); + assert_eq!(migrations.len(), 74); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1469,7 +1510,148 @@ mod tests { // The grant is stored, never recomputed: a price edit mid-flight must // not change what an already-paid purchase is worth. assert!(packs.contains("ADD COLUMN grant_nanousd BIGINT")); + + // NIP-PMA private managed-agent FTS exclusion (0074): same + // wrap-the-existing-expression shape as 0014 so brownfield databases + // stop tokenizing private managed-agent ciphertext without a policy + // rewrite. (The migration itself still rewrites the events heap and + // rebuilds the GIN index - see the 0074 header for the cost.) + // + // Upstream's kind for this payload is 30179; here 30179 is + // KIND_COMPANY_PROFILE, which must stay searchable, and the private + // managed-agent definition is KIND_PRIVATE_MANAGED_AGENT = 30194. + assert_eq!(migrations[73].version, 74); + let private_agent_fts = migrations[73].sql.as_str(); + assert!(private_agent_fts.contains("kind = 30194")); + assert!(private_agent_fts.contains("search_tsv")); + assert!( + !private_agent_fts.contains("30179"), + "30179 is KIND_COMPANY_PROFILE here and must remain indexed" + ); + assert_eq!( + buzz_core::kind::KIND_PRIVATE_MANAGED_AGENT, + 30194, + "the migration literal must track the registered kind" + ); + assert!(!migrations[0].sql.as_str().contains("30194")); + // Colony's desired-state schema uses a positive allowlist rather than + // upstream's negative skip-set, so a fresh install already excludes the + // private managed-agent kind and schema.sql needs no change. Pin that. + assert!(desired_schema.contains("CASE WHEN kind IN (0, 9, 40002, 45001, 45003)")); + assert!( + !desired_schema.contains("30194"), + "the allowlist must never gain the private managed-agent kind" + ); + + // Mixed-version channel-roster fence: old canonical replacement writers + // acquire their replacement key before INSERT; this trigger then takes + // the membership key and validates the exact active pubkey/role p-tag set. + assert_eq!(migrations[72].version, 73); + let roster_fence = migrations[72].sql.as_str(); + assert!(roster_fence.contains("CREATE TRIGGER trg_events_guard_channel_roster_snapshot")); + assert!(roster_fence.contains("NEW.kind <> 39002")); + assert!(roster_fence.contains("'buzz_channel_membership:'")); + assert!(roster_fence.contains("cm.removed_at IS NULL")); + assert!(roster_fence.contains("cm.role::text")); + assert!(roster_fence.contains("jsonb_array_length(roster_tag.tag_json) <> 4")); + assert!(roster_fence.contains("roster_tag.tag_json->>3")); + assert!(roster_fence.contains("snapshot_members IS DISTINCT FROM canonical_members")); + assert!(roster_fence.contains("ERRCODE = '23514'")); + // Fresh desired-state bootstrap must install the identical executable + // fence as migration 0073. CI and isolated relay startup use schema.sql + // without running migrations, so drift reopens rolling-deploy races. + fn extract_roster_fence(sql: &str) -> &str { + let fence_start = "CREATE OR REPLACE FUNCTION guard_channel_roster_snapshot()"; + let fence_end = " FOR EACH ROW EXECUTE FUNCTION guard_channel_roster_snapshot();"; + let start = sql.find(fence_start).expect("roster fence function"); + let relative_end = sql[start..].find(fence_end).expect("roster fence trigger"); + &sql[start..start + relative_end + fence_end.len()] + } + assert_eq!( + extract_roster_fence(roster_fence), + extract_roster_fence(desired_schema) + ); + + // The single-row heartbeat table is updated continuously. Prevent + // autovacuum from truncating its heap so standby queries are not + // cancelled by the ACCESS EXCLUSIVE truncation lock replay. + assert_eq!(migrations[71].version, 72); + let heartbeat_vacuum = migrations[71].sql.as_str(); + assert!(heartbeat_vacuum.contains("ALTER TABLE replica_heartbeat")); + assert!(heartbeat_vacuum.contains("vacuum_truncate = false")); + assert!(desired_schema.contains("vacuum_truncate = false")); + + // pgschema intentionally reconciles DDL, not seed DML or table storage + // parameters. Its post-apply reconciliation must restore and verify + // both parts of the live heartbeat contract for fresh bootstraps. + let pgschema_reconciliation = + include_str!("../../../../scripts/reconcile-schema-after-pgschema.sql"); + assert!(pgschema_reconciliation + .contains("ALTER TABLE replica_heartbeat SET (vacuum_truncate = false)")); + assert!(pgschema_reconciliation.contains("INSERT INTO replica_heartbeat (id) VALUES (1)")); + assert!(pgschema_reconciliation.contains("ON CONFLICT (id) DO NOTHING")); + assert!(pgschema_reconciliation.contains("pg_class")); + assert!(pgschema_reconciliation.contains("reloptions")); } + + #[test] + fn every_pgschema_apply_runs_post_apply_reconciliation() { + fn files_under(root: &Path) -> Vec { + let mut pending = vec![root.to_owned()]; + let mut files = Vec::new(); + + while let Some(path) = pending.pop() { + for entry in fs::read_dir(&path) + .unwrap_or_else(|error| panic!("could not read {}: {error}", path.display())) + { + let path = entry.expect("directory entry").path(); + if path.is_dir() { + pending.push(path); + } else { + files.push(path); + } + } + } + + files + } + + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let roots = [ + repo_root.join("scripts"), + repo_root.join(".github/workflows"), + ]; + let mut apply_count = 0; + + for path in roots.iter().flat_map(|root| files_under(root)) { + let Ok(contents) = fs::read_to_string(&path) else { + continue; + }; + let lines: Vec<_> = contents.lines().collect(); + + for (index, line) in lines.iter().enumerate() { + if !line.contains("./bin/pgschema apply") { + continue; + } + + apply_count += 1; + let following_lines = &lines[index + 1..(index + 7).min(lines.len())]; + assert!( + following_lines.iter().any(|line| line.contains( + "scripts/reconcile-schema-after-pgschema.sql" + )), + "{} must run scripts/reconcile-schema-after-pgschema.sql immediately after pgschema apply", + path.display() + ); + } + } + + assert!( + apply_count > 0, + "expected at least one pgschema apply caller" + ); + } + #[test] fn block_action_claim_migration_is_community_scoped() { let migration = MIGRATOR @@ -2168,7 +2350,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn populated_upgrade_preserves_search_policy_except_for_push_leases() { + async fn populated_upgrade_preserves_search_policy_except_for_private_kinds() { let pool = connect_test_pool().await; reset_public_schema(&pool).await; MIGRATOR @@ -2184,7 +2366,7 @@ mod tests { .await .expect("insert community"); - for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32)] { + for (marker, kind) in [(1_u8, 1_i32), (2_u8, 30_350_i32), (3_u8, 30_194_i32)] { sqlx::query( "INSERT INTO events \ (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at) \ @@ -2211,19 +2393,37 @@ mod tests { .fetch_all(&pool) .await .expect("read pre-push search behavior"); - assert_eq!(before, vec![(1, true), (30_350, true)]); + assert_eq!(before, vec![(1, true), (30_194, true), (30_350, true)]); + + // 0014 fixes 30350 only. A brownfield database that stopped short of + // 0074 still tokenized kind:30194 ciphertext - the gap 0074 closes. + MIGRATOR + .run_to(73, &pool) + .await + .expect("apply migrations through 73"); + let pre_0074: Vec<(i32, Option)> = sqlx::query_as( + "SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \ + FROM events ORDER BY kind", + ) + .fetch_all(&pool) + .await + .expect("read pre-0074 search behavior"); + assert_eq!( + pre_0074, + vec![(1, Some(true)), (30_194, Some(true)), (30_350, None)] + ); run_migrations(&pool) .await - .expect("apply push migrations to populated database"); + .expect("apply remaining migrations to populated database"); let after: Vec<(i32, Option)> = sqlx::query_as( "SELECT kind, search_tsv @@ plainto_tsquery('simple', 'needle') \ FROM events ORDER BY kind", ) .fetch_all(&pool) .await - .expect("read post-push search behavior"); - assert_eq!(after, vec![(1, Some(true)), (30_350, None)]); + .expect("read post-upgrade search behavior"); + assert_eq!(after, vec![(1, Some(true)), (30_194, None), (30_350, None)]); } #[tokio::test] diff --git a/crates/buzz-db/src/runtime/mod.rs b/crates/buzz-db/src/runtime/mod.rs index a87b32b1b..046362111 100644 --- a/crates/buzz-db/src/runtime/mod.rs +++ b/crates/buzz-db/src/runtime/mod.rs @@ -6,6 +6,7 @@ /// Embedded database migrations. pub mod migration; +pub(crate) mod observability; /// Replica freshness fence for keyset-cursor read routing. pub mod replica_fence; @@ -31,7 +32,9 @@ pub async fn insert_mentions( event: &nostr::Event, channel_id: Option, ) -> Result<()> { - let mut tx = pool.begin().await?; + let connection = + observability::acquire_writer(pool, observability::WriterOperation::EventWrite).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; insert_mentions_in_transaction(&mut tx, community_id, event, channel_id).await?; tx.commit().await?; Ok(()) @@ -480,6 +483,26 @@ pub struct DbPoolStats { pub max: u32, } +/// Bounded outcome of the Postgres portion of a relay readiness check. +/// +/// The variants deliberately separate waiting for a pooled connection from +/// executing the health query. Callers may safely use the variant names as +/// low-cardinality metric labels; detailed SQLx errors remain in logs rather +/// than becoming labels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DbReadinessOutcome { + /// A writer-pool connection was acquired and `SELECT 1` succeeded. + Success, + /// No writer-pool connection became available before the readiness deadline. + PoolTimeout, + /// The writer pool returned a non-timeout acquisition error. + PoolError, + /// A connection was acquired, but `SELECT 1` exceeded the readiness deadline. + QueryTimeout, + /// A connection was acquired, but `SELECT 1` returned an error. + QueryError, +} + /// Configuration for the Postgres connection pool. #[derive(Debug, Clone)] pub struct DbConfig { @@ -509,6 +532,16 @@ pub struct DbConfig { /// than the staleness gate never routes anyway, so a larger budget /// would only misrepresent the config. pub replica_read_max_age_ms: u64, + /// Session `lock_timeout` in milliseconds for writer connections (env + /// `BUZZ_DB_LOCK_TIMEOUT_MS`). `0` disables the timeout. + pub lock_timeout_ms: u64, + /// Session `idle_in_transaction_session_timeout` in milliseconds for + /// writer connections (env `BUZZ_DB_IDLE_TXN_TIMEOUT_MS`). `0` disables. + pub idle_txn_timeout_ms: u64, + /// Session `statement_timeout` in milliseconds for writer connections + /// (env `BUZZ_DB_STATEMENT_TIMEOUT_MS`). `0` disables it and is the + /// default because migrations and backfills may legitimately run long. + pub statement_timeout_ms: u64, } impl Default for DbConfig { @@ -526,50 +559,119 @@ impl Default for DbConfig { max_lifetime_secs: 1800, idle_timeout_secs: 600, replica_read_max_age_ms: 0, + lock_timeout_ms: DEFAULT_LOCK_TIMEOUT_MS, + idle_txn_timeout_ms: DEFAULT_IDLE_TXN_TIMEOUT_MS, + statement_timeout_ms: 0, } } } use route_proof::ChannelScoped; -impl Db { - /// Reader acquire timeout — deliberately far below the writer's - /// (seconds-denominated) timeout. Failing closed to the writer must be - /// fast: a saturated reader pool that made routed reads wait the full - /// writer-style timeout would add dead latency during exactly the load - /// spike the offload exists for. A miss here surfaces as - /// `writer/reader_acquire_timeout` (see [`Db::proved_reader`] for why - /// the reason names the mechanism rather than a diagnosis). - const READER_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(150); +/// Default writer `lock_timeout` in milliseconds. +pub const DEFAULT_LOCK_TIMEOUT_MS: u64 = 5_000; + +/// Default writer `idle_in_transaction_session_timeout` in milliseconds. +pub const DEFAULT_IDLE_TXN_TIMEOUT_MS: u64 = 60_000; + +impl DbConfig { + /// Overlay writer session timeouts from the shared `BUZZ_DB_*_TIMEOUT_MS` + /// environment variables. Missing or invalid values retain the existing + /// configuration; explicit zeroes pass through to disable a timeout. + /// + /// This belongs in `buzz-db` so relay, admin, deletion, and audit writers + /// share one policy. The separately deployed push gateway owns its own + /// database and session policy. + pub fn with_session_timeouts_from_env(mut self) -> Self { + fn parse(key: &str) -> Option { + std::env::var(key) + .ok() + .and_then(|value| value.parse::().ok()) + } + + if let Some(value) = parse("BUZZ_DB_LOCK_TIMEOUT_MS") { + self.lock_timeout_ms = value; + } + if let Some(value) = parse("BUZZ_DB_IDLE_TXN_TIMEOUT_MS") { + self.idle_txn_timeout_ms = value; + } + if let Some(value) = parse("BUZZ_DB_STATEMENT_TIMEOUT_MS") { + self.statement_timeout_ms = value; + } + self + } +} - /// Connect one pool with the sizing knobs from `config`. +impl Db { + /// Connect the writer pool with all session-level safety premises. /// - /// `arm_floor_guard` sets the `buzz.created_at_floor` session GUC on - /// every connection, arming the deferred commit-time trigger from - /// migration 0021. Writer pools must arm it; replica pools are read-only - /// so the trigger never fires there. - async fn connect_pool(config: &DbConfig, url: &str, arm_floor_guard: bool) -> Result { - let mut options = PgPoolOptions::new() + /// SQLx stores one `after_connect` hook, so the floor guard and transaction + /// isolation assertion must remain in this single closure. Registering a + /// second hook replaces the first and silently disarms the floor trigger. + /// Additional writer pools, including the relay audit pool, must use this + /// constructor so they inherit the timeout, floor-guard, and isolation + /// policy installed by [`Db::new`]. + pub async fn connect_writer_pool(config: &DbConfig) -> Result { + let lock_timeout_ms = config.lock_timeout_ms; + let idle_txn_timeout_ms = config.idle_txn_timeout_ms; + let statement_timeout_ms = config.statement_timeout_ms; + let options = PgPoolOptions::new() .max_connections(config.max_connections) .min_connections(config.min_connections) .acquire_timeout(Duration::from_secs(config.acquire_timeout_secs)) .max_lifetime(Duration::from_secs(config.max_lifetime_secs)) - .idle_timeout(Duration::from_secs(config.idle_timeout_secs)); - if arm_floor_guard { - options = options.after_connect(|conn, _meta| { + .idle_timeout(Duration::from_secs(config.idle_timeout_secs)) + .after_connect(move |conn, _meta| { Box::pin(async move { // `SET` cannot take bind parameters; `set_config` can. sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(conn) + .execute(&mut *conn) + .await?; + // `lock_timeout` fails the waiting statement; it does not + // cancel the holder. `idle_in_transaction_session_timeout` + // reaps only holders idling inside an open transaction, + // while actively executing holders are bounded only by + // `statement_timeout` (off by default). Bare values are + // milliseconds. Migration/schema-destruction connections + // reset lock and statement timeouts before their intentional + // long wait (see `with_exclusive_schema_destruction_lock`). + sqlx::query( + "SELECT set_config('lock_timeout', $1, false), \ + set_config('idle_in_transaction_session_timeout', $2, false), \ + set_config('statement_timeout', $3, false)", + ) + .bind(lock_timeout_ms.to_string()) + .bind(idle_txn_timeout_ms.to_string()) + .bind(statement_timeout_ms.to_string()) + .execute(&mut *conn) + .await?; + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(&mut *conn) .await?; + if isolation != "read committed" { + return Err(sqlx::Error::Configuration( + format!( + "writer pool requires READ COMMITTED transaction isolation, got {isolation}" + ) + .into(), + )); + } Ok(()) }) }); - } - Ok(options.connect(url).await?) + Ok(options.connect(&config.database_url).await?) } + /// Reader acquire timeout — deliberately far below the writer's + /// (seconds-denominated) timeout. Failing closed to the writer must be + /// fast: a saturated reader pool that made routed reads wait the full + /// writer-style timeout would add dead latency during exactly the load + /// spike the offload exists for. A miss here surfaces as + /// `writer/reader_acquire_timeout` (see [`Db::proved_reader`] for why + /// the reason names the mechanism rather than a diagnosis). + const READER_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(150); + /// Connect the read-replica pool **lazily** — no connection is /// attempted at construction, so a reader that is down at boot cannot /// crash the relay (it starts all-writer with the fence closed and @@ -583,7 +685,7 @@ impl Db { /// the pool back up, which is fine — routed reads re-fill it on demand. /// /// No floor guard: replica sessions are read-only, the trigger never - /// fires there (see [`Db::connect_pool`]). + /// fires there. fn connect_read_pool(config: &DbConfig, url: &str, max_connections: u32) -> Result { Ok(PgPoolOptions::new() .max_connections(max_connections) @@ -626,6 +728,7 @@ impl Db { async fn proved_reader( &self, read_pool: &PgPool, + operation: observability::ReaderOperation, ) -> std::result::Result< ( sqlx::Transaction<'static, sqlx::Postgres>, @@ -639,7 +742,9 @@ impl Db { // `read_pool` separately would spend a second budget whenever the // capability is uncached — i.e. after a failed boot ping, which is // precisely the reader-unavailable case the bound must hold for. - let conn = match read_pool.acquire().await { + let conn = match observability::acquire_reader_with_legacy_metrics(read_pool, operation) + .await + { Ok(conn) => conn, Err(sqlx::Error::PoolTimedOut) => { tracing::warn!("reader pool acquire timed out; routing to writer"); @@ -761,6 +866,7 @@ impl Db { &self, path: &'static str, predicate: RoutePredicate, + operation: observability::ReaderOperation, ) -> RouteDecision { let Some(read_pool) = &self.read_pool else { Self::record_route(path, "writer", "disabled"); @@ -803,7 +909,7 @@ impl Db { Self::record_route(path, "writer", reason); return RouteDecision::Writer; } - match self.proved_reader(read_pool).await { + match self.proved_reader(read_pool, operation).await { Ok((tx, entry)) => { // Re-evaluate against the entry the session actually proved // (it may be older than the shared newest). @@ -854,7 +960,7 @@ impl Db { /// `buzz.created_at_floor` GUC — this is what makes the replica fence /// proof hold for every insert path that goes through this pool. pub async fn new(config: &DbConfig) -> Result { - let pool = Self::connect_pool(config, &config.database_url, true).await?; + let pool = Self::connect_writer_pool(config).await?; let read_max_connections = config .read_max_connections .unwrap_or(config.max_connections); @@ -894,25 +1000,43 @@ impl Db { return; }; let aurora_identity = self.reader_aurora_identity.clone(); - tokio::spawn(async move { - match read_pool.acquire().await { - Ok(mut conn) => { - tracing::info!("read replica reachable at boot"); - match crate::replica_fence::reader_supports_aurora_identity(&mut conn).await { - Ok(supported) => { - let _ = aurora_identity.set(supported); - } - Err(e) => tracing::debug!( - error = %e, - "aurora identity boot prime failed; first routed read will probe" - ), + tokio::spawn(Self::read_pool_boot_ping_once(read_pool, aurora_identity)); + } + + async fn read_pool_boot_ping_once( + read_pool: PgPool, + aurora_identity: std::sync::Arc>, + ) { + match observability::acquire_reader_with_legacy_metrics( + &read_pool, + observability::ReaderOperation::Bootstrap, + ) + .await + { + Ok(mut conn) => { + tracing::info!("read replica reachable at boot"); + match crate::replica_fence::reader_supports_aurora_identity(&mut conn).await { + Ok(supported) => { + let _ = aurora_identity.set(supported); } + Err(e) => tracing::debug!( + error = %e, + "aurora identity boot prime failed; first routed read will probe" + ), } - Err(e) => tracing::warn!( - "read replica unreachable at boot; serving all-writer until it recovers: {e}" - ), } - }); + Err(e) => tracing::warn!( + "read replica unreachable at boot; serving all-writer until it recovers: {e}" + ), + } + } + + #[cfg(test)] + pub(crate) async fn read_pool_boot_ping_for_tests(&self) { + let Some(read_pool) = self.read_pool.clone() else { + return; + }; + Self::read_pool_boot_ping_once(read_pool, self.reader_aurora_identity.clone()).await; } /// Creates a `Db` from an existing `PgPool` (useful in tests). @@ -978,8 +1102,7 @@ impl Db { if self.read_pool.is_none() { return Ok(false); } - crate::replica_fence::verify_floor_guard_catalog(&self.pool).await?; - crate::replica_fence::verify_floor_guard_behavior(&self.pool).await?; + self.verify_replica_fence_at_boot().await?; tokio::spawn(crate::replica_fence::run_probe( self.pool.clone(), std::sync::Arc::clone(&self.fence), @@ -987,6 +1110,17 @@ impl Db { Ok(true) } + /// Verify replica-fence catalog shape and behavior through attributed + /// writer/bootstrap acquisitions without starting the recurring probe. + pub(crate) async fn verify_replica_fence_at_boot(&self) -> Result<()> { + let mut connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::Bootstrap) + .await?; + crate::replica_fence::verify_floor_guard_catalog(&mut *connection).await?; + drop(connection); + crate::replica_fence::verify_floor_guard_behavior(&self.pool).await + } + /// Whether a distinct read-replica pool is configured. pub fn has_read_pool(&self) -> bool { self.read_pool.is_some() @@ -999,7 +1133,60 @@ impl Db { /// Returns `true` if the database is reachable (used by readiness probes). pub async fn ping(&self) -> bool { - sqlx::query("SELECT 1").execute(&self.pool).await.is_ok() + let Ok(mut connection) = + observability::acquire_writer(&self.pool, observability::WriterOperation::Readiness) + .await + else { + return false; + }; + sqlx::query("SELECT 1") + .execute(&mut *connection) + .await + .is_ok() + } + + /// Checks writer-pool acquisition and query execution against one deadline. + /// + /// Unlike [`Self::ping`], this preserves whether readiness was blocked while + /// borrowing a connection or failed after a connection had been acquired. + /// The query runs on the already-acquired connection so the two phases + /// cannot be collapsed into a second implicit pool acquisition. + pub async fn readiness_check(&self, deadline: tokio::time::Instant) -> DbReadinessOutcome { + self.readiness_check_sql(deadline, "SELECT 1").await + } + + /// Production-bound seam for classifying failures after pool acquisition. + /// Tests vary only the SQL so timeout/error/cancellation paths execute the + /// same acquisition and classification code as [`Self::readiness_check`]. + async fn readiness_check_sql( + &self, + deadline: tokio::time::Instant, + query: &'static str, + ) -> DbReadinessOutcome { + let mut connection = match observability::acquire_writer_until( + &self.pool, + observability::WriterOperation::Readiness, + deadline, + ) + .await + { + Err(sqlx::Error::PoolTimedOut) => return DbReadinessOutcome::PoolTimeout, + Err(error) => { + tracing::debug!(error = %error, "Postgres readiness pool acquisition failed"); + return DbReadinessOutcome::PoolError; + } + Ok(connection) => connection, + }; + + match tokio::time::timeout_at(deadline, sqlx::query(query).execute(&mut *connection)).await + { + Err(_) => DbReadinessOutcome::QueryTimeout, + Ok(Err(error)) => { + tracing::debug!(error = %error, "Postgres readiness query failed"); + DbReadinessOutcome::QueryError + } + Ok(Ok(_)) => DbReadinessOutcome::Success, + } } /// Returns pool utilisation stats for metrics emission. @@ -1015,6 +1202,15 @@ impl Db { } } + /// Refresh all expected operation-specific waiter gauges, including zero. + /// + /// The relay pool sampler calls this periodically so an exporter idle + /// timeout cannot make a healthy zero indistinguishable from missing + /// telemetry. + pub fn refresh_pool_waiter_metrics(&self) { + observability::refresh_pool_waiters(self.read_pool.is_some()); + } + /// Pool utilisation stats for the read-replica pool, when configured. /// /// `max` is the **reader's** ceiling ([`Db::read_max_connections`]), not @@ -1035,8 +1231,27 @@ impl Db { /// /// Returns a `'static` transaction because `PgPool` is `Arc`-backed internally. /// The transaction holds an owned pool handle, not a borrow. + pub async fn begin_event_write_transaction( + &self, + ) -> Result> { + let connection = observability::acquire_writer_with_legacy_metrics( + &self.pool, + observability::WriterOperation::EventWrite, + ) + .await?; + sqlx::Transaction::begin(connection, None) + .await + .map_err(Into::into) + } + + /// Begin an event-write transaction through the pre-operation API name. + /// + /// New callers should use [`Self::begin_event_write_transaction`] so the + /// semantic intent is explicit. This alias preserves the crate's public + /// API while emitting the same operation-aware and compatibility metrics. + #[deprecated(note = "use Db::begin_event_write_transaction")] pub async fn begin_transaction(&self) -> Result> { - self.pool.begin().await.map_err(Into::into) + self.begin_event_write_transaction().await } /// Insert an event while holding and validating an admitted serving-write @@ -1061,7 +1276,10 @@ impl Db { return Err(DbError::EphemeralEventRejected(kind_u16)); } - let mut tx = self.pool.begin().await?; + let connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::EventWrite) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; self.deletion_store() .guard_transaction_with_serving_lease(&mut tx, lease) .await?; diff --git a/crates/buzz-db/src/runtime/observability.rs b/crates/buzz-db/src/runtime/observability.rs new file mode 100644 index 000000000..905618891 --- /dev/null +++ b/crates/buzz-db/src/runtime/observability.rs @@ -0,0 +1,1883 @@ +//! Bounded-cardinality database pressure instrumentation primitives. +//! +//! Label values come only from the closed enums in this module. Callers must +//! never derive labels from tenant data, events, SQL text, or query identifiers. + +use std::future::Future; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +/// One valid pool/operation acquisition family. +/// +/// Keeping role and operation in one enum makes invalid combinations +/// unrepresentable at call sites and gives the series budget one exhaustive +/// source of truth. +#[repr(usize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PoolOperation { + WriterBootstrap, + ReaderBootstrap, + WriterReadiness, + WriterTenantResolution, + WriterAuthentication, + WriterAuthorization, + ReaderAuthorization, + WriterSubscriptionHistory, + ReaderSubscriptionHistory, + WriterEventWrite, + WriterMaintenance, +} + +/// Writer-pool operations. Reader-only combinations cannot be constructed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum WriterOperation { + Bootstrap, + Readiness, + TenantResolution, + Authentication, + Authorization, + SubscriptionHistory, + EventWrite, + Maintenance, +} + +impl WriterOperation { + #[cfg(test)] + const ALL: [Self; 8] = [ + Self::Bootstrap, + Self::Readiness, + Self::TenantResolution, + Self::Authentication, + Self::Authorization, + Self::SubscriptionHistory, + Self::EventWrite, + Self::Maintenance, + ]; + + const fn pair(self) -> PoolOperation { + match self { + Self::Bootstrap => PoolOperation::WriterBootstrap, + Self::Readiness => PoolOperation::WriterReadiness, + Self::TenantResolution => PoolOperation::WriterTenantResolution, + Self::Authentication => PoolOperation::WriterAuthentication, + Self::Authorization => PoolOperation::WriterAuthorization, + Self::SubscriptionHistory => PoolOperation::WriterSubscriptionHistory, + Self::EventWrite => PoolOperation::WriterEventWrite, + Self::Maintenance => PoolOperation::WriterMaintenance, + } + } +} + +/// Reader-pool operations. Writer-only combinations cannot be constructed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReaderOperation { + Bootstrap, + /// Upstream constructs this from the replica-routed relay-membership read. + /// Colony's `Db::is_relay_member` still reads the writer directly, so the + /// variant is currently only reachable through the metric contract in + /// [`POOL_ACQUIRE_VALID_PAIRS`]. + #[allow(dead_code)] + Authorization, + SubscriptionHistory, +} + +impl ReaderOperation { + #[cfg(test)] + const ALL: [Self; 3] = [ + Self::Bootstrap, + Self::Authorization, + Self::SubscriptionHistory, + ]; + + const fn pair(self) -> PoolOperation { + match self { + Self::Bootstrap => PoolOperation::ReaderBootstrap, + Self::Authorization => PoolOperation::ReaderAuthorization, + Self::SubscriptionHistory => PoolOperation::ReaderSubscriptionHistory, + } + } +} + +impl PoolOperation { + pub(crate) const ALL: [Self; 11] = [ + Self::WriterBootstrap, + Self::ReaderBootstrap, + Self::WriterReadiness, + Self::WriterTenantResolution, + Self::WriterAuthentication, + Self::WriterAuthorization, + Self::ReaderAuthorization, + Self::WriterSubscriptionHistory, + Self::ReaderSubscriptionHistory, + Self::WriterEventWrite, + Self::WriterMaintenance, + ]; + + pub(crate) const fn pool_role(self) -> &'static str { + match self { + Self::ReaderBootstrap | Self::ReaderAuthorization | Self::ReaderSubscriptionHistory => { + "reader" + } + _ => "writer", + } + } + + pub(crate) const fn operation(self) -> &'static str { + match self { + Self::WriterBootstrap | Self::ReaderBootstrap => "bootstrap", + Self::WriterReadiness => "readiness", + Self::WriterTenantResolution => "tenant_resolution", + Self::WriterAuthentication => "authentication", + Self::WriterAuthorization | Self::ReaderAuthorization => "authorization", + Self::WriterSubscriptionHistory | Self::ReaderSubscriptionHistory => { + "subscription_history" + } + Self::WriterEventWrite => "event_write", + Self::WriterMaintenance => "maintenance", + } + } + + const fn index(self) -> usize { + self as usize + } +} + +pub(crate) const POOL_ACQUIRE_VALID_PAIRS: [(&str, &str); 11] = [ + ("writer", "bootstrap"), + ("reader", "bootstrap"), + ("writer", "readiness"), + ("writer", "tenant_resolution"), + ("writer", "authentication"), + ("writer", "authorization"), + ("reader", "authorization"), + ("writer", "subscription_history"), + ("reader", "subscription_history"), + ("writer", "event_write"), + ("writer", "maintenance"), +]; + +/// Eleven valid pairs × (12 histogram series + 4 outcome counters + 1 gauge). +pub(crate) const POOL_ACQUIRE_RAW_SERIES_PER_POD: usize = POOL_ACQUIRE_VALID_PAIRS.len() * 17; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum LockType { + Replacement, + Membership, + PushGate, + Deletion, + MigrationSchemaSafety, +} + +impl LockType { + #[cfg(test)] + pub(crate) const ALL: [Self; 5] = [ + Self::Replacement, + Self::Membership, + Self::PushGate, + Self::Deletion, + Self::MigrationSchemaSafety, + ]; + + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Replacement => "replacement", + Self::Membership => "membership", + Self::PushGate => "push_gate", + Self::Deletion => "deletion", + Self::MigrationSchemaSafety => "migration_schema_safety", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Outcome { + Success, + Error, + Timeout, + Cancelled, +} + +impl Outcome { + #[cfg(test)] + pub(crate) const ALL: [Self; 4] = [Self::Success, Self::Error, Self::Timeout, Self::Cancelled]; + + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Error => "error", + Self::Timeout => "timeout", + Self::Cancelled => "cancelled", + } + } + + fn from_sqlx_error(error: &sqlx::Error) -> Self { + match error { + sqlx::Error::PoolTimedOut => Self::Timeout, + sqlx::Error::Database(database) if database.code().as_deref() == Some("55P03") => { + Self::Timeout + } + _ => Self::Error, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TransactionOperation { + ReplaceParameterizedEvent, + ReplaceAddressableEvent, + PublishNip43MembershipLocked, + AcceptPushLeaseEvent, + BeginCommunityDeletionQuiescing, + FenceCommunityDeletion, +} + +impl TransactionOperation { + #[cfg(test)] + pub(crate) const ALL: [Self; 6] = [ + Self::ReplaceParameterizedEvent, + Self::ReplaceAddressableEvent, + Self::PublishNip43MembershipLocked, + Self::AcceptPushLeaseEvent, + Self::BeginCommunityDeletionQuiescing, + Self::FenceCommunityDeletion, + ]; + + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::ReplaceParameterizedEvent => "replace_parameterized_event", + Self::ReplaceAddressableEvent => "replace_addressable_event", + Self::PublishNip43MembershipLocked => "publish_nip43_membership_locked", + Self::AcceptPushLeaseEvent => "accept_push_lease_event", + Self::BeginCommunityDeletionQuiescing => "begin_community_deletion_quiescing", + Self::FenceCommunityDeletion => "fence_community_deletion", + } + } + + const fn writer_operation(self) -> WriterOperation { + match self { + Self::ReplaceParameterizedEvent + | Self::ReplaceAddressableEvent + | Self::PublishNip43MembershipLocked + | Self::AcceptPushLeaseEvent => WriterOperation::EventWrite, + Self::BeginCommunityDeletionQuiescing | Self::FenceCommunityDeletion => { + WriterOperation::Maintenance + } + } + } +} + +fn record_pool_acquire( + pair: PoolOperation, + outcome: Outcome, + elapsed: Duration, + emit_legacy: bool, +) { + // Preserve the original observed population for existing dashboards. + // Newly instrumented raw-pool seams must not create a deployment-time + // discontinuity in these compatibility families. + if emit_legacy && outcome != Outcome::Cancelled { + metrics::histogram!( + "buzz_db_pool_acquire_wait_seconds", + "pool_role" => pair.pool_role(), + "outcome" => outcome.as_str(), + ) + .record(elapsed.as_secs_f64()); + metrics::counter!( + "buzz_db_pool_acquisitions_total", + "pool_role" => pair.pool_role(), + "outcome" => outcome.as_str(), + ) + .increment(1); + } + + metrics::histogram!( + "buzz_db_pool_acquire_duration_seconds", + "pool_role" => pair.pool_role(), + "operation" => pair.operation(), + ) + .record(elapsed.as_secs_f64()); + metrics::counter!( + "buzz_db_pool_acquire_attempts_total", + "pool_role" => pair.pool_role(), + "operation" => pair.operation(), + "outcome" => outcome.as_str(), + ) + .increment(1); +} + +static POOL_WAITERS: [Mutex; PoolOperation::ALL.len()] = + [const { Mutex::new(0) }; PoolOperation::ALL.len()]; + +#[cfg(test)] +static POOL_METRICS_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +#[cfg(test)] +#[derive(Clone)] +struct WaiterPublishTestHook { + pair: PoolOperation, + value: u64, + entered: std::sync::Arc, + release: std::sync::Arc, + armed: std::sync::Arc, +} + +#[cfg(test)] +static WAITER_PUBLISH_TEST_HOOK: Mutex> = Mutex::new(None); + +#[cfg(test)] +static WAITER_LAST_PUBLISHED: [Mutex; PoolOperation::ALL.len()] = + [const { Mutex::new(u64::MAX) }; PoolOperation::ALL.len()]; + +fn publish_waiters(pair: PoolOperation, value: u64) { + #[cfg(test)] + { + let hook = WAITER_PUBLISH_TEST_HOOK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(hook) = hook { + if hook.pair == pair + && hook.value == value + && hook.armed.swap(false, std::sync::atomic::Ordering::SeqCst) + { + hook.entered.wait(); + hook.release.wait(); + } + } + *WAITER_LAST_PUBLISHED[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = value; + } + metrics::gauge!( + "buzz_db_pool_waiters", + "pool_role" => pair.pool_role(), + "operation" => pair.operation(), + ) + .set(value as f64); +} + +/// Re-publish every valid waiter pair, including healthy zero, so exporter +/// idle eviction cannot turn an expected zero into ambiguous missing data. +pub(crate) fn refresh_pool_waiters(include_reader: bool) { + for pair in PoolOperation::ALL { + if pair.pool_role() == "reader" && !include_reader { + continue; + } + let waiters = POOL_WAITERS[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + publish_waiters(pair, *waiters); + } +} + +/// Owns one polled connection acquisition until exactly one terminal. +/// +/// Because async function bodies do not run until first poll, a future that is +/// constructed and immediately dropped emits nothing. Once armed, dropping it +/// while awaiting SQLx records `cancelled`, duration, and the balanced waiter +/// decrement. +struct PoolAcquireAttempt { + pair: PoolOperation, + started: Instant, + emit_legacy: bool, + terminal: bool, +} + +impl PoolAcquireAttempt { + fn start(pair: PoolOperation, emit_legacy: bool) -> Self { + { + let mut waiters = POOL_WAITERS[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *waiters += 1; + publish_waiters(pair, *waiters); + } + Self { + pair, + started: Instant::now(), + emit_legacy, + terminal: false, + } + } + + fn finish(mut self, outcome: Outcome) { + self.terminal = true; + record_pool_acquire(self.pair, outcome, self.started.elapsed(), self.emit_legacy); + self.release_waiter(); + } + + fn release_waiter(&self) { + let mut waiters = POOL_WAITERS[self.pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + debug_assert!(*waiters > 0, "pool waiter balance underflow"); + *waiters = waiters.saturating_sub(1); + publish_waiters(self.pair, *waiters); + } +} + +impl Drop for PoolAcquireAttempt { + fn drop(&mut self) { + if !self.terminal { + record_pool_acquire( + self.pair, + Outcome::Cancelled, + self.started.elapsed(), + self.emit_legacy, + ); + self.release_waiter(); + self.terminal = true; + } + } +} + +async fn acquire( + pool: &sqlx::PgPool, + pair: PoolOperation, + emit_legacy: bool, +) -> sqlx::Result> { + let attempt = PoolAcquireAttempt::start(pair, emit_legacy); + let result = pool.acquire().await; + let outcome = result + .as_ref() + .map(|_| Outcome::Success) + .unwrap_or_else(Outcome::from_sqlx_error); + attempt.finish(outcome); + result +} + +/// Acquire from an authoritative writer pool for one valid writer operation. +pub(crate) async fn acquire_writer( + pool: &sqlx::PgPool, + operation: WriterOperation, +) -> sqlx::Result> { + acquire(pool, operation.pair(), false).await +} + +/// Acquire from a writer seam already covered by the pre-operation metric. +pub(crate) async fn acquire_writer_with_legacy_metrics( + pool: &sqlx::PgPool, + operation: WriterOperation, +) -> sqlx::Result> { + acquire(pool, operation.pair(), true).await +} + +/// Acquire from a reader seam already covered by the pre-operation metric. +pub(super) async fn acquire_reader_with_legacy_metrics( + pool: &sqlx::PgPool, + operation: ReaderOperation, +) -> sqlx::Result> { + acquire(pool, operation.pair(), true).await +} + +/// Acquire within an operation-owned absolute deadline. +/// +/// A deadline expiry is a timeout terminal. Dropping the enclosing future +/// before that deadline remains a cancellation terminal. +pub(crate) async fn acquire_writer_until( + pool: &sqlx::PgPool, + operation: WriterOperation, + deadline: tokio::time::Instant, +) -> sqlx::Result> { + let pair = operation.pair(); + let attempt = PoolAcquireAttempt::start(pair, false); + match tokio::time::timeout_at(deadline, pool.acquire()).await { + Err(_) => { + attempt.finish(Outcome::Timeout); + Err(sqlx::Error::PoolTimedOut) + } + Ok(result) => { + let outcome = result + .as_ref() + .map(|_| Outcome::Success) + .unwrap_or_else(Outcome::from_sqlx_error); + attempt.finish(outcome); + result + } + } +} + +pub(crate) async fn begin_transaction( + pool: &sqlx::PgPool, + operation: TransactionOperation, +) -> sqlx::Result<(sqlx::Transaction<'static, sqlx::Postgres>, TransactionTimer)> { + let connection = acquire_writer_with_legacy_metrics(pool, operation.writer_operation()).await?; + let transaction = sqlx::Transaction::begin(connection, None).await?; + Ok((transaction, TransactionTimer::start(operation))) +} + +pub(crate) async fn observe_advisory_lock(lock_type: LockType, future: F) -> sqlx::Result +where + F: Future>, +{ + let started = Instant::now(); + let result = future.await; + let outcome = result + .as_ref() + .map(|_| Outcome::Success) + .unwrap_or_else(Outcome::from_sqlx_error); + metrics::histogram!( + "buzz_db_advisory_lock_wait_seconds", + "lock_type" => lock_type.as_str(), + "outcome" => outcome.as_str(), + ) + .record(started.elapsed().as_secs_f64()); + metrics::counter!( + "buzz_db_advisory_lock_acquisitions_total", + "lock_type" => lock_type.as_str(), + "outcome" => outcome.as_str(), + ) + .increment(1); + result +} + +pub(crate) struct TransactionTimer { + operation: TransactionOperation, + started: Instant, + outcome: Outcome, +} + +impl TransactionTimer { + pub(crate) fn start(operation: TransactionOperation) -> Self { + Self { + operation, + started: Instant::now(), + outcome: Outcome::Error, + } + } + + pub(crate) async fn observe(mut self, future: F) -> Result + where + F: Future>, + { + let result = future.await; + if result.is_ok() { + self.outcome = Outcome::Success; + } + result + } +} + +impl Drop for TransactionTimer { + fn drop(&mut self) { + metrics::histogram!( + "buzz_db_transaction_duration_seconds", + "operation" => self.operation.as_str(), + "outcome" => self.outcome.as_str(), + ) + .record(self.started.elapsed().as_secs_f64()); + } +} + +#[cfg(test)] +mod tests { + use super::{ + acquire_reader_with_legacy_metrics, acquire_writer, acquire_writer_with_legacy_metrics, + observe_advisory_lock, record_pool_acquire, refresh_pool_waiters, LockType, Outcome, + PoolAcquireAttempt, PoolOperation, ReaderOperation, TransactionOperation, TransactionTimer, + WriterOperation, + }; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + use std::collections::{BTreeMap, BTreeSet}; + use std::sync::{Arc, Barrier}; + use std::time::Duration; + + #[test] + fn label_vocabularies_are_closed_and_documented() { + assert_eq!( + PoolOperation::ALL.map(|pair| (pair.pool_role(), pair.operation())), + super::POOL_ACQUIRE_VALID_PAIRS + ); + assert_eq!( + WriterOperation::ALL.map(WriterOperation::pair), + [ + PoolOperation::WriterBootstrap, + PoolOperation::WriterReadiness, + PoolOperation::WriterTenantResolution, + PoolOperation::WriterAuthentication, + PoolOperation::WriterAuthorization, + PoolOperation::WriterSubscriptionHistory, + PoolOperation::WriterEventWrite, + PoolOperation::WriterMaintenance, + ] + ); + assert_eq!( + ReaderOperation::ALL.map(ReaderOperation::pair), + [ + PoolOperation::ReaderBootstrap, + PoolOperation::ReaderAuthorization, + PoolOperation::ReaderSubscriptionHistory, + ] + ); + assert_eq!(super::POOL_ACQUIRE_RAW_SERIES_PER_POD, 187); + assert_eq!( + LockType::ALL.map(LockType::as_str), + [ + "replacement", + "membership", + "push_gate", + "deletion", + "migration_schema_safety", + ] + ); + assert_eq!( + Outcome::ALL.map(Outcome::as_str), + ["success", "error", "timeout", "cancelled"] + ); + assert_eq!( + TransactionOperation::ALL.map(TransactionOperation::as_str), + [ + "replace_parameterized_event", + "replace_addressable_event", + "publish_nip43_membership_locked", + "accept_push_lease_event", + "begin_community_deletion_quiescing", + "fence_community_deletion", + ] + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn transaction_timer_observe_classifies_result_outcomes() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let success = TransactionTimer::start(TransactionOperation::ReplaceParameterizedEvent) + .observe(async { Ok::<_, &str>("committed") }) + .await; + assert_eq!(success, Ok("committed")); + + let error = TransactionTimer::start(TransactionOperation::AcceptPushLeaseEvent) + .observe(async { Err::<(), _>("rollback") }) + .await; + assert_eq!(error, Err("rollback")); + + let keys = snapshotter + .snapshot() + .into_vec() + .into_iter() + .map(|(key, ..)| { + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + (key.key().name().to_owned(), labels) + }) + .collect::>(); + + for (operation, outcome) in [ + ("replace_parameterized_event", "success"), + ("accept_push_lease_event", "error"), + ] { + assert!(keys.contains(&( + "buzz_db_transaction_duration_seconds".to_owned(), + [ + ("operation".to_owned(), operation.to_owned()), + ("outcome".to_owned(), outcome.to_owned()), + ] + .into_iter() + .collect(), + ))); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn primitives_record_fixed_success_error_and_timeout_labels() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + record_pool_acquire( + PoolOperation::WriterReadiness, + Outcome::Success, + Duration::from_millis(12), + false, + ); + record_pool_acquire( + PoolOperation::ReaderSubscriptionHistory, + Outcome::Timeout, + Duration::from_millis(34), + true, + ); + let lock_ok: sqlx::Result<()> = + observe_advisory_lock(LockType::Replacement, async { Ok(()) }).await; + assert!(lock_ok.is_ok()); + let lock_error: sqlx::Result<()> = + observe_advisory_lock(LockType::Membership, async { Err(sqlx::Error::PoolClosed) }) + .await; + assert!(lock_error.is_err()); + + let committed: Result<(), ()> = + TransactionTimer::start(TransactionOperation::ReplaceParameterizedEvent) + .observe(async { Ok(()) }) + .await; + assert!(committed.is_ok()); + let rolled_back: Result<(), ()> = + TransactionTimer::start(TransactionOperation::AcceptPushLeaseEvent) + .observe(async { Err(()) }) + .await; + assert!(rolled_back.is_err()); + + let snapshot = snapshotter.snapshot().into_vec(); + let keys = snapshot + .iter() + .map(|(key, ..)| { + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + (key.key().name().to_owned(), labels) + }) + .collect::>(); + + for expected in [ + ( + "buzz_db_pool_acquire_wait_seconds", + [("outcome", "timeout"), ("pool_role", "reader")], + ), + ( + "buzz_db_pool_acquisitions_total", + [("outcome", "timeout"), ("pool_role", "reader")], + ), + ( + "buzz_db_advisory_lock_wait_seconds", + [("lock_type", "replacement"), ("outcome", "success")], + ), + ( + "buzz_db_advisory_lock_wait_seconds", + [("lock_type", "membership"), ("outcome", "error")], + ), + ( + "buzz_db_advisory_lock_acquisitions_total", + [("lock_type", "replacement"), ("outcome", "success")], + ), + ( + "buzz_db_advisory_lock_acquisitions_total", + [("lock_type", "membership"), ("outcome", "error")], + ), + ( + "buzz_db_transaction_duration_seconds", + [ + ("operation", "replace_parameterized_event"), + ("outcome", "success"), + ], + ), + ( + "buzz_db_transaction_duration_seconds", + [ + ("operation", "accept_push_lease_event"), + ("outcome", "error"), + ], + ), + ] { + let expected_labels = expected + .1 + .into_iter() + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect::>(); + assert!( + keys.contains(&(expected.0.to_owned(), expected_labels)), + "missing metric series {expected:?}; got {keys:?}" + ); + } + for name in [ + "buzz_db_pool_acquire_wait_seconds", + "buzz_db_pool_acquisitions_total", + ] { + assert!( + !keys.contains(&( + name.to_owned(), + [ + ("outcome".to_owned(), "success".to_owned()), + ("pool_role".to_owned(), "writer".to_owned()), + ] + .into_iter() + .collect(), + )), + "newly instrumented seams must not expand legacy metric population" + ); + } + + for (name, labels) in [ + ( + "buzz_db_pool_acquire_duration_seconds", + [("operation", "readiness"), ("pool_role", "writer")], + ), + ( + "buzz_db_pool_acquire_duration_seconds", + [ + ("operation", "subscription_history"), + ("pool_role", "reader"), + ], + ), + ] { + let labels = labels + .into_iter() + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect(); + assert!( + keys.contains(&(name.to_owned(), labels)), + "missing operation-aware pool duration for {name}" + ); + } + for (pool_role, operation, outcome) in [ + ("writer", "readiness", "success"), + ("reader", "subscription_history", "timeout"), + ] { + assert!(keys.contains(&( + "buzz_db_pool_acquire_attempts_total".to_owned(), + [ + ("operation".to_owned(), operation.to_owned()), + ("outcome".to_owned(), outcome.to_owned()), + ("pool_role".to_owned(), pool_role.to_owned()), + ] + .into_iter() + .collect(), + ))); + } + assert!(keys.iter().all(|(name, labels)| { + name != "buzz_db_pool_acquire_duration_seconds" + || (!labels.contains_key("outcome") && !labels.contains_key("result")) + })); + + for (key, _, _, value) in snapshot { + if key.key().name().ends_with("_seconds") { + let DebugValue::Histogram(samples) = value else { + panic!("seconds metrics must be histograms"); + }; + assert!(samples.iter().all(|sample| sample.into_inner() >= 0.0)); + } else if key.key().name().ends_with("_total") { + let DebugValue::Counter(value) = value else { + panic!("total metrics must be counters"); + }; + assert_eq!(value, 1); + } + } + } + + #[test] + fn cancelled_attempt_records_terminal_and_refreshes_zero() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let attempt = PoolAcquireAttempt::start(PoolOperation::WriterTenantResolution, false); + drop(attempt); + refresh_pool_waiters(true); + + let mut saw_cancelled = false; + let mut saw_zero = false; + for (key, _, _, value) in snapshotter.snapshot().into_vec() { + let labels = key + .key() + .labels() + .map(|label| (label.key(), label.value())) + .collect::>(); + if labels.get("operation") != Some(&"tenant_resolution") { + continue; + } + match key.key().name() { + "buzz_db_pool_acquire_attempts_total" => { + let DebugValue::Counter(value) = value else { + panic!("attempt terminals must be a counter"); + }; + saw_cancelled = labels.get("outcome") == Some(&"cancelled") && value == 1; + } + "buzz_db_pool_waiters" => { + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be a gauge"); + }; + saw_zero = value.into_inner() == 0.0; + } + _ => {} + } + } + assert!( + saw_cancelled, + "dropped armed attempt must terminalize cancellation" + ); + assert!(saw_zero, "periodic refresh must publish a healthy zero"); + } + + #[test] + fn waiter_refresh_omits_reader_pairs_when_no_reader_pool_is_configured() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + refresh_pool_waiters(false); + + let published = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_waiters" { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be gauges"); + }; + assert_eq!(value.into_inner(), 0.0); + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + Some((labels["pool_role"].clone(), labels["operation"].clone())) + }) + .collect::>(); + let expected = WriterOperation::ALL + .into_iter() + .map(|operation| { + let pair = operation.pair(); + (pair.pool_role().to_owned(), pair.operation().to_owned()) + }) + .collect::>(); + + assert_eq!(published, expected); + assert!(published.iter().all(|(pool_role, _)| pool_role == "writer")); + } + + #[tokio::test(flavor = "current_thread")] + async fn compatibility_metrics_only_cover_preexisting_acquisition_seams() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.lock().await; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy(&crate::test_support::database_url()) + .expect("construct lazy compatibility test pool"); + pool.close().await; + + let error = acquire_writer(&pool, WriterOperation::EventWrite) + .await + .expect_err("closed newly instrumented seam errors"); + assert!(matches!(error, sqlx::Error::PoolClosed)); + assert_eq!( + legacy_acquisition_count(&snapshotter.snapshot().into_vec()), + 0 + ); + + let error = acquire_writer_with_legacy_metrics(&pool, WriterOperation::EventWrite) + .await + .expect_err("closed legacy seam errors"); + assert!(matches!(error, sqlx::Error::PoolClosed)); + assert_eq!( + legacy_acquisition_count(&snapshotter.snapshot().into_vec()), + 1 + ); + } + + fn legacy_acquisition_count( + snapshot: &[( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + )], + ) -> u64 { + snapshot + .iter() + .filter_map(|(key, _, _, value)| { + (key.key().name() == "buzz_db_pool_acquisitions_total") + .then_some(value) + .map(|value| match value { + DebugValue::Counter(value) => *value, + _ => panic!("legacy acquisitions must be a counter"), + }) + }) + .sum() + } + + #[test] + fn concurrent_attempts_publish_an_exact_balanced_waiter_count() { + const ATTEMPTS: usize = 8; + + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let armed = Arc::new(Barrier::new(ATTEMPTS + 1)); + let release = Arc::new(Barrier::new(ATTEMPTS + 1)); + let threads = (0..ATTEMPTS) + .map(|_| { + let armed = Arc::clone(&armed); + let release = Arc::clone(&release); + std::thread::spawn(move || { + let attempt = + PoolAcquireAttempt::start(PoolOperation::WriterTenantResolution, false); + armed.wait(); + release.wait(); + drop(attempt); + }) + }) + .collect::>(); + + armed.wait(); + refresh_pool_waiters(true); + let live = waiter_value( + &snapshotter.snapshot().into_vec(), + "writer", + "tenant_resolution", + ); + assert_eq!(live, Some(ATTEMPTS as f64)); + + release.wait(); + for thread in threads { + thread.join().expect("waiter thread completes"); + } + refresh_pool_waiters(true); + let balanced = waiter_value( + &snapshotter.snapshot().into_vec(), + "writer", + "tenant_resolution", + ); + assert_eq!(balanced, Some(0.0)); + } + + #[test] + fn waiter_publication_is_serialized_with_state_mutation() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let pair = PoolOperation::WriterTenantResolution; + let first = PoolAcquireAttempt::start(pair, false); + let second = PoolAcquireAttempt::start(pair, false); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + *super::WAITER_PUBLISH_TEST_HOOK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(super::WaiterPublishTestHook { + pair, + value: 1, + entered: Arc::clone(&entered), + release: Arc::clone(&release), + armed: Arc::new(std::sync::atomic::AtomicBool::new(true)), + }); + + let first_drop = std::thread::spawn(move || drop(first)); + entered.wait(); + let mutation_lock_held = super::POOL_WAITERS[pair.index()].try_lock().is_err(); + release.wait(); + first_drop.join().expect("first drop completes"); + drop(second); + *super::WAITER_PUBLISH_TEST_HOOK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + + assert!( + mutation_lock_held, + "waiter state mutation must remain locked until its publication completes" + ); + assert_eq!( + *super::WAITER_LAST_PUBLISHED[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + 0, + "the final directly published waiter value must be balanced without a refresh" + ); + } + + fn waiter_value( + snapshot: &[( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + )], + pool_role: &str, + operation: &str, + ) -> Option { + snapshot.iter().find_map(|(key, _, _, value)| { + let labels = key.key().labels().collect::>(); + if key.key().name() != "buzz_db_pool_waiters" + || !labels + .iter() + .any(|label| label.key() == "pool_role" && label.value() == pool_role) + || !labels + .iter() + .any(|label| label.key() == "operation" && label.value() == operation) + { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be gauges"); + }; + Some(value.into_inner()) + }) + } + + async fn pool_acquire_records_success_timeout_and_error_with_wait_time() { + // This timeout also bounds the pool's initial connection. Leave enough + // headroom for a cold PostgreSQL start under the lane's eight workers; + // the assertion below cares about classification, not a sub-second + // synthetic timeout budget. + let database_url = crate::test_support::database_url(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(5)) + .connect(&database_url) + .await + .expect("connect size-one test pool"); + let reader_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(5)) + .connect(&database_url) + .await + .expect("connect size-one reader test pool"); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let held = acquire_writer_with_legacy_metrics(&pool, WriterOperation::EventWrite) + .await + .expect("writer acquire succeeds"); + let mut cancelled = Box::pin(acquire_writer_with_legacy_metrics( + &pool, + WriterOperation::Authentication, + )); + tokio::select! { + result = &mut cancelled => panic!("blocked acquisition unexpectedly completed: {result:?}"), + () = tokio::time::sleep(Duration::from_millis(40)) => {} + } + let before_cancel = snapshotter.snapshot().into_vec(); + let live_waiter = before_cancel.iter().find_map(|(key, _, _, value)| { + let labels = key.key().labels().collect::>(); + if key.key().name() != "buzz_db_pool_waiters" + || !labels + .iter() + .any(|label| label.key() == "pool_role" && label.value() == "writer") + || !labels + .iter() + .any(|label| label.key() == "operation" && label.value() == "authentication") + { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be a gauge"); + }; + Some(value.into_inner()) + }); + assert_eq!(live_waiter, Some(1.0)); + let legacy_before_cancel = legacy_acquisition_count(&before_cancel); + assert_eq!( + legacy_before_cancel, 1, + "the completed legacy acquisition must be counted exactly once" + ); + let writer_success = before_cancel.iter().any(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_wait_seconds" { + return false; + } + let labels = key.key().labels().collect::>(); + let DebugValue::Histogram(samples) = value else { + panic!("pool wait must be a histogram"); + }; + labels + .iter() + .any(|label| label.key() == "pool_role" && label.value() == "writer") + && labels + .iter() + .any(|label| label.key() == "outcome" && label.value() == "success") + && !samples.is_empty() + }); + drop(cancelled); + let after_cancel = snapshotter.snapshot().into_vec(); + let legacy_after_cancel = legacy_acquisition_count(&after_cancel); + let mut cancelled_terminal = None; + let mut balanced_waiter = None; + for (key, _, _, value) in after_cancel { + let labels = key.key().labels().collect::>(); + let label = |name: &str| { + labels + .iter() + .find(|label| label.key() == name) + .map(|label| label.value()) + }; + if label("pool_role") != Some("writer") || label("operation") != Some("authentication") + { + continue; + } + match key.key().name() { + "buzz_db_pool_waiters" => { + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be a gauge"); + }; + balanced_waiter = Some(value.into_inner()); + } + "buzz_db_pool_acquire_attempts_total" if label("outcome") == Some("cancelled") => { + let DebugValue::Counter(value) = value else { + panic!("cancelled acquisition terminal must be a counter"); + }; + cancelled_terminal = Some(value); + } + _ => {} + } + } + assert_eq!(balanced_waiter, Some(0.0)); + assert_eq!(cancelled_terminal, Some(1)); + assert_eq!( + legacy_after_cancel, 0, + "cancelling a legacy seam must not expand its historical population" + ); + let held_reader = reader_pool + .acquire() + .await + .expect("hold the reader test connection"); + let timeout = + acquire_reader_with_legacy_metrics(&reader_pool, ReaderOperation::SubscriptionHistory) + .await + .expect_err("reader-labeled checkout times out while pool is saturated"); + assert!(matches!(timeout, sqlx::Error::PoolTimedOut)); + drop(held_reader); + drop(held); + pool.close().await; + let closed = acquire_writer_with_legacy_metrics(&pool, WriterOperation::Readiness) + .await + .expect_err("closed pool acquire errors"); + assert!(matches!(closed, sqlx::Error::PoolClosed)); + + let mut outcomes = BTreeMap::<(String, String), Vec>::new(); + for (key, _, _, value) in snapshotter.snapshot().into_vec() { + let labels = key.key().labels().collect::>(); + let label = |name: &str| { + labels + .iter() + .find(|label| label.key() == name) + .map(|label| label.value().to_owned()) + .unwrap_or_default() + }; + if key.key().name() == "buzz_db_pool_acquire_wait_seconds" { + let DebugValue::Histogram(samples) = value else { + panic!("pool wait must be a histogram"); + }; + if samples.is_empty() { + continue; + } + outcomes.insert( + (label("pool_role"), label("outcome")), + samples + .into_iter() + .map(|sample| sample.into_inner()) + .collect(), + ); + } + } + assert!(writer_success); + assert!( + !outcomes.contains_key(&("writer".to_owned(), "cancelled".to_owned())), + "legacy compatibility families must not add a cancellation population" + ); + assert!(outcomes.contains_key(&("writer".to_owned(), "error".to_owned()))); + let timeout_samples = outcomes + .get(&("reader".to_owned(), "timeout".to_owned())) + .expect("reader timeout series"); + assert!( + timeout_samples.iter().any(|sample| *sample >= 0.05), + "timeout wait must include the saturated checkout delay: {timeout_samples:?}" + ); + } + + async fn deletion_catalog_readiness_records_timeout_and_recovers() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.lock().await; + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(5)) + .connect(&crate::test_support::database_url()) + .await + .expect("connect size-one deletion readiness pool"); + let db = crate::Db::from_pool(pool.clone()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let held = pool.acquire().await.expect("hold the only pool connection"); + let timeout = db + .validate_deletion_serving_catalog_for_readiness( + tokio::time::Instant::now() + Duration::from_millis(40), + ) + .await + .expect_err("saturated deletion catalog checkout must time out"); + assert!(matches!( + timeout, + crate::DbError::Sqlx(sqlx::Error::PoolTimedOut) + )); + drop(held); + db.validate_deletion_serving_catalog_for_readiness( + tokio::time::Instant::now() + Duration::from_secs(2), + ) + .await + .expect("deletion catalog readiness must recover after pool release"); + + let snapshot = snapshotter.snapshot().into_vec(); + assert_eq!( + waiter_value(&snapshot, "writer", "readiness"), + Some(0.0), + "deadline terminal must directly balance the readiness waiter" + ); + for outcome in ["timeout", "success"] { + assert!( + snapshot.iter().any(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_attempts_total" { + return false; + } + let labels = key.key().labels().collect::>(); + let has = |name: &str, expected: &str| { + labels + .iter() + .any(|label| label.key() == name && label.value() == expected) + }; + has("pool_role", "writer") + && has("operation", "readiness") + && has("outcome", outcome) + && matches!(value, DebugValue::Counter(1)) + }), + "missing writer/readiness/{outcome} acquisition terminal" + ); + } + } + + async fn production_db_methods_emit_exact_pool_operation_labels() { + use buzz_core::CommunityId; + use chrono::Utc; + use uuid::Uuid; + + let database_url = crate::test_support::database_url(); + // Build the writer pool the way production does. `connect_writer_pool` + // is the only constructor that arms `buzz.created_at_floor`, the three + // session timeouts and the isolation assertion in its single + // `after_connect` hook, so a bare `PgPoolOptions` here would exercise + // checkout attribution against a pool the relay never creates - and + // `verify_replica_fence_at_boot` below fails closed on the missing GUC. + let writer_pool = crate::Db::connect_writer_pool(&crate::DbConfig { + database_url: database_url.clone(), + max_connections: 4, + min_connections: 0, + acquire_timeout_secs: 5, + ..crate::DbConfig::default() + }) + .await + .expect("connect production-method writer pool"); + let reader_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(4) + .acquire_timeout(Duration::from_secs(5)) + .connect(&database_url) + .await + .expect("connect production-method reader pool"); + let writer_db = crate::Db::from_pool(writer_pool.clone()); + let mut routed_db = crate::Db::from_pools(writer_pool.clone(), reader_pool); + routed_db.set_replica_read_max_age_for_tests(Some(Duration::from_secs(5))); + + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let test_scope = CommunityId::from_uuid(Uuid::new_v4()); + let query = crate::EventQuery::for_community(test_scope); + + routed_db.read_pool_boot_ping_for_tests().await; + routed_db + .verify_replica_fence_at_boot() + .await + .expect("real startup fence verification succeeds"); + let _ = crate::replica_fence::probe_once(&writer_pool, routed_db.fence()).await; + routed_db.fence().force_open_for_tests(Utc::now()); + assert_eq!( + writer_db + .readiness_check(tokio::time::Instant::now() + Duration::from_secs(1)) + .await, + crate::DbReadinessOutcome::Success + ); + let _ = writer_db + .lookup_community_by_host("pool-operation-matrix.invalid") + .await; + let _ = writer_db + .lookup_community_by_host_for_management("pool-operation-matrix.invalid") + .await; + let _ = writer_db.list_communities_owned_by(&"a".repeat(64)).await; + let _ = writer_db.lookup_community_host(test_scope).await; + let _ = writer_db + .set_community_icon(test_scope, Some("pool-operation-matrix")) + .await; + let _ = writer_db + .create_community_with_owner( + &format!("pool-operation-matrix-{}.invalid", Uuid::new_v4().simple()), + &"b".repeat(64), + ) + .await; + let _ = writer_db + .archive_community_owned_by( + "pool-operation-matrix.invalid", + &"c".repeat(64), + "protected.invalid", + ) + .await; + let _ = writer_db + .unarchive_community_owned_by("pool-operation-matrix.invalid", &"c".repeat(64)) + .await; + let _ = writer_db.community_of_channel(Uuid::new_v4()).await; + let _ = writer_db.communities_of_channels(&[Uuid::new_v4()]).await; + let _ = writer_db + .ensure_user_for_authorization(test_scope, &[17; 32]) + .await; + let _ = writer_db + .set_agent_owner_for_authorization(test_scope, &[18; 32], &[19; 32]) + .await; + let _ = writer_db.is_pubkey_allowed(test_scope, &[7; 32]).await; + let _ = writer_db + .is_agent_owner(test_scope, &[8; 32], &[9; 32]) + .await; + let _ = writer_db + .moderation_restriction_state(test_scope, &[14; 32]) + .await; + let _ = writer_db + .get_agent_channel_policy(test_scope, &[15; 32]) + .await; + let _ = writer_db + .get_thread_metadata_by_event(test_scope, &[10; 32]) + .await; + let _ = writer_db.get_thread_summary(test_scope, &[16; 32]).await; + let _ = writer_db + .get_channel_for_event_write(test_scope, Uuid::new_v4()) + .await; + let _ = writer_db + .get_members_for_event_write(test_scope, Uuid::new_v4()) + .await; + let _ = writer_db + .get_users_bulk_for_event_write(test_scope, &[vec![11; 32]]) + .await; + let _ = writer_db + .huddle_started_link_exists_for_event_write( + test_scope, + Uuid::new_v4(), + Uuid::new_v4(), + &[12; 32], + ) + .await; + let _ = writer_db + .huddle_started_link_exists(test_scope, Uuid::new_v4(), Uuid::new_v4(), &[13; 32]) + .await; + let _ = writer_db.list_archived(test_scope).await; + let _ = writer_db + .query_events_routed("pool_operation_matrix_writer", &query) + .await; + let write_tx = writer_db + .begin_event_write_transaction() + .await + .expect("event-write semantic entry point begins a real transaction"); + write_tx + .rollback() + .await + .expect("rollback operation-label fixture"); + let _ = writer_db + .is_community_active_for_maintenance(test_scope) + .await; + let _ = writer_db.usage_community_count().await; + let _ = writer_db.reap_expired_ephemeral_channels().await; + let deletion_store = writer_db.deletion_store(); + let _ = deletion_store.reap_expired_serving_write_leases(1).await; + let _ = deletion_store.serving_lease_stats().await; + let _ = routed_db.is_relay_member(test_scope, &"a".repeat(64)).await; + let _ = routed_db + .query_events_routed("pool_operation_matrix_reader", &query) + .await; + routed_db.refresh_pool_waiter_metrics(); + + let snapshot = snapshotter.snapshot().into_vec(); + let attempt_labels = snapshot + .iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_attempts_total" { + return None; + } + let DebugValue::Counter(value) = value else { + panic!("acquisition attempts must be counters"); + }; + if *value == 0 { + return None; + } + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + assert_eq!(labels.get("outcome").map(String::as_str), Some("success")); + Some((labels["pool_role"].clone(), labels["operation"].clone())) + }) + .collect::>(); + // Upstream exercises all eleven valid pairs here. Colony reaches ten: + // `reader/authorization` is the one pair no Colony production method + // can emit, because `Db::is_relay_member` reads the writer directly. + // Upstream routes that single permission read on the bounded replica + // arm "by explicit product decision", and its own comment calls it + // "not precedent for routing other permission reads"; Colony has never + // made that decision (no `route_read` has ever existed in + // relay_members.rs here, at this branch or before the Phase 2 store + // split). Routing it would make a revoked membership readable for up + // to the freshness budget, which is a product call, not a port detail. + // + // The pair stays in POOL_ACQUIRE_VALID_PAIRS and in the 187-series + // budget: that vocabulary is what the label space ALLOWS, and + // `ReaderOperation::Authorization` remains constructible. Only this + // test, which asserts what production actually emits, is narrowed. + // Delete the exclusion the moment `is_relay_member` starts routing. + const UNREACHABLE_IN_COLONY: (&str, &str) = ("reader", "authorization"); + assert!( + super::POOL_ACQUIRE_VALID_PAIRS.contains(&UNREACHABLE_IN_COLONY), + "the excluded pair must still be a valid label combination" + ); + let expected = super::POOL_ACQUIRE_VALID_PAIRS + .into_iter() + .map(|(pool_role, operation)| (pool_role.to_owned(), operation.to_owned())) + .collect::>(); + let reached = expected + .iter() + .filter(|(pool_role, operation)| { + (pool_role.as_str(), operation.as_str()) != UNREACHABLE_IN_COLONY + }) + .cloned() + .collect::>(); + assert_eq!( + attempt_labels, reached, + "real production Db/store methods must emit every exact valid operation pair \ + that Colony can reach" + ); + + let duration_labels = snapshot + .iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_duration_seconds" { + return None; + } + let DebugValue::Histogram(samples) = value else { + panic!("acquisition duration must be a histogram"); + }; + assert!(!samples.is_empty()); + let labels = key + .key() + .labels() + .map(|label| (label.key(), label.value())) + .collect::>(); + assert!(!labels.contains_key("outcome")); + Some(( + labels["pool_role"].to_owned(), + labels["operation"].to_owned(), + )) + }) + .collect::>(); + assert_eq!(duration_labels, reached); + + let waiter_labels = snapshot + .iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_waiters" { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be gauges"); + }; + assert_eq!(value.into_inner(), 0.0); + let labels = key + .key() + .labels() + .map(|label| (label.key(), label.value())) + .collect::>(); + Some(( + labels["pool_role"].to_owned(), + labels["operation"].to_owned(), + )) + }) + .collect::>(); + // The waiter gauge is published for EVERY valid pair, including the one + // no Colony method reaches: `refresh_pool_waiters` exists so a healthy + // zero is distinguishable from a missing series, which only works if + // the full vocabulary is emitted. + assert_eq!(waiter_labels, expected); + } + + async fn serving_write_gate_records_cancel_timeout_success_and_recovery() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.lock().await; + let database_url = crate::test_support::database_url(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + // The same budget also covers the pool's initial physical + // connection. Keep enough headroom for a cold CI database; the + // held size-one connection below still deterministically drives + // the checkout timeout terminal. + .acquire_timeout(Duration::from_secs(1)) + .connect(&database_url) + .await + .expect("connect size-one serving-write test pool"); + let db = crate::Db::from_pool(pool.clone()); + if std::env::var("BUZZ_TEST_SCHEMA_MODE").as_deref() != Ok("desired") { + db.migrate().await.expect("migrate serving-write test DB"); + } + let test_scope = db + .ensure_configured_community(&format!( + "pool-observability-{}.example", + uuid::Uuid::new_v4().simple() + )) + .await + .expect("create serving-write test community") + .id; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let held = pool.acquire().await.expect("hold sole writer connection"); + + let store = db.deletion_store(); + let mut cancelled = Box::pin(store.is_serving_active(test_scope)); + tokio::select! { + result = &mut cancelled => panic!("blocked serving-write gate unexpectedly completed: {result:?}"), + () = tokio::time::sleep(Duration::from_millis(25)) => {} + } + assert_eq!( + waiter_value(&snapshotter.snapshot().into_vec(), "writer", "event_write"), + Some(1.0) + ); + drop(cancelled); + + let timeout = store + .is_serving_active(test_scope) + .await + .expect_err("saturated serving-write gate times out"); + assert!(matches!( + timeout, + crate::DbError::Sqlx(sqlx::Error::PoolTimedOut) + )); + drop(held); + + assert!(store + .is_serving_active(test_scope) + .await + .expect("serving-write gate recovers after release")); + let lease = store + .acquire_serving_write_lease( + test_scope, + "pool_observability", + "pool-observability-test", + Duration::from_secs(5), + ) + .await + .expect("serving-write lease acquires through event-write seam"); + assert!(store + .release_serving_write_lease(&lease) + .await + .expect("serving-write lease release")); + refresh_pool_waiters(false); + + let snapshot = snapshotter.snapshot().into_vec(); + assert_eq!(waiter_value(&snapshot, "writer", "event_write"), Some(0.0)); + assert_eq!(attempt_count(&snapshot, "event_write", "cancelled"), 1); + assert_eq!(attempt_count(&snapshot, "event_write", "timeout"), 1); + assert!( + attempt_count(&snapshot, "event_write", "success") >= 3, + "gate recovery plus lease acquire/release must emit successes" + ); + } + + fn attempt_count( + snapshot: &[( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + )], + operation: &str, + outcome: &str, + ) -> u64 { + snapshot + .iter() + .find_map(|(key, _, _, value)| { + let labels = key.key().labels().collect::>(); + (key.key().name() == "buzz_db_pool_acquire_attempts_total" + && labels + .iter() + .any(|label| label.key() == "operation" && label.value() == operation) + && labels + .iter() + .any(|label| label.key() == "outcome" && label.value() == outcome)) + .then(|| match value { + DebugValue::Counter(value) => *value, + _ => panic!("pool attempts must be a counter"), + }) + }) + .unwrap_or(0) + } + + async fn advisory_lock_records_success_contention_timeout_and_error() { + let database_url = crate::test_support::database_url(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(4) + .connect(&database_url) + .await + .expect("connect advisory-lock test pool"); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let mut success_tx = pool.begin().await.expect("begin success transaction"); + observe_advisory_lock( + LockType::Replacement, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(0x62757a7a6f627331_i64) + .execute(&mut *success_tx), + ) + .await + .expect("uncontended lock succeeds"); + success_tx + .rollback() + .await + .expect("rollback success transaction"); + + let contention_key = 0x62757a7a6f627332_i64; + let mut holder = pool.begin().await.expect("begin lock holder"); + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(contention_key) + .execute(&mut *holder) + .await + .expect("holder acquires contention key"); + let mut waiter = pool.begin().await.expect("begin lock waiter"); + let waiter_task = tokio::spawn(async move { + let result = observe_advisory_lock( + LockType::Deletion, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(contention_key) + .execute(&mut *waiter), + ) + .await; + (waiter, result) + }); + tokio::time::sleep(Duration::from_millis(60)).await; + assert!( + !waiter_task.is_finished(), + "waiter must be blocked by holder" + ); + holder.commit().await.expect("release contention key"); + let (waiter, waited) = waiter_task.await.expect("join lock waiter"); + waited.expect("contended lock succeeds after release"); + waiter.rollback().await.expect("rollback waiter"); + + let timeout_key = 0x62757a7a6f627333_i64; + let mut timeout_holder = pool.begin().await.expect("begin timeout holder"); + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(timeout_key) + .execute(&mut *timeout_holder) + .await + .expect("holder acquires timeout key"); + let mut timeout_waiter = pool.begin().await.expect("begin timeout waiter"); + sqlx::query("SET LOCAL lock_timeout = '30ms'") + .execute(&mut *timeout_waiter) + .await + .expect("set test-only lock timeout"); + let timed_out = observe_advisory_lock( + LockType::MigrationSchemaSafety, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(timeout_key) + .execute(&mut *timeout_waiter), + ) + .await + .expect_err("lock wait times out"); + assert_eq!( + timed_out + .as_database_error() + .and_then(|error| error.code()) + .as_deref(), + Some("55P03") + ); + timeout_holder + .rollback() + .await + .expect("release timeout key"); + + let mut aborted = pool.begin().await.expect("begin error transaction"); + sqlx::query("SELECT 1 / 0") + .execute(&mut *aborted) + .await + .expect_err("abort transaction before lock"); + observe_advisory_lock( + LockType::Membership, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(0x62757a7a6f627334_i64) + .execute(&mut *aborted), + ) + .await + .expect_err("lock statement fails in aborted transaction"); + aborted + .rollback() + .await + .expect("rollback aborted transaction"); + + let mut outcomes = BTreeMap::<(String, String), Vec>::new(); + for (key, _, _, value) in snapshotter.snapshot().into_vec() { + if key.key().name() != "buzz_db_advisory_lock_wait_seconds" { + continue; + } + let DebugValue::Histogram(samples) = value else { + panic!("lock wait must be a histogram"); + }; + let labels = key.key().labels().collect::>(); + let label = |name: &str| { + labels + .iter() + .find(|label| label.key() == name) + .map(|label| label.value().to_owned()) + .unwrap_or_default() + }; + outcomes.insert( + (label("lock_type"), label("outcome")), + samples + .into_iter() + .map(|sample| sample.into_inner()) + .collect(), + ); + } + assert!(outcomes.contains_key(&("replacement".to_owned(), "success".to_owned()))); + assert!(outcomes.contains_key(&("membership".to_owned(), "error".to_owned()))); + assert!( + outcomes.contains_key(&("migration_schema_safety".to_owned(), "timeout".to_owned())) + ); + let contention = outcomes + .get(&("deletion".to_owned(), "success".to_owned())) + .expect("deletion contention series"); + assert!( + contention.iter().any(|sample| *sample >= 0.04), + "lock timer must include the holder wait: {contention:?}" + ); + } + + mod postgres_tests { + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn pool_acquire_records_success_timeout_and_error_with_wait_time() { + super::pool_acquire_records_success_timeout_and_error_with_wait_time().await; + } + + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn production_db_methods_emit_exact_pool_operation_labels() { + super::production_db_methods_emit_exact_pool_operation_labels().await; + } + + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn deletion_catalog_readiness_records_timeout_and_recovers() { + super::deletion_catalog_readiness_records_timeout_and_recovers().await; + } + + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn serving_write_gate_records_cancel_timeout_success_and_recovery() { + super::serving_write_gate_records_cancel_timeout_success_and_recovery().await; + } + + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn advisory_lock_records_success_contention_timeout_and_error() { + super::advisory_lock_records_success_contention_timeout_and_error().await; + } + } +} diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 83322bea1..85d4ffd8a 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -395,7 +395,12 @@ pub async fn verify_floor_guard_behavior(pool: &PgPool) -> crate::Result<()> { } }; - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 1. Pool arming (Perci: assert the effective value, not the intent). let armed: String = sqlx::query_scalar("SHOW buzz.created_at_floor") @@ -552,7 +557,11 @@ pub enum ProbeError { /// a single SELECT would not guarantee evaluation order across the /// subexpressions, reopening the race this ordering exists to close. async fn sample_writer(writer: &PgPool) -> Result { - let mut conn = writer.acquire().await?; + let mut conn = crate::observability::acquire_writer( + writer, + crate::observability::WriterOperation::Maintenance, + ) + .await?; // 1. S first. let sampled_at: DateTime = sqlx::query_scalar("SELECT clock_timestamp()") @@ -671,11 +680,16 @@ pub async fn probe_once(writer: &PgPool, fence: &ReplicaFence) -> Result Db { Db::from_pool(pool) } +#[tokio::test] +async fn begin_transaction_compatibility_alias_is_preserved() { + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy(&crate::test_support::database_url()) + .expect("construct lazy compatibility pool"); + pool.close().await; + let db = Db::from_pool(pool); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + #[allow(deprecated)] + let result = db.begin_transaction().await; + assert!(matches!( + result, + Err(DbError::Sqlx(sqlx::Error::PoolClosed)) + )); + + let counters = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(key, _, _, value)| { + let name = key.key().name(); + if ![ + "buzz_db_pool_acquire_attempts_total", + "buzz_db_pool_acquisitions_total", + ] + .contains(&name) + { + return None; + } + let DebugValue::Counter(value) = value else { + panic!("pool acquisition terminals must be counters"); + }; + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + Some(((name.to_owned(), labels), value)) + }) + .collect::>(); + let expected = [ + ( + ( + "buzz_db_pool_acquire_attempts_total".to_owned(), + [ + ("operation".to_owned(), "event_write".to_owned()), + ("outcome".to_owned(), "error".to_owned()), + ("pool_role".to_owned(), "writer".to_owned()), + ] + .into_iter() + .collect(), + ), + 1, + ), + ( + ( + "buzz_db_pool_acquisitions_total".to_owned(), + [ + ("outcome".to_owned(), "error".to_owned()), + ("pool_role".to_owned(), "writer".to_owned()), + ] + .into_iter() + .collect(), + ), + 1, + ), + ] + .into_iter() + .collect::>(); + assert_eq!(counters, expected); +} + +#[test] +fn nip43_reconciliation_compatibility_alias_is_preserved() { + #[allow(deprecated)] + async fn call( + db: &Db, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + ) -> crate::Result { + db.nip43_membership_snapshot_needs_reconciliation(community_id, relay_pubkey) + .await + } + + let _ = call; +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn readiness_check_distinguishes_pool_exhaustion_from_success() { + let database_url = crate::test_support::database_url(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect(&database_url) + .await + .expect("connect size-one readiness test pool"); + let held = pool + .acquire() + .await + .expect("hold the only readiness test connection"); + let db = Db::from_pool(pool); + + let exhausted = db + .readiness_check(tokio::time::Instant::now() + std::time::Duration::from_millis(25)) + .await; + assert_eq!(exhausted, DbReadinessOutcome::PoolTimeout); + + drop(held); + let recovered = db + .readiness_check(tokio::time::Instant::now() + std::time::Duration::from_secs(1)) + .await; + assert_eq!(recovered, DbReadinessOutcome::Success); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn readiness_check_classifies_closed_pool_query_timeout_and_query_error() { + let database_url = crate::test_support::database_url(); + + let closed_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect(&database_url) + .await + .expect("connect closed readiness test pool"); + closed_pool.close().await; + let closed = Db::from_pool(closed_pool) + .readiness_check(tokio::time::Instant::now() + std::time::Duration::from_secs(1)) + .await; + assert_eq!(closed, DbReadinessOutcome::PoolError); + + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect(&database_url) + .await + .expect("connect query classification test pool"); + let db = Db::from_pool(pool); + + let timed_out = db + .readiness_check_sql( + tokio::time::Instant::now() + std::time::Duration::from_millis(25), + "SELECT pg_sleep(0.2)", + ) + .await; + assert_eq!(timed_out, DbReadinessOutcome::QueryTimeout); + + let query_error = db + .readiness_check_sql( + tokio::time::Instant::now() + std::time::Duration::from_secs(1), + "SELECT 1 / 0", + ) + .await; + assert_eq!(query_error, DbReadinessOutcome::QueryError); + + assert_eq!( + db.readiness_check(tokio::time::Instant::now() + std::time::Duration::from_secs(1)) + .await, + DbReadinessOutcome::Success, + "query failures must return the acquired connection to the pool" + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn readiness_check_cancellation_balances_waiter_and_inflight_connection() { + let database_url = crate::test_support::database_url(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect(&database_url) + .await + .expect("connect cancellation readiness test pool"); + let held = pool + .acquire() + .await + .expect("hold sole connection before waiter cancellation"); + let db = Db::from_pool(pool); + + let waiting_db = db.clone(); + let waiting = tokio::spawn(async move { + waiting_db + .readiness_check(tokio::time::Instant::now() + std::time::Duration::from_secs(5)) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + waiting.abort(); + assert!(waiting + .await + .expect_err("waiting check must be cancelled") + .is_cancelled()); + drop(held); + + assert_eq!( + db.readiness_check(tokio::time::Instant::now() + std::time::Duration::from_secs(1)) + .await, + DbReadinessOutcome::Success, + "cancelled pool waiter must not consume the released connection" + ); + + let querying_db = db.clone(); + let querying = tokio::spawn(async move { + querying_db + .readiness_check_sql( + tokio::time::Instant::now() + std::time::Duration::from_secs(5), + "SELECT pg_sleep(5)", + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + querying.abort(); + assert!(querying + .await + .expect_err("querying check must be cancelled") + .is_cancelled()); + + let recovered = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let outcome = db + .readiness_check( + tokio::time::Instant::now() + std::time::Duration::from_millis(250), + ) + .await; + match outcome { + DbReadinessOutcome::Success => break outcome, + DbReadinessOutcome::PoolTimeout => tokio::task::yield_now().await, + unexpected => panic!( + "cancelled in-flight query produced unexpected recovery outcome: {unexpected:?}" + ), + } + } + }) + .await + .expect("cancelled in-flight query must return or replace its connection"); + assert_eq!(recovered, DbReadinessOutcome::Success); +} + async fn make_community(pool: &PgPool) -> Uuid { let id = Uuid::new_v4(); let host = format!("communities-of-channels-{}.example", id.simple()); @@ -1381,9 +1620,13 @@ async fn admin_url() -> String { std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.into()) } -/// Create a fresh scratch database on the same server and run migrations. -/// Returns (pool, db_name); callers should `drop_scratch_db` when done. -async fn create_scratch_db(admin: &PgPool, prefix: &str) -> (PgPool, String) { +/// Create a fresh scratch database on the same server and optionally run +/// migrations only up to `target`. +async fn create_scratch_db_through( + admin: &PgPool, + prefix: &str, + target: Option, +) -> (PgPool, String) { let name = format!("{}_{}", prefix, Uuid::new_v4().simple()); sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) .execute(admin) @@ -1398,10 +1641,136 @@ async fn create_scratch_db(admin: &PgPool, prefix: &str) -> (PgPool, String) { let pool = PgPool::connect(&scratch_url) .await .expect("connect scratch db"); + match target { + Some(target) => crate::runtime::migration::run_migrations_through(&pool, target) + .await + .expect("migrate scratch db through target"), + None => crate::migration::run_migrations(&pool) + .await + .expect("migrate scratch db"), + } + (pool, name) +} + +/// Create a fresh scratch database on the same server and run migrations. +/// Returns (pool, db_name); callers should `drop_scratch_db` when done. +async fn create_scratch_db(admin: &PgPool, prefix: &str) -> (PgPool, String) { + create_scratch_db_through(admin, prefix, None).await +} + +/// Migration 0073's roster fence is a schema-before-code boundary: a relay +/// whose database predates it must refuse to open listeners, because the new +/// replacement protocol would look safe while an old pod could still overwrite +/// a newer canonical roster. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn unmigrated_roster_fence_blocks_startup_until_0073_is_applied() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (pool, scratch_name) = + create_scratch_db_through(&admin, "roster_fence_unmigrated", Some(72)).await; + let db = Db::from_pool(pool.clone()); + + let error = db + .verify_channel_roster_fence() + .await + .expect_err("pre-0073 schema must block roster publishers"); + assert!( + error.to_string().contains("channel roster fence trigger"), + "startup gate must report the missing schema fence: {error}" + ); + let rows_before: i64 = sqlx::query_scalar("SELECT count(*) FROM events WHERE kind = 39002") + .fetch_one(&pool) + .await + .expect("count pre-migration rosters"); + assert_eq!( + rows_before, 0, + "failed startup gate must not publish a roster" + ); + crate::migration::run_migrations(&pool) .await - .expect("migrate scratch db"); - (pool, name) + .expect("apply migration 0073"); + db.verify_channel_roster_fence() + .await + .expect("0073 must open the startup gate"); + + drop_scratch_db(&admin, pool, &scratch_name).await; +} + +/// The catalog check cannot see a trigger function that was replaced with an +/// inert body, so the behavior probe has to prove the semantics. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn channel_roster_fence_behavior_verification_detects_inert_function() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (pool, scratch_name) = create_scratch_db(&admin, "roster_fence_inert").await; + let db = Db::from_pool(pool.clone()); + + sqlx::raw_sql( + "CREATE OR REPLACE FUNCTION guard_channel_roster_snapshot() \ + RETURNS TRIGGER AS $$ BEGIN RETURN NEW; END; $$ LANGUAGE plpgsql;", + ) + .execute(&pool) + .await + .expect("replace roster fence with inert body"); + let error = db + .verify_channel_roster_fence() + .await + .expect_err("inert roster fence must fail closed"); + assert!( + error + .to_string() + .contains("stale probe roster was accepted"), + "behavior probe must identify inert semantics: {error}" + ); + + drop_scratch_db(&admin, pool, &scratch_name).await; +} + +/// Every attached partition needs the fence, not just the parent: a disabled +/// child trigger is an unfenced insert path. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn channel_roster_fence_catalog_verification_fails_closed() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (pool, scratch_name) = create_scratch_db(&admin, "roster_fence_catalog").await; + let db = Db::from_pool(pool.clone()); + + db.verify_channel_roster_fence() + .await + .expect("migrated roster fence must verify"); + + let child: String = sqlx::query_scalar( + "SELECT n.nspname || '.' || c.relname \ + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE i.inhparent = 'public.events'::regclass ORDER BY i.inhrelid LIMIT 1", + ) + .fetch_one(&pool) + .await + .expect("load event partition"); + sqlx::query(sqlx::AssertSqlSafe(format!( + "ALTER TABLE {child} DISABLE TRIGGER trg_events_guard_channel_roster_snapshot" + ))) + .execute(&pool) + .await + .expect("disable partition roster trigger"); + let error = db + .verify_channel_roster_fence() + .await + .expect_err("disabled partition roster fence must fail closed"); + assert!( + error.to_string().contains(&child), + "verification must identify the unfenced partition: {error}" + ); + + drop_scratch_db(&admin, pool, &scratch_name).await; } async fn drop_scratch_db(admin: &PgPool, pool: PgPool, name: &str) { @@ -3123,6 +3492,233 @@ async fn created_at_floor_guard_aborts_old_channel_rows_at_commit() { drop_scratch_db(&admin, pool, &name).await; } +#[test] +fn writer_pool_safety_hook_is_single_and_composed() { + let source = include_str!("mod.rs"); + let connect_pool = source + .split("async fn connect_writer_pool") + .nth(1) + .and_then(|tail| tail.split("const READER_ACQUIRE_TIMEOUT").next()) + .expect("connect_writer_pool source block"); + assert_eq!( + connect_pool.matches(".after_connect(").count(), + 1, + "SQLx replaces after_connect hooks; writer safety must use exactly one" + ); + assert!(connect_pool.contains("buzz.created_at_floor")); + assert!(connect_pool.contains("SHOW transaction_isolation")); + assert!(connect_pool.contains("'lock_timeout'")); + assert!(connect_pool.contains("'idle_in_transaction_session_timeout'")); + assert!(connect_pool.contains("'statement_timeout'")); + assert!(!connect_pool.contains("arm_floor_guard")); + assert!(!connect_pool.contains("_arm_floor_guard")); + assert!(!connect_pool.contains("allow(unused_variables)")); + + let reader_doc = source + .split("fn connect_read_pool") + .next() + .and_then(|prefix| prefix.rsplit("/// Connect the read-replica").next()) + .expect("reader pool documentation"); + assert!(reader_doc.contains("replica sessions are")); + assert!(reader_doc.contains("read-only")); + assert!(!reader_doc.contains("Db::connect_writer_pool")); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn writer_pool_rejects_non_read_committed_database_default() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (seed_pool, name) = create_scratch_db(&admin, "writer_isolation").await; + sqlx::query(sqlx::AssertSqlSafe(format!( + "ALTER DATABASE {name} SET default_transaction_isolation = 'repeatable read'" + ))) + .execute(&admin) + .await + .expect("set unsafe database default"); + seed_pool.close().await; + + let base = admin_url().await; + let idx = base.rfind('/').expect("db url has a path segment"); + let scratch_url = format!("{}/{}", &base[..idx], name); + let error = Db::new(&DbConfig { + database_url: scratch_url, + max_connections: 1, + min_connections: 1, + acquire_timeout_secs: 1, + ..DbConfig::default() + }) + .await + .expect_err("writer pool must reject pinned-snapshot database defaults"); + assert!( + error.to_string().contains("requires READ COMMITTED") + || error.to_string().contains("pool timed out"), + "unexpected isolation rejection: {error}" + ); + + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop isolation test database"); +} + +/// Session-timeout environment overrides retain PostgreSQL's `0 = disabled` +/// semantics and ignore invalid values. +#[test] +fn session_timeout_env_overlay_zero_passthrough_and_invalid_fallback() { + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _guard = ENV_LOCK.lock().unwrap(); + let keys = [ + "BUZZ_DB_LOCK_TIMEOUT_MS", + "BUZZ_DB_IDLE_TXN_TIMEOUT_MS", + "BUZZ_DB_STATEMENT_TIMEOUT_MS", + ]; + let previous: Vec<_> = keys.iter().map(std::env::var_os).collect(); + let read = |config: DbConfig| { + ( + config.lock_timeout_ms, + config.idle_txn_timeout_ms, + config.statement_timeout_ms, + ) + }; + + for key in keys { + std::env::remove_var(key); + } + let unset = read(DbConfig::default().with_session_timeouts_from_env()); + + std::env::set_var("BUZZ_DB_LOCK_TIMEOUT_MS", "2000"); + std::env::set_var("BUZZ_DB_IDLE_TXN_TIMEOUT_MS", "30000"); + std::env::set_var("BUZZ_DB_STATEMENT_TIMEOUT_MS", "10000"); + let overridden = read(DbConfig::default().with_session_timeouts_from_env()); + + for key in keys { + std::env::set_var(key, "0"); + } + let zero = read(DbConfig::default().with_session_timeouts_from_env()); + + for key in keys { + std::env::set_var(key, "not-a-number"); + } + let junk = read(DbConfig::default().with_session_timeouts_from_env()); + + for (key, value) in keys.iter().zip(previous) { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + + let defaults = (DEFAULT_LOCK_TIMEOUT_MS, DEFAULT_IDLE_TXN_TIMEOUT_MS, 0); + assert_eq!(unset, defaults, "unset env must keep the defaults"); + assert_eq!(overridden, (2000, 30000, 10000)); + assert_eq!(zero, (0, 0, 0), "explicit 0 must disable each timeout"); + assert_eq!(junk, defaults, "junk env must keep the defaults"); +} + +/// The production writer constructor installs all three timeout GUCs, bounds +/// ordinary lock waits, and exempts the intentional migration lock wait. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn session_timeouts_install_through_db_new_and_bound_lock_waits() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (seed_pool, name) = create_scratch_db(&admin, "session_timeouts").await; + seed_pool.close().await; + + let base = admin_url().await; + let idx = base.rfind('/').expect("db url has a path segment"); + let scratch_url = format!("{}/{}", &base[..idx], name); + let db = Db::new(&DbConfig { + database_url: scratch_url.clone(), + max_connections: 2, + lock_timeout_ms: 500, + idle_txn_timeout_ms: 60_000, + statement_timeout_ms: 0, + ..DbConfig::default() + }) + .await + .expect("connect Db with session timeouts"); + + let (lock, idle, statement): (String, String, String) = sqlx::query_as( + "SELECT current_setting('lock_timeout'), \ + current_setting('idle_in_transaction_session_timeout'), \ + current_setting('statement_timeout')", + ) + .fetch_one(&db.pool) + .await + .expect("read effective GUCs"); + assert_eq!(lock, "500ms"); + assert_eq!(idle, "1min"); + assert_eq!(statement, "0"); + + // SQLx keeps exactly one `after_connect` hook, so the session timeouts and + // the replica-fence floor guard share it. Asserting the timeouts alone + // would still pass if a future edit dropped the floor guard, and the + // serving write fence would then be unarmed on every writer connection. + let floor: String = sqlx::query_scalar("SELECT current_setting('buzz.created_at_floor')") + .fetch_one(&db.pool) + .await + .expect("writer connections must arm the created_at floor guard"); + assert_eq!( + floor, + crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string(), + "the floor guard GUC must carry the compiled-in floor" + ); + + let mut holder = db.pool.acquire().await.expect("holder connection"); + sqlx::raw_sql("BEGIN; LOCK TABLE events IN ACCESS EXCLUSIVE MODE") + .execute(&mut *holder) + .await + .expect("hold relation lock"); + let waited = std::time::Instant::now(); + let mut waiter_txn = db.pool.begin().await.expect("waiter transaction"); + let error = sqlx::query("LOCK TABLE events IN ACCESS SHARE MODE") + .execute(&mut *waiter_txn) + .await + .expect_err("waiter must time out, not park"); + drop(waiter_txn); + let code = match &error { + sqlx::Error::Database(db_error) => db_error.code().map(|code| code.to_string()), + other => panic!("expected database error, got {other:?}"), + }; + assert_eq!(code.as_deref(), Some("55P03")); + assert!(waited.elapsed() < std::time::Duration::from_secs(5)); + + let mut advisory_holder = PgPool::connect(&scratch_url) + .await + .expect("advisory holder pool") + .acquire() + .await + .expect("advisory holder conn") + .detach(); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(crate::deletion::SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(&mut advisory_holder) + .await + .expect("hold schema advisory lock"); + let release = tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(1_500)).await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(crate::deletion::SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(&mut advisory_holder) + .await; + let _ = advisory_holder.close().await; + }); + db.migrate() + .await + .expect("migrate must wait out the advisory holder"); + release.await.expect("release task"); + + let _ = sqlx::query("ROLLBACK").execute(&mut *holder).await; + drop(holder); + drop_scratch_db(&admin, db.pool.clone(), &name).await; +} + /// The armed writer pool (`Db::new`) must enforce the floor end-to-end /// through the public insert APIs, and the session GUC must be verifiably /// set on pooled connections. diff --git a/crates/buzz-db/src/store/allowlist.rs b/crates/buzz-db/src/store/allowlist.rs index e0c0d4808..c7635cf4e 100644 --- a/crates/buzz-db/src/store/allowlist.rs +++ b/crates/buzz-db/src/store/allowlist.rs @@ -23,12 +23,17 @@ pub struct AllowlistEntry { impl Db { /// Check if a pubkey is in the allowlist for `community`. pub async fn is_pubkey_allowed(&self, community: CommunityId, pubkey: &[u8]) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authentication, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) as cnt FROM pubkey_allowlist WHERE community_id = $1 AND pubkey = $2", ) .bind(community.as_uuid()) .bind(pubkey) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; Ok(cnt > 0) @@ -36,10 +41,15 @@ impl Db { /// Check if the community allowlist has any entries (i.e. is enforcement active). pub async fn has_allowlist_entries(&self, community: CommunityId) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authentication, + ) + .await?; let row = sqlx::query("SELECT COUNT(*) as cnt FROM pubkey_allowlist WHERE community_id = $1") .bind(community.as_uuid()) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; Ok(cnt > 0) @@ -53,6 +63,11 @@ impl Db { added_by: &[u8], note: Option<&str>, ) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let result = sqlx::query( "INSERT INTO pubkey_allowlist (community_id, pubkey, added_by, note) VALUES ($1, $2, $3, $4) \ ON CONFLICT DO NOTHING", @@ -61,7 +76,7 @@ impl Db { .bind(pubkey) .bind(added_by) .bind(note) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } @@ -72,22 +87,32 @@ impl Db { community: CommunityId, pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let result = sqlx::query("DELETE FROM pubkey_allowlist WHERE community_id = $1 AND pubkey = $2") .bind(community.as_uuid()) .bind(pubkey) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } /// List all pubkeys in the community allowlist. pub async fn list_allowlist(&self, community: CommunityId) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( "SELECT pubkey, added_by, added_at, note FROM pubkey_allowlist WHERE community_id = $1 ORDER BY added_at DESC", ) .bind(community.as_uuid()) - .fetch_all(&self.pool) + .fetch_all(&mut *connection) .await?; let mut out = Vec::with_capacity(rows.len()); diff --git a/crates/buzz-db/src/store/archived_identities.rs b/crates/buzz-db/src/store/archived_identities.rs index a4e18c4d6..d9e728932 100644 --- a/crates/buzz-db/src/store/archived_identities.rs +++ b/crates/buzz-db/src/store/archived_identities.rs @@ -33,11 +33,16 @@ pub struct ArchivedIdentity { /// Returns `true` if `pubkey` (64-char hex) is archived in `community_id`. pub async fn is_archived(pool: &PgPool, community_id: CommunityId, pubkey: &str) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query("SELECT 1 FROM archived_identities WHERE community_id = $1 AND pubkey = $2") .bind(community_id.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.is_some()) } @@ -58,6 +63,11 @@ pub async fn archive( replaced_by: Option<&str>, request_event_id: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "INSERT INTO archived_identities \ (community_id, pubkey, consent_path, actor, reason, replaced_by, request_event_id) \ @@ -71,7 +81,7 @@ pub async fn archive( .bind(reason) .bind(replaced_by) .bind(request_event_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -82,11 +92,16 @@ pub async fn archive( /// Returns `true` if a row was deleted, `false` if the identity was not archived /// in that community. pub async fn unarchive(pool: &PgPool, community_id: CommunityId, pubkey: &str) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query("DELETE FROM archived_identities WHERE community_id = $1 AND pubkey = $2") .bind(community_id.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -97,12 +112,17 @@ pub async fn list_archived( pool: &PgPool, community_id: CommunityId, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let rows = sqlx::query( "SELECT pubkey, consent_path, actor, reason, replaced_by, request_event_id, archived_at \ FROM archived_identities WHERE community_id = $1 ORDER BY archived_at ASC", ) .bind(community_id.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() diff --git a/crates/buzz-db/src/store/channel.rs b/crates/buzz-db/src/store/channel.rs index 8115aa6fc..6ff755b0f 100644 --- a/crates/buzz-db/src/store/channel.rs +++ b/crates/buzz-db/src/store/channel.rs @@ -4,6 +4,7 @@ //! - `open`: searchable, anyone can join //! - `private`: hidden, invite-only +use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; use sqlx::{PgPool, Row}; use uuid::Uuid; @@ -17,6 +18,27 @@ use buzz_core::CommunityId; // without pulling in sqlx/tokio. pub use buzz_core::channel::{ChannelType, ChannelVisibility, MemberRole}; +async fn begin_event_write_transaction( + pool: &PgPool, +) -> Result> { + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + Ok(sqlx::Transaction::begin(connection, None).await?) +} + +async fn acquire_event_write_connection( + pool: &PgPool, +) -> Result> { + Ok(crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?) +} + /// A channel row as returned from the database. #[derive(Debug, Clone)] pub struct ChannelRecord { @@ -90,7 +112,7 @@ pub async fn create_channel( let id = Uuid::new_v4(); - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; sqlx::query( r#" @@ -183,7 +205,7 @@ pub async fn create_channel_with_id( return Err(DbError::InvalidData("channel name is required".into())); } - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; let rows_affected = sqlx::query( r#" @@ -255,6 +277,22 @@ pub async fn get_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result { + get_channel_with_operation( + pool, + community_id, + channel_id, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn get_channel_with_operation( + pool: &PgPool, + community_id: CommunityId, + channel_id: Uuid, + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let row = sqlx::query( r#" SELECT id, name, channel_type::text AS channel_type, visibility::text AS visibility, @@ -269,7 +307,7 @@ pub async fn get_channel( ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await? .ok_or(DbError::ChannelNotFound(channel_id))?; @@ -282,6 +320,22 @@ pub async fn list_channels( community_id: CommunityId, visibility: Option<&str>, ) -> Result> { + list_channels_with_operation( + pool, + community_id, + visibility, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn list_channels_with_operation( + pool: &PgPool, + community_id: CommunityId, + visibility: Option<&str>, + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let rows = if let Some(vis) = visibility { sqlx::query( r#" @@ -300,7 +354,7 @@ pub async fn list_channels( ) .bind(community_id.as_uuid()) .bind(vis) - .fetch_all(pool) + .fetch_all(&mut *connection) .await? } else { sqlx::query( @@ -319,7 +373,7 @@ pub async fn list_channels( "#, ) .bind(community_id.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await? }; @@ -477,7 +531,7 @@ pub async fn update_channel( // this transition — whose own deadline reset is then the latest word. // Non-TTL updates don't touch the fast path and skip the lock. if updates.ttl_seconds.is_some() { - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") .bind(format!( "buzz_channel_ttl:{}:{}", @@ -492,13 +546,20 @@ pub async fn update_channel( } tx.commit().await?; } else { - let result = q.execute(pool).await?; + let mut connection = acquire_event_write_connection(pool).await?; + let result = q.execute(&mut *connection).await?; if result.rows_affected() == 0 { return Err(DbError::ChannelNotFound(channel_id)); } } - get_channel(pool, community_id, channel_id).await + get_channel_with_operation( + pool, + community_id, + channel_id, + crate::observability::WriterOperation::EventWrite, + ) + .await } /// Sets the topic for a channel, recording who set it and when. @@ -509,6 +570,7 @@ pub async fn set_topic( topic: &str, set_by: &[u8], ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; let result = sqlx::query( "UPDATE channels SET topic = $1, topic_set_by = $2, topic_set_at = NOW() \ WHERE community_id = $3 AND id = $4 AND deleted_at IS NULL", @@ -517,7 +579,7 @@ pub async fn set_topic( .bind(set_by) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() == 0 { return Err(DbError::ChannelNotFound(channel_id)); @@ -533,6 +595,7 @@ pub async fn set_purpose( purpose: &str, set_by: &[u8], ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; let result = sqlx::query( "UPDATE channels SET purpose = $1, purpose_set_by = $2, purpose_set_at = NOW() \ WHERE community_id = $3 AND id = $4 AND deleted_at IS NULL", @@ -541,7 +604,7 @@ pub async fn set_purpose( .bind(set_by) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() == 0 { return Err(DbError::ChannelNotFound(channel_id)); @@ -558,13 +621,14 @@ pub async fn archive_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // First check: does the channel exist and what is its state? let row = sqlx::query( "SELECT archived_at FROM channels WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -585,7 +649,7 @@ pub async fn archive_channel( ) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(()) @@ -600,13 +664,14 @@ pub async fn unarchive_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // First check: does the channel exist and what is its state? let row = sqlx::query( "SELECT archived_at FROM channels WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -629,7 +694,7 @@ pub async fn unarchive_channel( ) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(()) @@ -644,12 +709,13 @@ pub async fn soft_delete_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result { + let mut connection = acquire_event_write_connection(pool).await?; let result = sqlx::query( "UPDATE channels SET deleted_at = NOW() WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -661,6 +727,11 @@ pub async fn soft_delete_channel( /// `archived_at IS NULL` guard prevents double-archiving even if called /// concurrently from multiple relay pods. pub async fn reap_expired_ephemeral_channels(pool: &PgPool) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let rows = sqlx::query( "UPDATE channels AS ch SET archived_at = NOW() \ FROM communities AS c \ @@ -673,7 +744,7 @@ pub async fn reap_expired_ephemeral_channels(pool: &PgPool) -> Result Result { + get_channel_with_operation( + &self.pool, + community_id, + channel_id, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Lists channels, optionally filtered by visibility. pub async fn list_channels( &self, @@ -763,6 +851,22 @@ impl Db { list_channels(&self.pool, community_id, visibility).await } + /// Lists channels during startup reconciliation. + #[datastore_span(name = "list_channels_for_bootstrap", system = "postgresql")] + pub async fn list_channels_for_bootstrap( + &self, + community_id: CommunityId, + visibility: Option<&str>, + ) -> Result> { + list_channels_with_operation( + &self.pool, + community_id, + visibility, + crate::observability::WriterOperation::Bootstrap, + ) + .await + } + /// Updates a channel's name and/or description. pub async fn update_channel( &self, diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 0e92df84b..c2d4204d4 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -3,6 +3,7 @@ //! Membership mutations share one advisory-lock namespace. Relay-authored //! roster snapshots hold that same lock through replacement publication. +use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; use sqlx::{PgPool, Postgres, Row, Transaction}; use uuid::Uuid; @@ -44,6 +45,134 @@ pub struct MemberRecord { pub removed_at: Option>, } +/// Verify that migration 0032's roster fence is active on the partitioned +/// `events` parent and every attached partition. +/// +/// New roster publishers depend on this database-side guard to serialize with +/// legacy publishers during a rolling deployment. If the migration has not +/// been applied, publishing with the new lock protocol would falsely appear +/// safe while an old pod could still overwrite it with stale membership. +pub async fn verify_channel_roster_fence_catalog<'e>( + executor: impl sqlx::PgExecutor<'e>, +) -> Result<()> { + // tgtype bits: 1 = ROW, 2 = BEFORE, 4 = INSERT, 16 = UPDATE, 64 = INSTEAD. + // Required: ROW + BEFORE + INSERT set; UPDATE + INSTEAD clear. + let missing: Vec = sqlx::query_scalar( + r#" + SELECT n.nspname || '.' || c.relname + FROM ( + SELECT 'public.events'::regclass AS oid + UNION ALL + SELECT inhrelid FROM pg_inherits WHERE inhparent = 'public.events'::regclass + ) rels + JOIN pg_class c ON c.oid = rels.oid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE NOT EXISTS ( + SELECT 1 FROM pg_trigger t + WHERE t.tgrelid = rels.oid + AND t.tgname = 'trg_events_guard_channel_roster_snapshot' + AND t.tgfoid = to_regprocedure('public.guard_channel_roster_snapshot()') + AND t.tgenabled IN ('O', 'A') + AND t.tgtype & 1 = 1 -- row-level + AND t.tgtype & 2 = 2 -- BEFORE + AND t.tgtype & 4 = 4 -- fires on INSERT + AND t.tgtype & 16 = 0 -- not UPDATE + AND t.tgtype & 64 = 0 -- not INSTEAD OF + ) + "#, + ) + .fetch_all(executor) + .await?; + if !missing.is_empty() { + return Err(DbError::InvalidData(format!( + "channel roster fence trigger missing, disabled, or mis-shaped on: {}", + missing.join(", ") + ))); + } + Ok(()) +} + +/// Prove migration 0032's roster fence semantics through the live writer pool. +/// +/// The catalog check cannot detect a no-op or otherwise corrupted trigger +/// function. This rolled-back probe verifies that a canonical empty roster is +/// accepted while a stale roster member is rejected with `check_violation`. +pub async fn verify_channel_roster_fence_behavior(pool: &sqlx::PgPool) -> Result<()> { + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + let community_id = Uuid::new_v4(); + let channel_id = Uuid::new_v4(); + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community_id) + .bind(format!( + "roster-fence-verify-{}.invalid", + community_id.simple() + )) + .execute(&mut *tx) + .await?; + + let insert = |id: Vec, tags: serde_json::Value| { + sqlx::query( + "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \ + VALUES ($1, $2, $3, NOW(), 39002, $4, '', $5, NOW(), $6, $7)", + ) + .bind(community_id) + .bind(id) + .bind(vec![0u8; 32]) + .bind(tags) + .bind(vec![0u8; 64]) + .bind(channel_id) + .bind(channel_id.to_string()) + }; + + insert( + vec![0u8; 32], + serde_json::json!([["d", channel_id.to_string()]]), + ) + .execute(&mut *tx) + .await + .map_err(|error| { + DbError::InvalidData(format!( + "channel roster fence rejected a canonical probe roster: {error}" + )) + })?; + + sqlx::query("SAVEPOINT roster_fence_probe") + .execute(&mut *tx) + .await?; + let stale = insert( + vec![1u8; 32], + serde_json::json!([ + ["d", channel_id.to_string()], + ["p", hex::encode([2u8; 32]), "", "member"] + ]), + ) + .execute(&mut *tx) + .await; + match stale { + Err(sqlx::Error::Database(error)) if error.code().as_deref() == Some("23514") => {} + Ok(_) => { + return Err(DbError::InvalidData( + "channel roster fence is inert: a stale probe roster was accepted".into(), + )); + } + Err(error) => { + return Err(DbError::InvalidData(format!( + "channel roster fence probe failed unexpectedly: {error}" + ))); + } + } + sqlx::query("ROLLBACK TO SAVEPOINT roster_fence_probe") + .execute(&mut *tx) + .await?; + tx.rollback().await?; + Ok(()) +} + /// Take the per-channel membership lock. MUST be the first statement in the /// transaction that then reads roles/owner counts and writes membership, so the /// whole check-then-write sequence is atomic against a concurrent one. @@ -52,14 +181,17 @@ async fn acquire_channel_membership_lock( community_id: CommunityId, channel_id: Uuid, ) -> Result<()> { - sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") - .bind(format!( - "{CHANNEL_MEMBERSHIP_LOCK_NAMESPACE}{}:{}", - community_id.as_uuid(), - channel_id - )) - .execute(&mut **tx) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::Membership, + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!( + "{CHANNEL_MEMBERSHIP_LOCK_NAMESPACE}{}:{}", + community_id.as_uuid(), + channel_id + )) + .execute(&mut **tx), + ) + .await?; Ok(()) } @@ -194,7 +326,12 @@ pub async fn lock_member_snapshot( channel_id: Uuid, relay_pubkey: &[u8], ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // Match the canonical replacement writer's lock order. Old binaries take // this key before INSERT; migration 0032 then takes the membership key in // the INSERT trigger. Taking both in that order avoids mixed-version @@ -205,10 +342,13 @@ pub async fn lock_member_snapshot( relay_pubkey, Some(channel_id.as_bytes()), ); - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(replacement_lock) - .execute(&mut *tx) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::Replacement, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(replacement_lock) + .execute(&mut *tx), + ) + .await?; acquire_channel_membership_lock(&mut tx, community_id, channel_id).await?; let rows = sqlx::query( r#" @@ -264,7 +404,12 @@ pub async fn add_member( ))); } - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // First statement: serialize the whole role-check / owner-count / upsert // sequence against concurrent membership writes on this channel. @@ -445,7 +590,12 @@ pub async fn remove_member( crate::user::is_agent_owner(pool, community_id, pubkey, actor_pubkey).await? }; - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // First statement: serialize the actor-role check, the last-owner count and // the UPDATE against concurrent membership writes on this channel (same key @@ -516,6 +666,11 @@ pub async fn is_member( channel_id: Uuid, pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) as cnt FROM channel_members cm \ JOIN channels c ON cm.community_id = c.community_id AND cm.channel_id = c.id AND c.deleted_at IS NULL \ @@ -524,7 +679,7 @@ pub async fn is_member( .bind(community_id.as_uuid()) .bind(channel_id) .bind(pubkey) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; Ok(cnt > 0) @@ -542,6 +697,11 @@ pub async fn membership_pairs( if channel_ids.is_empty() || pubkeys.is_empty() { return Ok(Vec::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( "SELECT cm.channel_id, cm.pubkey FROM channel_members cm \ JOIN channels c ON cm.community_id = c.community_id AND cm.channel_id = c.id AND c.deleted_at IS NULL \ @@ -550,14 +710,19 @@ pub async fn membership_pairs( .bind(community_id.as_uuid()) .bind(channel_ids) .bind(pubkeys) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() .map(|row| Ok((row.try_get("channel_id")?, row.try_get("pubkey")?))) .collect() } -/// Returns all active members of the given channel. +/// Returns all active members of the given channel, ordered by `joined_at`. +/// +/// The roster is returned in full and is never truncated: callers use it to +/// build the kind 39002 (NIP-29 group members) snapshot and to resolve actor +/// roles for admin-event authorization, so a partial list silently hides late +/// joiners from channel discovery and makes them read as non-members. /// /// Returns an empty list if the channel has been soft-deleted. pub async fn get_members( @@ -565,6 +730,22 @@ pub async fn get_members( community_id: CommunityId, channel_id: Uuid, ) -> Result> { + get_members_with_operation( + pool, + community_id, + channel_id, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn get_members_with_operation( + pool: &PgPool, + community_id: CommunityId, + channel_id: Uuid, + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let rows = sqlx::query( r#" SELECT cm.channel_id, cm.pubkey, cm.role::text AS role, cm.joined_at, cm.invited_by, cm.removed_at @@ -572,12 +753,11 @@ pub async fn get_members( JOIN channels c ON cm.community_id = c.community_id AND cm.channel_id = c.id AND c.deleted_at IS NULL WHERE cm.community_id = $1 AND cm.channel_id = $2 AND cm.removed_at IS NULL ORDER BY cm.joined_at ASC - LIMIT 1000 "#, ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter().map(row_to_member_record).collect() } @@ -597,6 +777,11 @@ pub async fn get_members_bulk( if channel_ids.is_empty() { return Ok(Vec::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT cm.channel_id, cm.pubkey, cm.role::text AS role, cm.joined_at, cm.invited_by, cm.removed_at @@ -608,7 +793,7 @@ pub async fn get_members_bulk( ) .bind(community_id.as_uuid()) .bind(channel_ids) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter().map(row_to_member_record).collect() } @@ -622,6 +807,11 @@ pub async fn get_accessible_channel_ids( community_id: CommunityId, pubkey: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT cm.channel_id @@ -636,7 +826,7 @@ pub async fn get_accessible_channel_ids( ) .bind(community_id.as_uuid()) .bind(pubkey) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() @@ -647,6 +837,86 @@ pub async fn get_accessible_channel_ids( .collect() } +/// A large channel whose canonical active-member count may need its legacy +/// discovery snapshot repaired. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LargeChannelRoster { + /// Community that owns the channel. + pub community_id: CommunityId, + /// Canonical host for the owning community. + pub host: String, + /// Channel whose roster snapshot differs from canonical membership. + pub channel_id: Uuid, + /// Canonical active-member count. + pub member_count: i64, +} + +/// Returns active channels whose canonical roster exceeds `minimum_members`. +/// +/// This is an internal cross-community maintenance read. Callers must preserve +/// the returned community id when reading or rewriting discovery state. +pub async fn list_large_channel_rosters_needing_reconciliation( + pool: &PgPool, + minimum_members: i64, + relay_pubkey: &[u8], +) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; + let rows = sqlx::query( + r#" + WITH large_rosters AS ( + SELECT cm.community_id, cm.channel_id, COUNT(*) AS member_count + FROM channel_members cm + JOIN channels ch + ON ch.community_id = cm.community_id + AND ch.id = cm.channel_id + AND ch.deleted_at IS NULL + WHERE cm.removed_at IS NULL + GROUP BY cm.community_id, cm.channel_id + HAVING COUNT(*) > $1 + ) + SELECT lr.community_id, community.host, lr.channel_id, lr.member_count + FROM large_rosters lr + JOIN communities community ON community.id = lr.community_id + JOIN LATERAL ( + SELECT roster.tags + FROM events roster + WHERE roster.community_id = lr.community_id + AND roster.channel_id = lr.channel_id + AND roster.kind = 39002 + AND roster.pubkey = $2 + AND roster.deleted_at IS NULL + ORDER BY roster.created_at DESC, roster.id ASC + LIMIT 1 + ) live_roster ON true + WHERE lr.member_count <> ( + SELECT COUNT(*) + FROM jsonb_array_elements(live_roster.tags) tag + WHERE tag->>0 = 'p' + ) + ORDER BY lr.community_id, lr.channel_id + "#, + ) + .bind(minimum_members) + .bind(relay_pubkey) + .fetch_all(&mut *connection) + .await?; + + rows.into_iter() + .map(|row| { + Ok(LargeChannelRoster { + community_id: CommunityId::from_uuid(row.try_get("community_id")?), + host: row.try_get("host")?, + channel_id: row.try_get("channel_id")?, + member_count: row.try_get("member_count")?, + }) + }) + .collect() +} + /// Transaction-aware variant of [`get_active_role_tx`]. async fn get_active_role_tx( tx: &mut Transaction<'_, Postgres>, @@ -753,6 +1023,11 @@ pub async fn get_accessible_channels( visibility_filter: Option<&str>, member_only: Option, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; // When `member_only` is `Some(true)`, restrict to channels where the user // has an active membership (cm.channel_id IS NOT NULL). This is a strict // subset of the default result set and is pushed into SQL so the LIMIT 1000 @@ -797,7 +1072,7 @@ pub async fn get_accessible_channels( query }; - let rows = query.fetch_all(pool).await?; + let rows = query.fetch_all(&mut *connection).await?; rows.into_iter() .map(|row| { let is_member: bool = row.try_get("is_member").unwrap_or(false); @@ -816,6 +1091,11 @@ pub async fn get_bot_members( pool: &PgPool, community_id: CommunityId, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT cm.pubkey, u.display_name, u.agent_type, u.capabilities, @@ -829,7 +1109,7 @@ pub async fn get_bot_members( "#, ) .bind(community_id.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; let mut out = Vec::with_capacity(rows.len()); @@ -860,10 +1140,26 @@ pub async fn get_users_bulk( pool: &PgPool, community_id: CommunityId, pubkeys: &[Vec], +) -> Result> { + get_users_bulk_with_operation( + pool, + community_id, + pubkeys, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await +} + +async fn get_users_bulk_with_operation( + pool: &PgPool, + community_id: CommunityId, + pubkeys: &[Vec], + operation: crate::observability::WriterOperation, ) -> Result> { if pubkeys.is_empty() { return Ok(Vec::new()); } + let mut connection = crate::observability::acquire_writer(pool, operation).await?; // Build a parameterised IN clause: ($2, $3, ...); $1 is community_id. let placeholders = (2..(pubkeys.len() + 2)) @@ -880,7 +1176,7 @@ pub async fn get_users_bulk( q = q.bind(pk); } - let rows = q.fetch_all(pool).await?; + let rows = q.fetch_all(&mut *connection).await?; let mut out = Vec::with_capacity(rows.len()); for row in rows { @@ -913,12 +1209,17 @@ pub async fn get_member_count( community_id: CommunityId, channel_id: Uuid, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) as cnt FROM channel_members WHERE community_id = $1 AND channel_id = $2 AND removed_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; Ok(row.try_get("cnt")?) } @@ -935,6 +1236,11 @@ pub async fn get_member_counts_bulk( if channel_ids.is_empty() { return Ok(std::collections::HashMap::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( "SELECT channel_id, COUNT(*) as cnt FROM channel_members \ @@ -948,7 +1254,7 @@ pub async fn get_member_counts_bulk( } qb.push(") GROUP BY channel_id"); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *connection).await?; let mut map = std::collections::HashMap::with_capacity(rows.len()); for row in rows { @@ -968,6 +1274,11 @@ pub async fn get_member_role( channel_id: Uuid, pubkey: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT cm.role::text AS role FROM channel_members cm \ JOIN channels c ON cm.community_id = c.community_id AND cm.channel_id = c.id AND c.deleted_at IS NULL \ @@ -976,12 +1287,51 @@ pub async fn get_member_role( .bind(community_id.as_uuid()) .bind(channel_id) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.map(|r| r.try_get("role")).transpose()?) } impl Db { + /// Verify the mixed-version channel-roster database fence end to end. + #[datastore_span(name = "verify_channel_roster_fence", system = "postgresql")] + pub async fn verify_channel_roster_fence(&self) -> Result<()> { + { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + verify_channel_roster_fence_catalog(&mut *connection).await?; + } + verify_channel_roster_fence_behavior(&self.pool).await + } + + /// Capture the active roster while holding the membership-writer lock. + #[datastore_span(name = "lock_member_snapshot", system = "postgresql")] + pub async fn lock_member_snapshot( + &self, + community_id: CommunityId, + channel_id: Uuid, + relay_pubkey: &[u8], + ) -> Result { + lock_member_snapshot(&self.pool, community_id, channel_id, relay_pubkey).await + } + + /// Returns large active-channel rosters whose relay-authored snapshots differ. + #[datastore_span( + name = "list_large_channel_rosters_needing_reconciliation", + system = "postgresql" + )] + pub async fn list_large_channel_rosters_needing_reconciliation( + &self, + minimum_members: i64, + relay_pubkey: &[u8], + ) -> Result> { + list_large_channel_rosters_needing_reconciliation(&self.pool, minimum_members, relay_pubkey) + .await + } + /// Adds a member to a channel. pub async fn add_member( &self, @@ -1043,6 +1393,22 @@ impl Db { get_members(&self.pool, community_id, channel_id).await } + /// Return a channel roster used to build or validate an event mutation. + #[datastore_span(name = "get_members_for_event_write", system = "postgresql")] + pub async fn get_members_for_event_write( + &self, + community_id: CommunityId, + channel_id: Uuid, + ) -> Result> { + get_members_with_operation( + &self.pool, + community_id, + channel_id, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Returns active members for multiple channels in a single query. pub async fn get_members_bulk( &self, @@ -1093,6 +1459,22 @@ impl Db { get_users_bulk(&self.pool, community_id, pubkeys).await } + /// Bulk-fetch user names while constructing an event and its mention tags. + #[datastore_span(name = "get_users_bulk_for_event_write", system = "postgresql")] + pub async fn get_users_bulk_for_event_write( + &self, + community_id: CommunityId, + pubkeys: &[Vec], + ) -> Result> { + get_users_bulk_with_operation( + &self.pool, + community_id, + pubkeys, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Returns the count of active members in a channel. pub async fn get_member_count( &self, @@ -1128,8 +1510,9 @@ mod tests { use crate::channel::{ChannelType, ChannelVisibility}; use crate::user::{ensure_user, set_agent_owner}; use nostr::Keys; + use sqlx::postgres::PgPoolOptions; - const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; + const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 -- local test-only credentials async fn setup_pool() -> PgPool { PgPool::connect(TEST_DB_URL) @@ -1346,6 +1729,267 @@ mod tests { assert_eq!(channel_ids.len(), channel_count as usize); } + /// `get_members` must return the complete roster, not a truncated prefix. + /// + /// The relay builds the kind 39002 (NIP-29 group members) snapshot and every + /// admin role lookup from this list, so a cap silently hides late joiners: + /// their clients never discover the channel, and an owner past the cutoff + /// reads as a non-member. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn get_members_returns_full_roster_beyond_1000() { + let database_url = + std::env::var("BUZZ_TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.to_string()); + let pool = PgPool::connect(&database_url) + .await + .expect("connect to test DB"); + let community_id = make_test_community(&pool).await; + let community = CommunityId::from_uuid(community_id); + let creator = random_pubkey(); + + // create_test_channel also inserts the creator as the first (owner) member. + let channel = create_test_channel( + &pool, + community_id, + "high-volume-roster", + ChannelType::Stream, + ChannelVisibility::Open, + None, + &creator, + None, + ) + .await + .expect("create test channel"); + + // Bulk-insert additional members with strictly increasing `joined_at`, so + // member N lands at roster position N (the creator holds position 0). + // The final member is an owner joining well past the old 1000-row cutoff. + let extra_members = 1_500; + sqlx::query( + r#" + INSERT INTO channel_members (community_id, channel_id, pubkey, role, joined_at) + SELECT + $1, + $2, + decode(lpad(to_hex(n), 64, '0'), 'hex'), + (CASE WHEN n = $3 THEN 'owner' ELSE 'member' END)::member_role, + NOW() + (n || ' seconds')::interval + FROM generate_series(1, $3) n + "#, + ) + .bind(community_id) + .bind(channel.id) + .bind(extra_members) + .execute(&pool) + .await + .expect("insert high-volume channel members"); + + let members = get_members(&pool, community, channel.id) + .await + .expect("load channel members"); + + assert_eq!( + members.len(), + extra_members as usize + 1, + "get_members truncated the roster" + ); + + // The last joiner sits at the final roster position — past any + // 1000-row cap — which also pins the documented `joined_at` ordering. + let late_owner = hex::decode(format!("{:064x}", extra_members)).expect("hex pubkey"); + let late = members.last().expect("roster is non-empty"); + assert_eq!( + late.pubkey, late_owner, + "member who joined after the 1000th must be present and ordered last" + ); + assert_eq!( + late.role, "owner", + "role of a late-joining owner must resolve correctly" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn large_roster_reconciliation_candidates_respect_snapshot_count_and_signer() { + let pool = setup_pool().await; + let community_id = make_test_community(&pool).await; + let community = CommunityId::from_uuid(community_id); + let creator = random_pubkey(); + let relay_pubkey = random_pubkey(); + let other_relay_pubkey = random_pubkey(); + let channel = create_test_channel( + &pool, + community_id, + "stale-large-roster", + ChannelType::Stream, + ChannelVisibility::Open, + None, + &creator, + None, + ) + .await + .expect("create test channel"); + + let extra_members = 1_500; + sqlx::query( + r#" + INSERT INTO channel_members (community_id, channel_id, pubkey, role, joined_at) + SELECT $1, $2, decode(lpad(to_hex(n), 64, '0'), 'hex'), 'member', + NOW() + (n || ' seconds')::interval + FROM generate_series(1, $3) n + "#, + ) + .bind(community_id) + .bind(channel.id) + .bind(extra_members) + .execute(&pool) + .await + .expect("insert large roster"); + + let stale_tags: Vec = + std::iter::once(serde_json::json!(["d", channel.id.to_string()])) + .chain((0..1_000).map(|n| serde_json::json!(["p", format!("{n:064x}")]))) + .collect(); + let complete_tags: Vec = + std::iter::once(serde_json::json!(["d", channel.id.to_string()])) + .chain((0..1_501).map(|n| serde_json::json!(["p", format!("{n:064x}")]))) + .collect(); + + // Insert canonical-looking history first, then corrupt the newest row + // with UPDATE to model a stale snapshot that predates migration 0032's + // INSERT fence. New stale snapshots cannot be inserted once that fence + // is deployed. + sqlx::query( + r#" + INSERT INTO events + (community_id, id, pubkey, created_at, kind, tags, content, sig, channel_id, d_tag) + VALUES + ($1, $2, $3, NOW() - INTERVAL '1 minute', 39002, $4, '', $5, $6, $7), + ($1, $8, $3, NOW(), 39002, $4, '', $5, $6, $7) + "#, + ) + .bind(community_id) + .bind(random_pubkey()) + .bind(&relay_pubkey) + .bind(serde_json::Value::Array(complete_tags.clone())) + .bind(vec![0u8; 64]) + .bind(channel.id) + .bind(channel.id.to_string()) + .bind(random_pubkey()) + .execute(&pool) + .await + .expect("insert historical duplicate snapshots"); + sqlx::query( + "UPDATE events SET tags = $1 WHERE community_id = $2 AND channel_id = $3 \ + AND kind = 39002 AND pubkey = $4 AND created_at = (SELECT MAX(created_at) \ + FROM events WHERE community_id = $2 AND channel_id = $3 AND kind = 39002 AND pubkey = $4)", + ) + .bind(serde_json::Value::Array(stale_tags)) + .bind(community_id) + .bind(channel.id) + .bind(&relay_pubkey) + .execute(&pool) + .await + .expect("simulate pre-fence stale live snapshot"); + + // The same channel UUID in another tenant is deliberately valid. A + // complete snapshot there must not mask this tenant's stale head. + let other_community_id = make_test_community(&pool).await; + sqlx::query( + r#" + INSERT INTO channels + (id, community_id, name, channel_type, visibility, created_by) + VALUES ($1, $2, 'same-id-complete-roster', 'stream', 'open', $3) + "#, + ) + .bind(channel.id) + .bind(other_community_id) + .bind(&creator) + .execute(&pool) + .await + .expect("insert same channel id in other tenant"); + sqlx::query( + r#" + INSERT INTO channel_members (community_id, channel_id, pubkey, role, joined_at) + SELECT $1, $2, decode(lpad(to_hex(n), 64, '0'), 'hex'), 'member', + NOW() + (n || ' seconds')::interval + FROM generate_series(0, 1500) n + "#, + ) + .bind(other_community_id) + .bind(channel.id) + .execute(&pool) + .await + .expect("insert complete other-tenant roster"); + sqlx::query( + r#" + INSERT INTO events + (community_id, id, pubkey, created_at, kind, tags, content, sig, channel_id, d_tag) + VALUES ($1, $2, $3, NOW(), 39002, $4, '', $5, $6, $7) + "#, + ) + .bind(other_community_id) + .bind(random_pubkey()) + .bind(&relay_pubkey) + .bind(serde_json::Value::Array(complete_tags.clone())) + .bind(vec![0u8; 64]) + .bind(channel.id) + .bind(channel.id.to_string()) + .execute(&pool) + .await + .expect("insert complete other-tenant snapshot"); + + // Put the stale channel behind the 1,000 newest channels that the old + // list_channels-based sweep could see. This set-based scan has no such + // pagination ceiling. + sqlx::query( + r#" + INSERT INTO channels + (id, community_id, name, channel_type, visibility, created_by, created_at) + SELECT gen_random_uuid(), $1, 'newer-decoy-' || n, 'stream', 'open', $2, + NOW() + (n || ' seconds')::interval + FROM generate_series(1, 1000) n + "#, + ) + .bind(community_id) + .bind(&creator) + .execute(&pool) + .await + .expect("insert channels beyond old list ceiling"); + + let candidates = + list_large_channel_rosters_needing_reconciliation(&pool, 1_000, &relay_pubkey) + .await + .expect("find stale snapshot"); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].community_id, community); + assert_eq!(candidates[0].channel_id, channel.id); + assert_eq!(candidates[0].member_count, 1_501); + + let other_signer_candidates = + list_large_channel_rosters_needing_reconciliation(&pool, 1_000, &other_relay_pubkey) + .await + .expect("other signer is isolated from relay-authored snapshot"); + assert!(other_signer_candidates.is_empty()); + + sqlx::query( + "UPDATE events SET tags = $1, created_at = NOW() + INTERVAL '1 minute' WHERE community_id = $2 AND channel_id = $3 AND kind = 39002 AND pubkey = $4 AND created_at = (SELECT MAX(created_at) FROM events WHERE community_id = $2 AND channel_id = $3 AND kind = 39002 AND pubkey = $4 AND deleted_at IS NULL)", + ) + .bind(serde_json::Value::Array(complete_tags)) + .bind(community_id) + .bind(channel.id) + .bind(&relay_pubkey) + .execute(&pool) + .await + .expect("complete snapshot"); + + let converged = + list_large_channel_rosters_needing_reconciliation(&pool, 1_000, &relay_pubkey) + .await + .expect("check converged snapshot"); + assert!(converged.is_empty()); + } + /// A random non-admin, non-owner user cannot remove someone else's bot. #[tokio::test] #[ignore = "requires Postgres"] @@ -1829,6 +2473,96 @@ mod tests { (community, channel.id, owner_a, owner_b) } + /// A captured roster holds the same lock as membership writers until the + /// publisher explicitly releases it. This is the freshness fence used by + /// rolling-deploy reconciliation. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn locked_member_snapshot_blocks_post_capture_membership_mutation() { + let pool = setup_pool().await; + let community_id = make_test_community(&pool).await; + let community = CommunityId::from_uuid(community_id); + let owner = random_pubkey(); + let newcomer = random_pubkey(); + let channel = create_test_channel( + &pool, + community_id, + "snapshot-freshness-fence", + ChannelType::Stream, + ChannelVisibility::Open, + None, + &owner, + None, + ) + .await + .expect("create channel"); + + let snapshot_pool = PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(1)) + .connect(TEST_DB_URL) + .await + .expect("connect one-connection pool"); + let relay_keys = Keys::generate(); + let mut snapshot = lock_member_snapshot( + &snapshot_pool, + community, + channel.id, + &relay_keys.public_key().to_bytes(), + ) + .await + .expect("capture locked roster"); + assert_eq!(snapshot.members.len(), 1); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(39002), "") + .tags(vec![ + nostr::Tag::parse(["d", &channel.id.to_string()]).expect("d tag"), + nostr::Tag::parse(["p", &hex::encode(&owner)]).expect("p tag"), + ]) + .sign_with_keys(&relay_keys) + .expect("sign roster"); + let (_, inserted) = snapshot + .replace_member_event(community, channel.id, &event) + .await + .expect("replace roster on held connection"); + assert!(inserted); + + let mut contender = pool.begin().await.expect("begin membership writer"); + let acquired: bool = + sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!( + "{CHANNEL_MEMBERSHIP_LOCK_NAMESPACE}{}:{}", + community.as_uuid(), + channel.id + )) + .fetch_one(&mut *contender) + .await + .expect("try membership writer lock"); + assert!( + !acquired, + "membership mutation must wait until the captured roster is published" + ); + contender.rollback().await.expect("rollback contender"); + + snapshot.release().await.expect("release snapshot fence"); + add_member( + &pool, + community, + channel.id, + &newcomer, + MemberRole::Member, + None, + ) + .await + .expect("membership mutation after publication"); + assert_eq!( + get_members(&pool, community, channel.id) + .await + .expect("fresh roster") + .len(), + 2 + ); + } + /// The lock must be shared with `remove_member`: a demotion racing an owner /// removal goes through a separate count/update path, so both must serialize /// on the same key or they can jointly empty the owner set. diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 04f08d2a9..289da979b 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -1,6 +1,7 @@ //! Community lifecycle and host-map persistence. use buzz_core::CommunityId; +use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; use sqlx::Row; use uuid::Uuid; @@ -89,6 +90,11 @@ impl Db { &self, normalized_host: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::TenantResolution, + ) + .await?; let row = sqlx::query( r#" SELECT id, host @@ -98,7 +104,7 @@ impl Db { "#, ) .bind(normalized_host) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { @@ -115,11 +121,37 @@ impl Db { /// Returns whether a community id still exists in the active lifecycle state. pub async fn is_community_active(&self, community_id: CommunityId) -> Result { + self.is_community_active_with_operation( + community_id, + crate::observability::WriterOperation::Authorization, + ) + .await + } + + /// Background lifecycle revalidation variant of [`Self::is_community_active`]. + #[datastore_span(name = "is_community_active_for_maintenance", system = "postgresql")] + pub async fn is_community_active_for_maintenance( + &self, + community_id: CommunityId, + ) -> Result { + self.is_community_active_with_operation( + community_id, + crate::observability::WriterOperation::Maintenance, + ) + .await + } + + async fn is_community_active_with_operation( + &self, + community_id: CommunityId, + operation: crate::observability::WriterOperation, + ) -> Result { + let mut connection = crate::observability::acquire_writer(&self.pool, operation).await?; let active = sqlx::query_scalar::<_, bool>( "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL)", ) .bind(community_id.as_uuid()) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; Ok(active) } @@ -129,9 +161,14 @@ impl Db { &self, normalized_host: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query("SELECT id, host FROM communities WHERE lower(host) = lower($1)") .bind(normalized_host) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { Ok(CommunityRecord { @@ -151,6 +188,11 @@ impl Db { owner_pubkey: &str, ) -> Result> { let owner_pubkey = owner_pubkey.to_ascii_lowercase(); + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT c.id, c.host, c.created_at, c.archived_at @@ -162,7 +204,7 @@ impl Db { "#, ) .bind(owner_pubkey) - .fetch_all(&self.pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() @@ -192,6 +234,11 @@ impl Db { /// community is authoritative; the host is read back for labelling only and /// is never used to re-derive the community. pub async fn lookup_community_host(&self, community_id: CommunityId) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::TenantResolution, + ) + .await?; let row = sqlx::query( r#" SELECT host @@ -201,7 +248,7 @@ impl Db { "#, ) .bind(community_id.as_uuid()) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { @@ -240,6 +287,11 @@ impl Db { community_id: CommunityId, icon: Option<&str>, ) -> Result<()> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; sqlx::query( r#" UPDATE communities @@ -249,7 +301,7 @@ impl Db { ) .bind(community_id.as_uuid()) .bind(icon) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(()) } @@ -263,6 +315,35 @@ impl Db { &self, normalized_host: &str, ) -> Result { + self.ensure_configured_community_with_operation( + normalized_host, + crate::observability::WriterOperation::Authorization, + ) + .await + } + + /// Ensure the deployment-configured community during process bootstrap. + #[datastore_span( + name = "ensure_configured_community_for_bootstrap", + system = "postgresql" + )] + pub async fn ensure_configured_community_for_bootstrap( + &self, + normalized_host: &str, + ) -> Result { + self.ensure_configured_community_with_operation( + normalized_host, + crate::observability::WriterOperation::Bootstrap, + ) + .await + } + + async fn ensure_configured_community_with_operation( + &self, + normalized_host: &str, + operation: crate::observability::WriterOperation, + ) -> Result { + let mut connection = crate::observability::acquire_writer(&self.pool, operation).await?; let row = sqlx::query( r#" INSERT INTO communities (host) @@ -272,7 +353,7 @@ impl Db { "#, ) .bind(normalized_host) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; let id: Uuid = row.try_get("id")?; @@ -297,16 +378,24 @@ impl Db { owner_pubkey: &str, ) -> Result { let owner_pubkey = owner_pubkey.to_ascii_lowercase(); - let mut tx = self.pool.begin().await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // Serialize on the owner pubkey so concurrent creates to the same // owner cannot both pass the ownership count check. - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(crate::relay_members::owner_count_advisory_lock_key( - &owner_pubkey, - )) - .execute(&mut *tx) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::Membership, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(crate::relay_members::owner_count_advisory_lock_key( + &owner_pubkey, + )) + .execute(&mut *tx), + ) + .await?; let row = sqlx::query( r#" @@ -384,6 +473,11 @@ impl Db { owner_pubkey: &str, protected_deployment_host: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#"UPDATE communities c SET archived_at = COALESCE(c.archived_at, now()) @@ -398,7 +492,7 @@ impl Db { .bind(normalized_host) .bind(owner_pubkey) .bind(protected_deployment_host) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { Ok(ArchivedCommunityRecord { @@ -416,6 +510,11 @@ impl Db { normalized_host: &str, owner_pubkey: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#"UPDATE communities c SET archived_at = NULL @@ -428,7 +527,7 @@ impl Db { ) .bind(normalized_host) .bind(owner_pubkey) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { Ok(UnarchivedCommunityRecord { @@ -444,6 +543,11 @@ impl Db { /// Internal relay producers use this to derive tenant context from the row /// they are acting on, rather than falling back to an implicit default. pub async fn community_of_channel(&self, channel_id: Uuid) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::TenantResolution, + ) + .await?; let row = sqlx::query( r#" SELECT community_id @@ -453,7 +557,7 @@ impl Db { "#, ) .bind(channel_id) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { @@ -488,6 +592,11 @@ impl Db { if channel_ids.is_empty() { return Ok(std::collections::HashMap::new()); } + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let rows = sqlx::query( r#" SELECT id, community_id @@ -497,7 +606,7 @@ impl Db { "#, ) .bind(channel_ids) - .fetch_all(&self.pool) + .fetch_all(&mut *connection) .await?; let mut out = std::collections::HashMap::with_capacity(rows.len()); diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 36af83989..f18e9f8a7 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -754,13 +754,22 @@ impl DeletionStore { /// Validate the deletion catalog contract required by relay serving. pub async fn validate_serving_catalog(&self) -> Result<()> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + self.validate_serving_catalog_on(&mut connection).await + } + + async fn validate_serving_catalog_on(&self, conn: &mut PgConnection) -> Result<()> { let runtime_columns = sqlx::query( "SELECT attname, format_type(atttypid, atttypmod) AS type_name, attnotnull \ FROM pg_attribute WHERE attrelid = 'communities'::regclass \ AND attname IN ('deletion_state', 'deletion_fence_generation', 'deleted_at') \ AND NOT attisdropped ORDER BY attname", ) - .fetch_all(&self.pool) + .fetch_all(&mut *conn) .await?; let column_contract = runtime_columns .iter() @@ -806,7 +815,7 @@ impl DeletionStore { ORDER BY table_name", ) .bind(&required_table_names) - .fetch_all(&self.pool) + .fetch_all(&mut *conn) .await? .into_iter() .collect(); @@ -826,7 +835,7 @@ impl DeletionStore { .copied() .map(str::to_owned) .collect::>(); - let live_fences = self.live_fenced_tables().await?; + let live_fences = live_fenced_tables_on(&mut *conn).await?; let missing_fences = required_fences .difference(&live_fences) .cloned() @@ -852,7 +861,7 @@ impl DeletionStore { AND p.proname = 'enforce_community_tombstone' \ AND NOT t.tgisinternal AND t.tgenabled = 'O')", ) - .fetch_one(&self.pool) + .fetch_one(&mut *conn) .await?; if !required_objects_present { return Err(DbError::DeletionSafety( @@ -1198,12 +1207,15 @@ impl DeletionStore { /// Already-acquired leases remain renewable, verifiable, and releasable so /// admitted remote effects retain their exclusion proof until completion. pub async fn begin_quiescing(&self, token: &LeaseToken) -> Result<()> { - let mut tx = self.pool.begin().await?; + let (mut tx, transaction_timer) = crate::observability::begin_transaction( + &self.pool, + crate::observability::TransactionOperation::BeginCommunityDeletionQuiescing, + ) + .await?; + transaction_timer + .observe(async { verify_lease(&mut tx, token, DeletionStage::Approved).await?; - sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") - .bind(token.community_id.as_uuid()) - .execute(&mut *tx) - .await?; + lock_community_deletion(&mut tx, token.community_id).await?; verify_lease(&mut tx, token, DeletionStage::Approved).await?; let (generation, archived_at): (i64, Option>) = sqlx::query_as( "SELECT deletion_fence_generation, archived_at FROM communities WHERE id = $1 FOR UPDATE", @@ -1247,16 +1259,21 @@ impl DeletionStore { .await?; tx.commit().await?; Ok(()) + }) + .await } /// Acquire the universal durable fence after all pre-quiesce serving leases drain. pub async fn fence(&self, token: &LeaseToken) -> Result { - let mut tx = self.pool.begin().await?; + let (mut tx, transaction_timer) = crate::observability::begin_transaction( + &self.pool, + crate::observability::TransactionOperation::FenceCommunityDeletion, + ) + .await?; + transaction_timer + .observe(async { verify_lease(&mut tx, token, DeletionStage::Approved).await?; - sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") - .bind(token.community_id.as_uuid()) - .execute(&mut *tx) - .await?; + lock_community_deletion(&mut tx, token.community_id).await?; verify_lease(&mut tx, token, DeletionStage::Approved).await?; let active_serving_writes = sqlx::query( "SELECT count(*)::BIGINT AS active_count, \ @@ -1319,6 +1336,8 @@ impl DeletionStore { .await?; tx.commit().await?; Ok(generation) + }) + .await } /// Freeze the exact post-fence storage binding manifest. @@ -1955,10 +1974,7 @@ impl DeletionStore { .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; // Every lifecycle transition takes the community lock before any row lock. // Inverting this order lets abort and the executor deadlock each other. - sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") - .bind(community_id.as_uuid()) - .execute(&mut *tx) - .await?; + lock_community_deletion(&mut tx, community_id).await?; let row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1 FOR UPDATE") .bind(request_id) .fetch_optional(&mut *tx) @@ -2242,10 +2258,7 @@ impl DeletionStore { tx: &mut Transaction<'_, Postgres>, community: CommunityId, ) -> Result<()> { - sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") - .bind(community.as_uuid()) - .execute(&mut **tx) - .await?; + lock_community_deletion_shared(tx, community).await?; let state: Option = sqlx::query_scalar( "SELECT deletion_state FROM communities WHERE id = $1 AND deleted_at IS NULL", ) @@ -2274,10 +2287,7 @@ impl DeletionStore { tx: &mut Transaction<'_, Postgres>, lease: &ServingWriteLease, ) -> Result<()> { - sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") - .bind(lease.community_id.as_uuid()) - .execute(&mut **tx) - .await?; + lock_community_deletion_shared(tx, lease.community_id).await?; let valid: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ JOIN communities community ON community.id = lease.community_id \ @@ -2330,7 +2340,12 @@ impl DeletionStore { lease_duration: Duration, ) -> Result { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); - let mut tx = self.pool.begin().await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // The assertion owns both the shared ordering lock and the supported // READ COMMITTED check. The lease table is trigger-excluded, so this // explicit admission is its database-enforced write fence. @@ -2394,11 +2409,13 @@ impl DeletionStore { lease_duration: Duration, ) -> Result<()> { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); - let mut tx = self.pool.begin().await?; - sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") - .bind(lease.community_id.as_uuid()) - .execute(&mut *tx) - .await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + lock_community_deletion_shared(&mut tx, lease.community_id).await?; let lease_until: Option> = sqlx::query_scalar( "UPDATE community_serving_write_leases lease \ SET lease_until = now() + make_interval(secs => $6), heartbeat_at = now() \ @@ -2430,6 +2447,11 @@ impl DeletionStore { /// Release a serving side-effect lease. A stale release is harmless. pub async fn release_serving_write_lease(&self, lease: &ServingWriteLease) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let deleted = sqlx::query( "DELETE FROM community_serving_write_leases \ WHERE id = $1 AND community_id = $2 AND owner = $3 AND generation = $4 \ @@ -2440,7 +2462,7 @@ impl DeletionStore { .bind(&lease.owner) .bind(lease.generation) .bind(lease.fence_generation) - .execute(&self.pool) + .execute(&mut *connection) .await? .rows_affected(); Ok(deleted == 1) @@ -2452,11 +2474,13 @@ impl DeletionStore { /// work remains blocked, preserving an accurate drain without abandoning an /// admitted remote effect. pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { - let mut tx = self.pool.begin().await?; - sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") - .bind(lease.community_id.as_uuid()) - .execute(&mut *tx) - .await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + lock_community_deletion_shared(&mut tx, lease.community_id).await?; let valid: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ JOIN communities community ON community.id = lease.community_id \ @@ -2487,6 +2511,11 @@ impl DeletionStore { /// Delete expired serving leases in a bounded batch. pub async fn reap_expired_serving_write_leases(&self, limit: i64) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let affected = sqlx::query( "WITH expired AS ( \ SELECT id FROM community_serving_write_leases \ @@ -2496,7 +2525,7 @@ impl DeletionStore { USING expired WHERE lease.id = expired.id", ) .bind(limit.clamp(1, 10_000)) - .execute(&self.pool) + .execute(&mut *connection) .await? .rows_affected(); Ok(affected) @@ -2504,6 +2533,11 @@ impl DeletionStore { /// Return serving-lease counts and dead-tuple estimate for observability. pub async fn serving_lease_stats(&self) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let row = sqlx::query( "SELECT count(*) FILTER (WHERE lease_until >= now())::BIGINT AS active, \ count(*) FILTER (WHERE lease_until < now())::BIGINT AS expired, \ @@ -2511,7 +2545,7 @@ impl DeletionStore { WHERE relname = 'community_serving_write_leases'), 0) AS dead_tuples \ FROM community_serving_write_leases", ) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; Ok(ServingLeaseStats { active: row.try_get("active")?, @@ -2522,12 +2556,17 @@ impl DeletionStore { /// Whether a community remains active and serving-write eligible. pub async fn is_serving_active(&self, community: CommunityId) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", ) .bind(community.as_uuid()) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await .map_err(Into::into) } @@ -2560,6 +2599,34 @@ impl DeletionStore { } } +async fn lock_community_deletion( + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, +) -> Result<()> { + crate::observability::observe_advisory_lock( + crate::observability::LockType::Deletion, + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(community.as_uuid()) + .execute(&mut **tx), + ) + .await?; + Ok(()) +} + +async fn lock_community_deletion_shared( + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, +) -> Result<()> { + crate::observability::observe_advisory_lock( + crate::observability::LockType::Deletion, + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(community.as_uuid()) + .execute(&mut **tx), + ) + .await?; + Ok(()) +} + /// Take the shared schema/destruction advisory lock for the current /// transaction. /// @@ -2568,10 +2635,13 @@ impl DeletionStore { /// whole run (see [`crate::migration::run_migrations`]); shared holders do /// not block each other, so concurrent deletion executors are unaffected. async fn lock_schema_destruction_shared(conn: &mut PgConnection) -> Result<()> { - sqlx::query("SELECT pg_advisory_xact_lock_shared($1)") - .bind(SCHEMA_DESTRUCTION_LOCK_KEY) - .execute(conn) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::MigrationSchemaSafety, + sqlx::query("SELECT pg_advisory_xact_lock_shared($1)") + .bind(SCHEMA_DESTRUCTION_LOCK_KEY) + .execute(conn), + ) + .await?; Ok(()) } @@ -3033,7 +3103,38 @@ fn bound_text(input: &str, max: usize) -> String { impl Db { /// Validate the minimum deletion fence catalog required by serving paths. pub async fn validate_deletion_serving_catalog(&self) -> Result<()> { - self.deletion_store().validate_serving_catalog().await + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + self.deletion_store() + .validate_serving_catalog_on(&mut connection) + .await + } + + /// Validate the serving catalog inside the readiness request's absolute + /// deadline, attributing only the one real writer checkout to readiness. + pub async fn validate_deletion_serving_catalog_for_readiness( + &self, + deadline: tokio::time::Instant, + ) -> Result<()> { + let mut connection = crate::observability::acquire_writer_until( + &self.pool, + crate::observability::WriterOperation::Readiness, + deadline, + ) + .await?; + match tokio::time::timeout_at( + deadline, + self.deletion_store() + .validate_serving_catalog_on(&mut connection), + ) + .await + { + Err(_) => Err(sqlx::Error::PoolTimedOut.into()), + Ok(result) => result, + } } /// Validate the exact live community-deletion tenant catalog for destruction. @@ -3225,7 +3326,7 @@ mod postgres_tests { async fn store() -> (Db, DeletionStore) { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 -- local test-only credentials let db = Db::new(&DbConfig { database_url, max_connections: 5, @@ -3893,7 +3994,7 @@ mod postgres_tests { .expect("won claim"); let mut open_write = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("open write transaction"); sqlx::query("INSERT INTO pubkey_allowlist (community_id, pubkey) VALUES ($1, $2)") diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index ed6af6351..1f01f5aa5 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -8,6 +8,7 @@ use crate::insert_mentions; use crate::Db; use crate::RouteDecision; use crate::RoutePredicate; +use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; use nostr::Event; use sqlx::{PgConnection, PgPool, Postgres, QueryBuilder, Row, Transaction}; @@ -524,6 +525,26 @@ pub async fn huddle_started_link_exists( ephemeral_channel_id: Uuid, creator_pubkey: &[u8], ) -> Result { + huddle_started_link_exists_with_operation( + pool, + community_id, + parent_channel_id, + ephemeral_channel_id, + creator_pubkey, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn huddle_started_link_exists_with_operation( + pool: &PgPool, + community_id: CommunityId, + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + creator_pubkey: &[u8], + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let uuid_needle = format!("%{}%", ephemeral_channel_id); let candidates: Vec = sqlx::query_scalar( r#" @@ -547,7 +568,7 @@ pub async fn huddle_started_link_exists( .bind(HUDDLE_LINK_CONTENT_MAX_BYTES) .bind(uuid_needle) .bind(HUDDLE_LINK_CANDIDATE_LIMIT) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(candidates @@ -564,7 +585,11 @@ pub async fn insert_event( event: &Event, channel_id: Option, ) -> Result<(StoredEvent, bool)> { - let mut connection = pool.acquire().await?; + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; insert_event_on(&mut connection, community_id, event, channel_id).await } @@ -645,7 +670,20 @@ async fn insert_event_on( /// Uses `QueryBuilder` for dynamic filter composition — avoids string concatenation /// while keeping all user values in bind parameters. pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result> { - let mut conn = pool.acquire().await?; + query_events_with_operation( + pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await +} + +pub(crate) async fn query_events_with_operation( + pool: &PgPool, + q: &EventQuery, + operation: crate::observability::WriterOperation, +) -> Result> { + let mut conn = crate::observability::acquire_writer(pool, operation).await?; query_events_on(&mut conn, q).await } @@ -977,6 +1015,11 @@ pub async fn query_latest_owner_authored_heads( kind: i32, limit: i64, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let rows = sqlx::query( "SELECT DISTINCT ON (e.d_tag) \ e.id, e.pubkey, e.created_at, e.kind, e.tags, e.content, \ @@ -995,7 +1038,7 @@ pub async fn query_latest_owner_authored_heads( .bind(community_id.as_uuid()) .bind(kind) .bind(limit) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; let mut out = Vec::with_capacity(rows.len()); @@ -1011,7 +1054,11 @@ pub async fn query_latest_owner_authored_heads( /// /// Uses the same filter logic as `query_events` but returns only the count. pub async fn count_events(pool: &PgPool, q: &EventQuery) -> Result { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; count_events_on(&mut conn, q).await } @@ -1179,12 +1226,17 @@ pub async fn soft_delete_event( community_id: CommunityId, event_id: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(event_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -1211,6 +1263,11 @@ pub async fn soft_delete_by_coordinate( ) -> Result { let deletion_created_at = DateTime::from_timestamp(deletion_created_at_secs, 0) .ok_or(DbError::InvalidTimestamp(deletion_created_at_secs))?; + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() \ WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL \ @@ -1221,7 +1278,7 @@ pub async fn soft_delete_by_coordinate( .bind(pubkey) .bind(d_tag) .bind(deletion_created_at) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -1239,7 +1296,12 @@ pub async fn soft_delete_event_and_update_thread( parent_event_id: Option<&[u8]>, root_event_id: Option<&[u8]>, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", @@ -1287,6 +1349,11 @@ pub async fn get_last_message_at( community_id: CommunityId, channel_id: uuid::Uuid, ) -> Result>> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let row = sqlx::query( "SELECT created_at FROM events \ WHERE community_id = $1 AND channel_id = $2 AND deleted_at IS NULL \ @@ -1294,7 +1361,7 @@ pub async fn get_last_message_at( ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1331,6 +1398,11 @@ pub async fn get_last_authored_event_at( community_id: CommunityId, pubkey: &[u8], ) -> Result>> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let row = sqlx::query( "SELECT created_at FROM events \ WHERE community_id = $1 AND pubkey = $2 AND deleted_at IS NULL \ @@ -1338,7 +1410,7 @@ pub async fn get_last_authored_event_at( ) .bind(community_id.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1359,6 +1431,11 @@ pub async fn get_last_message_at_bulk( if channel_ids.is_empty() { return Ok(std::collections::HashMap::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let mut qb: QueryBuilder = QueryBuilder::new( "SELECT channel_id, MAX(created_at) as last_at FROM events \ @@ -1372,7 +1449,7 @@ pub async fn get_last_message_at_bulk( } qb.push(") GROUP BY channel_id"); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *connection).await?; let mut map = std::collections::HashMap::with_capacity(rows.len()); for row in rows { @@ -1393,13 +1470,29 @@ pub async fn get_event_by_id( community_id: CommunityId, id_bytes: &[u8], ) -> Result> { + get_event_by_id_with_operation( + pool, + community_id, + id_bytes, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +pub(crate) async fn get_event_by_id_with_operation( + pool: &PgPool, + community_id: CommunityId, + id_bytes: &[u8], + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let row = sqlx::query( "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id \ FROM events WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL ORDER BY created_at DESC LIMIT 1", ) .bind(community_id.as_uuid()) .bind(id_bytes) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1420,6 +1513,11 @@ pub async fn get_latest_global_replaceable( kind: i32, pubkey_bytes: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id \ FROM events \ @@ -1430,7 +1528,7 @@ pub async fn get_latest_global_replaceable( .bind(community_id.as_uuid()) .bind(kind) .bind(pubkey_bytes) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1449,13 +1547,29 @@ pub async fn get_event_by_id_including_deleted( community_id: CommunityId, id_bytes: &[u8], ) -> Result> { + get_event_by_id_including_deleted_with_operation( + pool, + community_id, + id_bytes, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +pub(crate) async fn get_event_by_id_including_deleted_with_operation( + pool: &PgPool, + community_id: CommunityId, + id_bytes: &[u8], + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let row = sqlx::query( "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id \ FROM events WHERE community_id = $1 AND id = $2 ORDER BY created_at DESC LIMIT 1", ) .bind(community_id.as_uuid()) .bind(id_bytes) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1472,11 +1586,26 @@ pub async fn get_events_by_ids( pool: &PgPool, community_id: CommunityId, ids: &[&[u8]], +) -> Result> { + get_events_by_ids_with_operation( + pool, + community_id, + ids, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await +} + +pub(crate) async fn get_events_by_ids_with_operation( + pool: &PgPool, + community_id: CommunityId, + ids: &[&[u8]], + operation: crate::observability::WriterOperation, ) -> Result> { if ids.is_empty() { return Ok(vec![]); } - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer(pool, operation).await?; get_events_by_ids_on(&mut conn, community_id, ids).await } @@ -1722,7 +1851,12 @@ pub async fn insert_event_with_thread_metadata( channel_id: Option, thread_meta: Option>, ) -> Result<(StoredEvent, bool)> { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let result = insert_event_with_thread_metadata_tx(&mut tx, community_id, event, channel_id, thread_meta) .await?; @@ -1743,7 +1877,12 @@ pub async fn insert_block_action_once( instance_event_id: &[u8], idempotency_key: Uuid, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let claimed_event_id: Option> = sqlx::query_scalar( r#" @@ -1824,7 +1963,12 @@ pub async fn insert_reaction_event_with_thread_metadata( actor_pubkey: &[u8], emoji: &str, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let target_row = sqlx::query( "SELECT created_at FROM events \ @@ -1985,6 +2129,11 @@ pub async fn query_in_progress_task_heads( pool: &PgPool, batch_limit: i64, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let kind_i32 = KIND_TASK as i32; let rows = sqlx::query( r#" @@ -2046,7 +2195,7 @@ pub async fn query_in_progress_task_heads( ) .bind(kind_i32) .bind(batch_limit) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -2100,6 +2249,11 @@ pub async fn query_due_snoozed_task_heads( now: i64, batch_limit: i64, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let kind_i32 = KIND_TASK as i32; let rows = sqlx::query( r#" @@ -2132,7 +2286,7 @@ pub async fn query_due_snoozed_task_heads( .bind(kind_i32) .bind(now) .bind(batch_limit) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -2162,6 +2316,11 @@ pub async fn claim_task_wake( task_id: &str, wake_at: i64, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let result = sqlx::query( r#" INSERT INTO task_wake_claims (community_id, task_id, wake_at) @@ -2172,7 +2331,7 @@ pub async fn claim_task_wake( .bind(community_id.as_uuid()) .bind(task_id) .bind(wake_at) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -2204,7 +2363,46 @@ impl Db { /// [`Db::query_events_routed`] instead — converting a caller is an /// explicit, per-callsite decision, never a change to this method. pub async fn query_events(&self, q: &EventQuery) -> Result> { - query_events(&self.pool, q).await + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::Authorization, + ) + .await + } + + /// Query authoritative event state that directly controls a durable event + /// mutation or its post-commit side effects. + #[datastore_span(name = "query_events_for_event_write", system = "postgresql")] + pub async fn query_events_for_event_write(&self, q: &EventQuery) -> Result> { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + + /// Query authoritative event state for startup reconciliation. + #[datastore_span(name = "query_events_for_bootstrap", system = "postgresql")] + pub async fn query_events_for_bootstrap(&self, q: &EventQuery) -> Result> { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::Bootstrap, + ) + .await + } + + /// Query authoritative event state for background reconciliation or repair. + #[datastore_span(name = "query_events_for_maintenance", system = "postgresql")] + pub async fn query_events_for_maintenance(&self, q: &EventQuery) -> Result> { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::Maintenance, + ) + .await } /// [`Db::query_events`] with replica routing — the opt-in fast path for @@ -2229,7 +2427,14 @@ impl Db { q: &EventQuery, ) -> Result> { let predicate = RoutePredicate::for_query(q, self.replica_read_max_age.is_some()); - match self.route_read(path, predicate).await { + match self + .route_read( + path, + predicate, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::query_events_on(&mut tx, q).await { Ok(events) => { @@ -2241,11 +2446,23 @@ impl Db { // writer rather than surfacing a routed error. tracing::warn!(path, "replica read failed; re-running on writer: {e}"); Self::record_route(path, "writer", "replica_error"); - crate::event::query_events(&self.pool, q).await + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } - RouteDecision::Writer => crate::event::query_events(&self.pool, q).await, + RouteDecision::Writer => { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await + } } } @@ -2262,7 +2479,14 @@ impl Db { path: &'static str, q: &EventQuery, ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::query_events_on(&mut tx, q).await { Ok(events) => { @@ -2272,11 +2496,23 @@ impl Db { Err(e) => { tracing::warn!(path, "replica read failed; re-running on writer: {e}"); Self::record_route(path, "writer", "replica_error"); - crate::event::query_events(&self.pool, q).await + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } - RouteDecision::Writer => crate::event::query_events(&self.pool, q).await, + RouteDecision::Writer => { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await + } } } @@ -2299,7 +2535,14 @@ impl Db { /// statement than a page briefly showing a deleted row. `Bounded` ties /// the error to the accepted budget `B`. pub async fn count_events_routed(&self, path: &'static str, q: &EventQuery) -> Result { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::count_events_on(&mut tx, q).await { Ok(count) => { @@ -2336,6 +2579,29 @@ impl Db { .await } + /// Validate a huddle link while admitting a huddle event for persistence. + #[datastore_span( + name = "huddle_started_link_exists_for_event_write", + system = "postgresql" + )] + pub async fn huddle_started_link_exists_for_event_write( + &self, + community_id: CommunityId, + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + creator_pubkey: &[u8], + ) -> Result { + crate::event::huddle_started_link_exists_with_operation( + &self.pool, + community_id, + parent_channel_id, + ephemeral_channel_id, + creator_pubkey, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Fetch the latest replaceable event for a (kind, pubkey) pair. /// /// Uses canonical NIP-16 ordering: `created_at DESC, id ASC`. @@ -2362,6 +2628,23 @@ impl Db { crate::event::get_event_by_id(&self.pool, community_id, id_bytes).await } + /// Fetch an event as a prerequisite of an event write or durable + /// post-write side effect. + #[datastore_span(name = "get_event_by_id_for_event_write", system = "postgresql")] + pub async fn get_event_by_id_for_event_write( + &self, + community_id: CommunityId, + id_bytes: &[u8], + ) -> Result> { + crate::event::get_event_by_id_with_operation( + &self.pool, + community_id, + id_bytes, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Fetches a single event by its raw ID bytes, **including soft-deleted rows**. pub async fn get_event_by_id_including_deleted( &self, @@ -2371,6 +2654,26 @@ impl Db { crate::event::get_event_by_id_including_deleted(&self.pool, community_id, id_bytes).await } + /// Fetch an event including tombstones as a prerequisite of an event + /// write or durable post-write side effect. + #[datastore_span( + name = "get_event_by_id_including_deleted_for_event_write", + system = "postgresql" + )] + pub async fn get_event_by_id_including_deleted_for_event_write( + &self, + community_id: CommunityId, + id_bytes: &[u8], + ) -> Result> { + crate::event::get_event_by_id_including_deleted_with_operation( + &self.pool, + community_id, + id_bytes, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Soft-deletes an event. Returns `Ok(true)` if deleted, `Ok(false)` if already deleted. pub async fn soft_delete_event( &self, @@ -2445,7 +2748,13 @@ impl Db { community_id: CommunityId, ids: &[&[u8]], ) -> Result> { - crate::event::get_events_by_ids(&self.pool, community_id, ids).await + crate::event::get_events_by_ids_with_operation( + &self.pool, + community_id, + ids, + crate::observability::WriterOperation::Authorization, + ) + .await } /// [`Db::get_events_by_ids`] with replica routing — same contract and @@ -2461,7 +2770,14 @@ impl Db { community_id: CommunityId, ids: &[&[u8]], ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::get_events_by_ids_on(&mut tx, community_id, ids).await { Ok(events) => { @@ -2471,12 +2787,24 @@ impl Db { Err(e) => { tracing::warn!(path, "replica read failed; re-running on writer: {e}"); Self::record_route(path, "writer", "replica_error"); - crate::event::get_events_by_ids(&self.pool, community_id, ids).await + crate::event::get_events_by_ids_with_operation( + &self.pool, + community_id, + ids, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } RouteDecision::Writer => { - crate::event::get_events_by_ids(&self.pool, community_id, ids).await + crate::event::get_events_by_ids_with_operation( + &self.pool, + community_id, + ids, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } @@ -2510,6 +2838,11 @@ impl Db { /// Idempotent — safe to call on every startup. No-ops when all rows are already populated. /// Runs a single UPDATE touching only NIP-33 rows with NULL d_tag. pub async fn backfill_d_tags(&self) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; let result = sqlx::query( "UPDATE events \ SET d_tag = COALESCE( \ @@ -2519,7 +2852,7 @@ impl Db { ) \ WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL", ) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } @@ -2531,6 +2864,11 @@ impl Db { channel_id: Uuid, relay_pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() \ WHERE community_id = $1 AND channel_id = $2 AND pubkey = $3 AND deleted_at IS NULL AND kind IN (39000, 39001, 39002)", @@ -2538,7 +2876,7 @@ impl Db { .bind(community_id.as_uuid()) .bind(channel_id) .bind(relay_pubkey) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } diff --git a/crates/buzz-db/src/store/feed.rs b/crates/buzz-db/src/store/feed.rs index 2c0dad5d0..60fa6a06f 100644 --- a/crates/buzz-db/src/store/feed.rs +++ b/crates/buzz-db/src/store/feed.rs @@ -137,7 +137,11 @@ pub async fn query_mentions( since: Option>, limit: i64, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; query_mentions_on( &mut conn, community, @@ -246,7 +250,11 @@ pub async fn query_needs_action( since: Option>, limit: i64, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; query_needs_action_on( &mut conn, community, @@ -315,7 +323,11 @@ pub async fn query_activity( since: Option>, limit: i64, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; query_activity_on(&mut conn, community, accessible_channel_ids, since, limit).await } @@ -371,7 +383,14 @@ impl Db { since: Option>, limit: i64, ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => match crate::feed::query_mentions_on( &mut tx, community, @@ -451,7 +470,14 @@ impl Db { since: Option>, limit: i64, ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => { match crate::feed::query_needs_action_on( &mut tx, @@ -519,7 +545,14 @@ impl Db { since: Option>, limit: i64, ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => match crate::feed::query_activity_on( &mut tx, community, @@ -1321,4 +1354,40 @@ mod tests { let unique: std::collections::HashSet> = byte_seqs.into_iter().collect(); assert_eq!(unique.len(), 5, "all channel IDs must be distinct"); } + + /// `insert_mentions` must index every p-tag even past Postgres's + /// bind-parameter statement cap. + /// + /// Relay-signed kind 39002 member snapshots carry one p-tag per channel + /// member, and a multi-row INSERT binds 6 parameters per row — a single + /// statement tops out at ~10.9k rows against the 65,535-parameter limit. + /// Clients discover their channels via `{kinds:[39002], "#p":[me]}`, so a + /// failed insert silently breaks discovery for the whole channel. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn insert_mentions_indexes_rosters_past_bind_parameter_cap() { + let pool = setup_pool().await; + let community = CommunityId::from_uuid(make_test_community(&pool).await); + let channel = insert_test_channel(&pool, community).await; + + // 11,000 rows x 6 binds = 66,000 > 65,535: overflows a single statement. + let mention_count = 11_000usize; + let tags: Vec = (1..=mention_count) + .map(|n| Tag::parse(["p", &format!("{n:064x}")]).expect("p tag")) + .collect(); + let event = store_feed_event(&pool, community, 39002, "", Some(channel), tags).await; + + let indexed: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM event_mentions WHERE community_id = $1 AND event_id = $2", + ) + .bind(community.as_uuid()) + .bind(event.id.as_bytes().as_slice()) + .fetch_one(&pool) + .await + .expect("count indexed mentions"); + assert_eq!( + indexed as usize, mention_count, + "every roster p-tag must land in event_mentions" + ); + } } diff --git a/crates/buzz-db/src/store/git_repo.rs b/crates/buzz-db/src/store/git_repo.rs index 1f7d043b7..17f45f206 100644 --- a/crates/buzz-db/src/store/git_repo.rs +++ b/crates/buzz-db/src/store/git_repo.rs @@ -50,13 +50,18 @@ pub async fn repo_name_owner( community: CommunityId, repo_id: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT owner_pubkey FROM git_repo_names \ WHERE community_id = $1 AND repo_id = $2", ) .bind(community.as_uuid()) .bind(repo_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|r| r.try_get("owner_pubkey")) .transpose() @@ -85,6 +90,11 @@ pub async fn reserve_repo_name( repo_id: &str, owner_pubkey: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; // Atomic claim: insert only if the (community, repo) is free. RETURNING is // non-empty exactly when *this* statement inserted the row, so it cleanly // distinguishes "I claimed it" from "someone already holds it" without a @@ -98,7 +108,7 @@ pub async fn reserve_repo_name( .bind(community.as_uuid()) .bind(repo_id) .bind(owner_pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; if inserted.is_some() { @@ -113,7 +123,7 @@ pub async fn reserve_repo_name( ) .bind(community.as_uuid()) .bind(repo_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match existing { @@ -145,13 +155,18 @@ pub async fn count_repos_for_owner( community: CommunityId, owner_pubkey: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) AS n FROM git_repo_names \ WHERE community_id = $1 AND owner_pubkey = $2", ) .bind(community.as_uuid()) .bind(owner_pubkey) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; row.try_get("n").map_err(crate::error::DbError::from) } @@ -168,6 +183,11 @@ pub async fn release_repo_name( repo_id: &str, owner_pubkey: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let result = sqlx::query( "DELETE FROM git_repo_names \ WHERE community_id = $1 AND repo_id = $2 AND owner_pubkey = $3", @@ -175,7 +195,7 @@ pub async fn release_repo_name( .bind(community.as_uuid()) .bind(repo_id) .bind(owner_pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } diff --git a/crates/buzz-db/src/store/moderation.rs b/crates/buzz-db/src/store/moderation.rs index d9f8545cc..781a7c6e9 100644 --- a/crates/buzz-db/src/store/moderation.rs +++ b/crates/buzz-db/src/store/moderation.rs @@ -444,6 +444,11 @@ pub async fn restriction_state( community: CommunityId, pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#" SELECT @@ -455,7 +460,7 @@ pub async fn restriction_state( ) .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { diff --git a/crates/buzz-db/src/store/operator_analytics.rs b/crates/buzz-db/src/store/operator_analytics.rs index 9c6f5b265..cab033b01 100644 --- a/crates/buzz-db/src/store/operator_analytics.rs +++ b/crates/buzz-db/src/store/operator_analytics.rs @@ -1287,7 +1287,7 @@ impl crate::Db { limit: i64, ) -> Result> { let limit = bounded_rollup_limit(limit); - let mut tx = self.begin_transaction().await?; + let mut tx = self.begin_event_write_transaction().await?; let rows = fetch_activity_batch_on(&mut tx, community_id, cursor, limit, None).await?; tx.rollback().await?; Ok(rows) @@ -1298,7 +1298,7 @@ impl crate::Db { &self, community_id: CommunityId, ) -> Result { - let mut tx = self.begin_transaction().await?; + let mut tx = self.begin_event_write_transaction().await?; let cursor = read_operator_cursor_on(&mut tx, community_id, false) .await? .unwrap_or_else(OperatorActivityCursor::start); @@ -1316,7 +1316,7 @@ impl crate::Db { limit: i64, ) -> Result { let limit = bounded_rollup_limit(limit); - let mut tx = self.begin_transaction().await?; + let mut tx = self.begin_event_write_transaction().await?; let lock_key = activity_lock_key(community_id); sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") .bind(lock_key) @@ -1405,7 +1405,7 @@ impl crate::Db { "operator activity rebuild batch size must be between 100 and 5000".to_owned(), )); } - let mut tx = self.begin_transaction().await?; + let mut tx = self.begin_event_write_transaction().await?; let lock_key = activity_lock_key(community_id); sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") .bind(lock_key) diff --git a/crates/buzz-db/src/store/partition.rs b/crates/buzz-db/src/store/partition.rs index 450370d38..72b5f8461 100644 --- a/crates/buzz-db/src/store/partition.rs +++ b/crates/buzz-db/src/store/partition.rs @@ -15,6 +15,11 @@ const PARTITIONED_TABLES: &[&str] = &["events", "delivery_log"]; /// Ensures monthly partition tables exist for the next `months_ahead` months. pub async fn ensure_future_partitions(pool: &PgPool, months_ahead: u32) -> Result<()> { let now = Utc::now(); + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; for i in 0..=(months_ahead as i32) { let year = now.year(); @@ -49,7 +54,7 @@ pub async fn ensure_future_partitions(pool: &PgPool, months_ahead: u32) -> Resul let end_str = end.format("%Y-%m-%d").to_string(); for table in PARTITIONED_TABLES { - ensure_partition(pool, table, &start_str, &end_str, &suffix).await?; + ensure_partition(&mut connection, table, &start_str, &end_str, &suffix).await?; } } @@ -73,7 +78,7 @@ fn validate_date_str(s: &str) -> bool { } async fn ensure_partition( - pool: &PgPool, + connection: &mut sqlx::PgConnection, table_name: &str, start_date_str: &str, end_date_str: &str, @@ -114,7 +119,7 @@ async fn ensure_partition( "#, ) .bind(&partition_name) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; @@ -128,7 +133,10 @@ async fn ensure_partition( FOR VALUES FROM ('{start_date_str}') TO ('{end_date_str}')" ); - match sqlx::query(sqlx::AssertSqlSafe(sql)).execute(pool).await { + match sqlx::query(sqlx::AssertSqlSafe(sql)) + .execute(&mut *connection) + .await + { Ok(_) => { info!("added partition {partition_name}"); Ok(()) diff --git a/crates/buzz-db/src/store/push.rs b/crates/buzz-db/src/store/push.rs index 8403429fb..002fa8fd0 100644 --- a/crates/buzz-db/src/store/push.rs +++ b/crates/buzz-db/src/store/push.rs @@ -13,6 +13,21 @@ use uuid::Uuid; use crate::error::Result; +async fn acquire_operation_connection( + pool: &PgPool, + operation: crate::observability::WriterOperation, +) -> Result> { + Ok(crate::observability::acquire_writer(pool, operation).await?) +} + +async fn begin_operation_transaction( + pool: &PgPool, + operation: crate::observability::WriterOperation, +) -> Result> { + let connection = acquire_operation_connection(pool, operation).await?; + Ok(sqlx::Transaction::begin(connection, None).await?) +} + /// Namespace for the per-community push-gate advisory lock. Must match the /// key built inside the `enqueue_push_match_job` trigger (migration 0023): /// event inserts take it SHARED there; every lease transition that can make @@ -26,10 +41,13 @@ async fn acquire_push_gate_lock( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, community: CommunityId, ) -> Result<()> { - sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") - .bind(format!("{PUSH_GATE_LOCK_NAMESPACE}{}", community.as_uuid())) - .execute(&mut **tx) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::PushGate, + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!("{PUSH_GATE_LOCK_NAMESPACE}{}", community.as_uuid())) + .execute(&mut **tx), + ) + .await?; Ok(()) } @@ -221,7 +239,13 @@ pub async fn accept_lease_event( max_active_leases: i64, ) -> Result { let author = event.pubkey.as_bytes(); - let mut tx = pool.begin().await?; + let (mut tx, transaction_timer) = crate::observability::begin_transaction( + pool, + crate::observability::TransactionOperation::AcceptPushLeaseEvent, + ) + .await?; + transaction_timer + .observe(async { let mut address_lock = Vec::with_capacity(16 + author.len() + installation_id.len()); address_lock.extend_from_slice(community.as_uuid().as_bytes()); address_lock.extend_from_slice(author); @@ -231,14 +255,20 @@ pub async fn accept_lease_event( author_lock.extend_from_slice(community.as_uuid().as_bytes()); author_lock.extend_from_slice(author); let author_lock = i64::from_le_bytes(Sha256::digest(&author_lock)[..8].try_into().unwrap()); - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(address_lock) - .execute(&mut *tx) - .await?; - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(author_lock) - .execute(&mut *tx) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::PushGate, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(address_lock) + .execute(&mut *tx), + ) + .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::PushGate, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(author_lock) + .execute(&mut *tx), + ) + .await?; // T1b: an activation can flip the community from "no eligible lease" to // "eligible", so it must serialize against the trigger's shared gate lock. // Acquired after the address/author locks to keep one global lock order. @@ -393,6 +423,8 @@ pub async fn accept_lease_event( } tx.commit().await?; Ok(AcceptLeaseOutcome::Accepted) + }) + .await } fn constraint_acceptance_outcome(error: &sqlx::Error) -> Option { @@ -472,7 +504,9 @@ async fn replace_lease( // lease" to "eligible"; serialize it against the trigger's shared gate // lock (gate → lease row, matching accept_lease_event's global order). // Revocations (is_active = false) never make eligibility true and skip it. - let mut tx = pool.begin().await?; + let mut tx = + begin_operation_transaction(pool, crate::observability::WriterOperation::EventWrite) + .await?; if is_active { acquire_push_gate_lock(&mut tx, community).await?; } @@ -625,7 +659,9 @@ pub async fn enqueue_wakes( if requests.is_empty() { return Ok(Vec::new()); } - let mut tx = pool.begin().await?; + let mut tx = + begin_operation_transaction(pool, crate::observability::WriterOperation::Maintenance) + .await?; // 1. Lock and read the current lease row for every distinct requested // (author, installation), in deterministic order. @@ -828,7 +864,13 @@ pub async fn claim_due_match_batch( lease_until, |pool, community, ids| async move { let refs: Vec<&[u8]> = ids.iter().map(Vec::as_slice).collect(); - crate::event::get_events_by_ids(&pool, community, &refs).await + crate::event::get_events_by_ids_with_operation( + &pool, + community, + &refs, + crate::observability::WriterOperation::Maintenance, + ) + .await }, ) .await @@ -845,6 +887,9 @@ where Fut: std::future::Future>>, { let claim_id = Uuid::new_v4(); + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let rows = sqlx::query( r#" WITH target AS ( @@ -879,11 +924,16 @@ where .bind(lease_until) .bind(MAX_MATCH_ATTEMPTS) .bind(limit) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; if rows.is_empty() { return Ok(None); } + // The claim query is a single autocommitted statement. Release its pool + // slot before loading source events, because the production loader owns a + // separately attributed acquisition. Holding this connection across the + // load would self-starve a supported size-one writer pool. + drop(connection); let community = CommunityId::from_uuid(rows[0].try_get("community_id")?); let mut attempts = std::collections::HashMap::with_capacity(rows.len()); for row in &rows { @@ -906,6 +956,9 @@ where // recoverable after their claim lease expires. let gone: Vec> = attempts.into_keys().collect(); if !gone.is_empty() { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; sqlx::query( "DELETE FROM push_match_queue \ WHERE community_id=$1 AND claim_id=$2 AND state='matching' AND event_id = ANY($3)", @@ -913,7 +966,7 @@ where .bind(community.as_uuid()) .bind(claim_id) .bind(&gone) - .execute(pool) + .execute(&mut *connection) .await?; } if jobs.is_empty() { @@ -933,26 +986,32 @@ where /// served by the due partial index, so putting it in every claim made claims /// slower exactly when a backlog needed them fastest. pub async fn reap_exhausted_matches(pool: &PgPool) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; Ok(sqlx::query( "DELETE FROM push_match_queue WHERE attempts >= $1 \ AND (state='pending' OR (state='matching' AND lease_until < now())) \ AND community_write_allowed(community_id)", ) .bind(MAX_MATCH_ATTEMPTS) - .execute(pool) + .execute(&mut *connection) .await? .rows_affected()) } /// Load active endpoint-enabled leases for one tenant. pub async fn active_match_leases(pool: &PgPool, community: CommunityId) -> Result> { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let rows = sqlx::query( "SELECT author, installation_id, generation, subscriptions, expires_at \ FROM push_leases WHERE community_id=$1 AND active AND endpoint_enabled \ AND expires_at > EXTRACT(EPOCH FROM now())::bigint", ) .bind(community.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() .map(|row| { @@ -979,6 +1038,9 @@ pub async fn complete_match_batch( if event_ids.is_empty() { return Ok(0); } + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; Ok(sqlx::query( "DELETE FROM push_match_queue \ WHERE community_id=$1 AND claim_id=$2 AND state='matching' AND event_id = ANY($3)", @@ -986,7 +1048,7 @@ pub async fn complete_match_batch( .bind(community.as_uuid()) .bind(claim_id) .bind(event_ids) - .execute(pool) + .execute(&mut *connection) .await? .rows_affected()) } @@ -1003,6 +1065,9 @@ pub async fn retry_match_batch( if event_ids.is_empty() { return Ok(0); } + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; Ok(sqlx::query( "UPDATE push_match_queue \ SET state='pending', claim_id=NULL, lease_until=NULL, next_attempt_at=$4 \ @@ -1012,7 +1077,7 @@ pub async fn retry_match_batch( .bind(claim_id) .bind(event_ids) .bind(next) - .execute(pool) + .execute(&mut *connection) .await? .rows_affected()) } @@ -1028,6 +1093,9 @@ pub async fn claim_due_wakes( lease_until: DateTime, ) -> Result> { let claim_id = Uuid::new_v4(); + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let rows = sqlx::query( r#" WITH candidates AS ( @@ -1074,7 +1142,7 @@ pub async fn claim_due_wakes( .bind(limit) .bind(claim_id) .bind(lease_until) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter().map(row_to_claimed_wake).collect() @@ -1091,6 +1159,9 @@ pub async fn revalidate_wake_for_send( id: Uuid, claim_id: Uuid, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let row = sqlx::query( r#" SELECT o.community_id, o.id, o.claim_id, o.event_id, e.channel_id, @@ -1121,7 +1192,7 @@ pub async fn revalidate_wake_for_send( .bind(community.as_uuid()) .bind(id) .bind(claim_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(row_to_claimed_wake) @@ -1138,6 +1209,9 @@ pub async fn complete_wake( id: Uuid, claim_id: Uuid, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_wake_outbox \ SET state = 'delivered', claim_id = NULL, lease_until = NULL \ @@ -1146,7 +1220,7 @@ pub async fn complete_wake( .bind(community.as_uuid()) .bind(id) .bind(claim_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1159,6 +1233,9 @@ pub async fn retry_wake( claim_id: Uuid, next_attempt_at: DateTime, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_wake_outbox \ SET state = 'pending', next_attempt_at = $4, claim_id = NULL, lease_until = NULL \ @@ -1168,7 +1245,7 @@ pub async fn retry_wake( .bind(id) .bind(claim_id) .bind(next_attempt_at) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1180,6 +1257,9 @@ pub async fn fail_wake( id: Uuid, claim_id: Uuid, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_wake_outbox \ SET state = 'failed', claim_id = NULL, lease_until = NULL \ @@ -1188,7 +1268,7 @@ pub async fn fail_wake( .bind(community.as_uuid()) .bind(id) .bind(claim_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1204,6 +1284,9 @@ pub async fn disable_endpoint_generation( installation_id: &str, generation: i64, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_leases SET endpoint_enabled = false, updated_at = now() \ WHERE community_id = $1 AND author = $2 AND installation_id = $3 \ @@ -1213,7 +1296,7 @@ pub async fn disable_endpoint_generation( .bind(author) .bind(installation_id) .bind(generation) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1228,6 +1311,9 @@ pub async fn prune_wake_outbox( community: CommunityId, before: DateTime, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "DELETE FROM push_wake_outbox o \ WHERE o.community_id = $1 AND o.created_at < $2 \ @@ -1240,7 +1326,7 @@ pub async fn prune_wake_outbox( ) .bind(community.as_uuid()) .bind(before) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } @@ -1422,13 +1508,15 @@ impl Db { #[cfg(test)] mod tests { use super::*; + use sqlx::postgres::PgPoolOptions; use std::sync::Arc; + use std::time::Duration; use tokio::sync::Barrier; async fn setup_pool() -> PgPool { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".into()); + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".into()); // sadscan:disable np.postgres.1 -- local test-only credentials let pool = PgPool::connect(&database_url) .await .expect("connect to test DB"); @@ -2114,6 +2202,51 @@ mod tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn matcher_claim_and_load_support_size_one_pool() { + let setup = setup_pool().await; + sqlx::query("DELETE FROM push_match_queue") + .execute(&setup) + .await + .expect("drain matcher queue"); + let community = make_community(&setup).await; + activate(&setup, community, &[83; 32], "install", &[84; 32], 1).await; + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "size one") + .sign_with_keys(&nostr::Keys::generate()) + .expect("sign event"); + crate::event::insert_event(&setup, community, &event, None) + .await + .expect("insert event"); + setup.close().await; + + let pool = PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_millis(250)) + .connect(&crate::test_support::database_url()) + .await + .expect("connect size-one matcher pool"); + let batch = tokio::time::timeout( + Duration::from_secs(2), + claim_due_match_batch(&pool, 16, Utc::now() + chrono::Duration::minutes(1)), + ) + .await + .expect("matcher must not self-starve on a size-one pool") + .expect("claim and source load must succeed") + .expect("seeded matcher job must be claimed"); + assert_eq!(batch.community, community); + assert_eq!(batch.jobs.len(), 1); + assert_eq!(batch.jobs[0].event.event.id, event.id); + + let ids = vec![event.id.as_bytes().to_vec()]; + assert_eq!( + complete_match_batch(&pool, community, batch.claim_id, &ids) + .await + .expect("complete size-one matcher batch"), + 1 + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn matcher_claim_is_exclusive_across_workers() { diff --git a/crates/buzz-db/src/store/reaction.rs b/crates/buzz-db/src/store/reaction.rs index da62899f5..d6bc9d469 100644 --- a/crates/buzz-db/src/store/reaction.rs +++ b/crates/buzz-db/src/store/reaction.rs @@ -91,6 +91,11 @@ pub async fn add_reaction( emoji: &str, reaction_event_id: Option<&[u8]>, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query(ADD_REACTION_SQL) .bind(community.as_uuid()) .bind(event_created_at) @@ -98,7 +103,7 @@ pub async fn add_reaction( .bind(pubkey) .bind(emoji) .bind(reaction_event_id) - .execute(pool) + .execute(&mut *connection) .await?; // Three cases: @@ -148,6 +153,11 @@ pub async fn remove_reaction( pubkey: &[u8], emoji: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( r#" UPDATE reactions @@ -165,7 +175,7 @@ pub async fn remove_reaction( .bind(event_id) .bind(pubkey) .bind(emoji) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -179,6 +189,11 @@ pub async fn remove_reaction_by_source_event_id( community: CommunityId, reaction_event_id: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( r#" UPDATE reactions @@ -190,7 +205,7 @@ pub async fn remove_reaction_by_source_event_id( ) .bind(community.as_uuid()) .bind(reaction_event_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -205,6 +220,11 @@ pub async fn get_active_reaction_record( pubkey: &[u8], emoji: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query( r#" SELECT reaction_event_id @@ -223,7 +243,7 @@ pub async fn get_active_reaction_record( .bind(event_created_at) .bind(pubkey) .bind(emoji) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| -> Result { @@ -247,6 +267,11 @@ pub async fn set_reaction_event_id( emoji: &str, reaction_event_id: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( r#" UPDATE reactions @@ -265,7 +290,7 @@ pub async fn set_reaction_event_id( .bind(event_id) .bind(pubkey) .bind(emoji) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -288,6 +313,11 @@ pub async fn get_reactions( limit: u32, _cursor: Option<&str>, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; // Two-step query: first get the limited set of distinct emoji groups, // then fetch all rows for those groups. This ensures `limit` applies to // emoji groups (the API contract), not raw rows — so one busy emoji @@ -317,7 +347,7 @@ pub async fn get_reactions( .bind(event_id) .bind(event_created_at) .bind(limit as i64) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; // Group individual rows by emoji in Rust. @@ -378,6 +408,11 @@ pub async fn get_reactions_bulk( // Run one query per event. For typical message-list sizes (<=100 events) // this is acceptable; a single-query approach with dynamic IN clauses over // composite keys can be added later if needed. + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let mut entries = Vec::new(); for (event_id, event_created_at) in event_ids { @@ -396,7 +431,7 @@ pub async fn get_reactions_bulk( .bind(community.as_uuid()) .bind(*event_id) .bind(event_created_at) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; if rows.is_empty() { diff --git a/crates/buzz-db/src/store/relay_invite.rs b/crates/buzz-db/src/store/relay_invite.rs index ec29fdcff..8970c5673 100644 --- a/crates/buzz-db/src/store/relay_invite.rs +++ b/crates/buzz-db/src/store/relay_invite.rs @@ -116,7 +116,12 @@ pub async fn mint_relay_invite( // community-scoped database write. The trigger remains the final backstop, // but this typed guard keeps a quiescing community from surfacing as an // opaque SQLSTATE/HTTP 500 at the API boundary. - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; crate::deletion::DeletionStore::new(pool.clone()) .guard_transaction(&mut tx, community) .await?; @@ -172,6 +177,11 @@ const RETENTION_SWEEP_BATCH_SIZE: i64 = 1_000; /// expiry index makes old rows drain first without turning cleanup into an /// unbounded transaction. pub async fn reap_expired_relay_invites(pool: &PgPool, cutoff: DateTime) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let result = sqlx::query( "DELETE FROM relay_invites \ WHERE (community_id, id) IN (\ @@ -184,7 +194,7 @@ pub async fn reap_expired_relay_invites(pool: &PgPool, cutoff: DateTime) -> ) .bind(cutoff) .bind(RETENTION_SWEEP_BATCH_SIZE) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) @@ -216,7 +226,12 @@ pub async fn claim_relay_invite( claimer_pubkey: &str, policy_version: Option<&str>, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 2. SELECT FOR UPDATE — lock the invite row for the duration of this txn. let row = sqlx::query( diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index b98b821ff..ad63dd1f5 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -9,11 +9,13 @@ use crate::error::DbError; use crate::Db; use buzz_core::StoredEvent; +use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; use sqlx::{PgPool, Row as _}; use uuid::Uuid; use crate::error::Result; +use crate::observability; use crate::CommunityId; /// A single relay member record. @@ -33,7 +35,8 @@ pub struct RelayMember { /// Returns `true` if `pubkey` (64-char hex) is a member of `community`. pub async fn is_relay_member(pool: &PgPool, community: CommunityId, pubkey: &str) -> Result { - let mut conn = pool.acquire().await?; + let mut conn = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; is_relay_member_on(&mut conn, community, pubkey).await } @@ -58,12 +61,14 @@ pub(crate) async fn is_relay_member_on( /// (`bootstrap_owner`) and operator provisioning still populate it — this is /// how the workspace-profile gate detects whether a steward exists. pub async fn has_admin_or_owner(pool: &PgPool, community: CommunityId) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let row = sqlx::query( "SELECT 1 FROM relay_members \ WHERE community_id = $1 AND role IN ('admin', 'owner') LIMIT 1", ) .bind(community.as_uuid()) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.is_some()) } @@ -74,13 +79,15 @@ pub async fn get_relay_member( community: CommunityId, pubkey: &str, ) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let row = sqlx::query( "SELECT pubkey, role, added_by, created_at, updated_at \ FROM relay_members WHERE community_id = $1 AND pubkey = $2", ) .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|r| -> std::result::Result { @@ -109,6 +116,8 @@ pub async fn list_relay_owners( community: CommunityId, limit: i64, ) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let owners = sqlx::query_scalar( "SELECT pubkey FROM relay_members \ WHERE community_id = $1 AND role = 'owner' \ @@ -116,19 +125,33 @@ pub async fn list_relay_owners( ) .bind(community.as_uuid()) .bind(limit) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(owners) } /// Returns all relay members of `community` ordered by `created_at` ascending. pub async fn list_relay_members(pool: &PgPool, community: CommunityId) -> Result> { + list_relay_members_with_operation( + pool, + community, + observability::WriterOperation::Authorization, + ) + .await +} + +async fn list_relay_members_with_operation( + pool: &PgPool, + community: CommunityId, + operation: observability::WriterOperation, +) -> Result> { + let mut connection = observability::acquire_writer(pool, operation).await?; let rows = sqlx::query( "SELECT pubkey, role, added_by, created_at, updated_at \ FROM relay_members WHERE community_id = $1 ORDER BY created_at ASC", ) .bind(community.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() @@ -157,6 +180,8 @@ pub async fn add_relay_member( role: &str, added_by: Option<&str>, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "INSERT INTO relay_members (community_id, pubkey, role, added_by) \ VALUES ($1, $2, $3, $4) ON CONFLICT (community_id, pubkey) DO NOTHING", @@ -165,7 +190,7 @@ pub async fn add_relay_member( .bind(pubkey) .bind(role) .bind(added_by) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } @@ -182,7 +207,9 @@ pub async fn claim_relay_membership( role: &str, policy_version: Option<&str>, ) -> Result { - let mut tx = pool.begin().await?; + let connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let inserted = sqlx::query( "INSERT INTO relay_members (community_id, pubkey, role, added_by) \ VALUES ($1, $2, $3, 'invite') \ @@ -219,6 +246,8 @@ pub async fn has_join_policy_acceptance( pubkey: &str, policy_version: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let row = sqlx::query( "SELECT 1 FROM join_policy_acceptances \ WHERE community_id = $1 AND pubkey = $2 AND policy_version = $3", @@ -226,7 +255,7 @@ pub async fn has_join_policy_acceptance( .bind(community.as_uuid()) .bind(pubkey) .bind(policy_version) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.is_some()) } @@ -254,13 +283,15 @@ pub async fn remove_relay_member( community: CommunityId, pubkey: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "DELETE FROM relay_members \ WHERE community_id = $1 AND pubkey = $2 AND role <> 'owner'", ) .bind(community.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() > 0 { @@ -272,7 +303,7 @@ pub async fn remove_relay_member( let exists = sqlx::query("SELECT 1 FROM relay_members WHERE community_id = $1 AND pubkey = $2") .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; if exists.is_some() { @@ -301,13 +332,15 @@ pub async fn remove_relay_member_if_role( pubkey: &str, expected_role: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "DELETE FROM relay_members WHERE community_id = $1 AND pubkey = $2 AND role = $3", ) .bind(community.as_uuid()) .bind(pubkey) .bind(expected_role) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() > 0 { @@ -319,7 +352,7 @@ pub async fn remove_relay_member_if_role( let row = sqlx::query("SELECT role FROM relay_members WHERE community_id = $1 AND pubkey = $2") .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -346,6 +379,8 @@ pub async fn update_relay_member_role( pubkey: &str, new_role: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "UPDATE relay_members SET role = $1, updated_at = now() \ WHERE community_id = $2 AND pubkey = $3 AND role <> 'owner'", @@ -353,7 +388,7 @@ pub async fn update_relay_member_role( .bind(new_role) .bind(community.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } @@ -377,9 +412,25 @@ pub async fn bootstrap_owner( pool: &PgPool, community: CommunityId, owner_pubkey: &str, +) -> Result<()> { + bootstrap_owner_with_operation( + pool, + community, + owner_pubkey, + observability::WriterOperation::Bootstrap, + ) + .await +} + +async fn bootstrap_owner_with_operation( + pool: &PgPool, + community: CommunityId, + owner_pubkey: &str, + operation: observability::WriterOperation, ) -> Result<()> { let pubkey = owner_pubkey.to_ascii_lowercase(); - let mut tx = pool.begin().await?; + let connection = observability::acquire_writer(pool, operation).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 1. Upsert the configured owner for this community. sqlx::query( @@ -498,14 +549,19 @@ pub async fn transfer_ownership( ) -> Result { let pubkey = new_owner_pubkey.to_ascii_lowercase(); let expected_owner = expected_owner_pubkey.to_ascii_lowercase(); - let mut tx = pool.begin().await?; + let connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 1. Serialize on the transferee so concurrent transfers to the same // recipient cannot both pass the ownership count check. - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(owner_count_advisory_lock_key(&pubkey)) - .execute(&mut *tx) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::Membership, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(owner_count_advisory_lock_key(&pubkey)) + .execute(&mut *tx), + ) + .await?; // 2. Lock the current owner row FOR UPDATE and verify the expected owner. // FOR UPDATE prevents the stale-owner race: a concurrent transfer that @@ -596,12 +652,14 @@ pub async fn transfer_ownership( /// The empty-table guard prevents re-adding members that were intentionally /// removed by an admin after the initial backfill. pub async fn backfill_from_allowlist(pool: &PgPool, community: CommunityId) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Bootstrap).await?; // Check if pubkey_allowlist table exists. let exists: bool = sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM information_schema.tables \ WHERE table_schema = 'public' AND table_name = 'pubkey_allowlist')", ) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; if !exists { @@ -614,7 +672,7 @@ pub async fn backfill_from_allowlist(pool: &PgPool, community: CommunityId) -> R let has_members: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM relay_members WHERE community_id = $1)") .bind(community.as_uuid()) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; if has_members { @@ -629,7 +687,7 @@ pub async fn backfill_from_allowlist(pool: &PgPool, community: CommunityId) -> R ON CONFLICT (community_id, pubkey) DO NOTHING", ) .bind(community.as_uuid()) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) @@ -757,6 +815,17 @@ impl Db { crate::relay_members::bootstrap_owner(&self.pool, community, owner_pubkey).await } + /// Ensure an owner during operator-driven community provisioning. + pub async fn provision_owner(&self, community: CommunityId, owner_pubkey: &str) -> Result<()> { + bootstrap_owner_with_operation( + &self.pool, + community, + owner_pubkey, + observability::WriterOperation::Authorization, + ) + .await + } + /// Atomically transfers ownership of `community` to `new_owner_pubkey`, /// demoting the previous owner(s) to `member`. Verifies /// `expected_owner_pubkey` matches the current owner inside the same @@ -790,23 +859,80 @@ impl Db { /// Snapshot and canonical rows are compared directly rather than by /// timestamp: relay membership events use whole-second Nostr timestamps, /// and multiple mutations within one second must still be repaired. + #[deprecated( + note = "use nip43_membership_snapshot_needs_reconciliation_for_bootstrap or nip43_membership_snapshot_needs_reconciliation_for_maintenance" + )] pub async fn nip43_membership_snapshot_needs_reconciliation( &self, community_id: CommunityId, relay_pubkey: &nostr::PublicKey, ) -> Result { - let snapshot = self - .query_events(&crate::event::EventQuery { + self.nip43_membership_snapshot_needs_reconciliation_with_operation( + community_id, + relay_pubkey, + observability::WriterOperation::Maintenance, + ) + .await + } + + /// Startup-attributed variant of the NIP-43 snapshot comparison. + #[datastore_span( + name = "nip43_membership_snapshot_needs_reconciliation_for_bootstrap", + system = "postgresql" + )] + pub async fn nip43_membership_snapshot_needs_reconciliation_for_bootstrap( + &self, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + ) -> Result { + self.nip43_membership_snapshot_needs_reconciliation_with_operation( + community_id, + relay_pubkey, + observability::WriterOperation::Bootstrap, + ) + .await + } + + /// Periodic maintenance variant of the NIP-43 snapshot comparison. + #[datastore_span( + name = "nip43_membership_snapshot_needs_reconciliation_for_maintenance", + system = "postgresql" + )] + pub async fn nip43_membership_snapshot_needs_reconciliation_for_maintenance( + &self, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + ) -> Result { + self.nip43_membership_snapshot_needs_reconciliation_with_operation( + community_id, + relay_pubkey, + observability::WriterOperation::Maintenance, + ) + .await + } + + async fn nip43_membership_snapshot_needs_reconciliation_with_operation( + &self, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + operation: observability::WriterOperation, + ) -> Result { + let snapshot = crate::event::query_events_with_operation( + &self.pool, + &crate::event::EventQuery { kinds: Some(vec![buzz_core::kind::KIND_NIP43_MEMBERSHIP_LIST as i32]), pubkey: Some(relay_pubkey.to_bytes().to_vec()), global_only: true, limit: Some(1), ..crate::event::EventQuery::for_community(community_id) - }) - .await? - .into_iter() - .next(); - let members = self.list_relay_members(community_id).await?; + }, + operation, + ) + .await? + .into_iter() + .next(); + let members = + list_relay_members_with_operation(&self.pool, community_id, operation).await?; let Some(snapshot) = snapshot else { return Ok(true); @@ -857,16 +983,25 @@ impl Db { None, ); - let mut tx = self.pool.begin().await?; + let (mut tx, transaction_timer) = crate::observability::begin_transaction( + &self.pool, + crate::observability::TransactionOperation::PublishNip43MembershipLocked, + ) + .await?; + let (event, received_at, was_inserted, member_count) = transaction_timer + .observe(async { // Acquire the per-community snapshot lock BEFORE reading members. // This serializes the entire read-build-write cycle: a concurrent // publication will block here until our transaction commits, then // read the updated membership state. - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(lock_key) - .execute(&mut *tx) - .await?; + crate::observability::observe_advisory_lock( + crate::observability::LockType::Membership, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(lock_key) + .execute(&mut *tx), + ) + .await?; // Read current members inside the locked transaction. let rows = sqlx::query( @@ -941,24 +1076,24 @@ impl Db { .await?; let was_inserted = insert_result.rows_affected() > 0; - if !was_inserted { + if was_inserted { + tx.commit().await?; + } else { tx.rollback().await?; - return Ok(( - StoredEvent::with_received_at(event, received_at, None, false), - false, - member_count, - )); } + Ok::<_, DbError>((event, received_at, was_inserted, member_count)) + }) + .await?; - tx.commit().await?; - - if let Err(e) = crate::insert_mentions(&self.pool, community_id, &event, None).await { - tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + if was_inserted { + if let Err(e) = crate::insert_mentions(&self.pool, community_id, &event, None).await { + tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + } } Ok(( - StoredEvent::with_received_at(event, received_at, None, true), - true, + StoredEvent::with_received_at(event, received_at, None, was_inserted), + was_inserted, member_count, )) } @@ -995,7 +1130,7 @@ mod tests { use super::*; use uuid::Uuid; - const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; + const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 -- local test-only credentials async fn setup_pool() -> PgPool { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 5f34a73da..dd76f3bc6 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -7,6 +7,7 @@ use sqlx::{Acquire, Postgres, Transaction}; use uuid::Uuid; use crate::event::ReplaceOutcome; +use crate::observability::{self, LockType, TransactionOperation}; use crate::{Db, DbError, Result}; /// Result category for a parameterized-replaceable event write. @@ -149,10 +150,13 @@ pub(crate) async fn replace_parameterized_event_in_transaction_impl( pubkey_bytes.as_slice(), Some(d_tag.as_bytes()), ); - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(lock_key) - .execute(&mut **tx) - .await?; + observability::observe_advisory_lock( + LockType::Replacement, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(lock_key) + .execute(&mut **tx), + ) + .await?; let d_tag_count = event .tags @@ -422,14 +426,23 @@ impl Db { channel_id.as_ref().map(|id| id.as_bytes().as_slice()), ); - let mut tx = self.pool.begin().await?; + let (mut tx, transaction_timer) = observability::begin_transaction( + &self.pool, + TransactionOperation::ReplaceAddressableEvent, + ) + .await?; + transaction_timer + .observe(async { // Serialize all writers for the same (kind, pubkey, channel_id) tuple. // Advisory lock is transaction-scoped — released on commit/rollback. - sqlx::query("SELECT pg_advisory_xact_lock($1)") - .bind(lock_key) - .execute(&mut *tx) - .await?; + observability::observe_advisory_lock( + LockType::Replacement, + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(lock_key) + .execute(&mut *tx), + ) + .await?; // Check for the newest existing event. ORDER BY + LIMIT 1 is defensive against // historical data where prior bugs may have left multiple live rows. @@ -540,6 +553,8 @@ impl Db { StoredEvent::with_received_at(event.clone(), received_at, channel_id, true), ReplaceOutcome::Inserted, )) + }) + .await } /// Replace a NIP-33 event inside a caller-owned transaction. @@ -578,24 +593,32 @@ impl Db { d_tag: &str, channel_id: Option, ) -> Result<(StoredEvent, bool)> { - let mut tx = self.pool.begin().await?; - let result = self - .replace_parameterized_event_in_transaction( - &mut tx, - community_id, - event, - d_tag, - channel_id, - ParameterizedReplacePrecondition::Unconditional, - ) - .await?; - let was_inserted = result.status == ParameterizedReplaceStatus::Inserted; - if was_inserted { - tx.commit().await?; - } else { - tx.rollback().await?; - } - Ok((result.event, was_inserted)) + let (mut tx, transaction_timer) = observability::begin_transaction( + &self.pool, + TransactionOperation::ReplaceParameterizedEvent, + ) + .await?; + transaction_timer + .observe(async { + let result = self + .replace_parameterized_event_in_transaction( + &mut tx, + community_id, + event, + d_tag, + channel_id, + ParameterizedReplacePrecondition::Unconditional, + ) + .await?; + let was_inserted = result.status == ParameterizedReplaceStatus::Inserted; + if was_inserted { + tx.commit().await?; + } else { + tx.rollback().await?; + } + Ok((result.event, was_inserted)) + }) + .await } } @@ -782,7 +805,7 @@ mod tests { ); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin caller transaction"); let result = db @@ -852,7 +875,10 @@ mod tests { .1 ); - let mut tx = db.begin_transaction().await.expect("begin replacement tx"); + let mut tx = db + .begin_event_write_transaction() + .await + .expect("begin replacement tx"); let outcome = db .replace_parameterized_event_in_transaction( &mut tx, @@ -886,7 +912,7 @@ mod tests { assert_eq!(live_id, old.id.as_bytes().to_vec()); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin stale revision tx"); let mismatch = db @@ -920,7 +946,7 @@ mod tests { .sign_with_keys(&keys) .expect("sign missing project"); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin missing revision tx"); let missing_result = db @@ -998,7 +1024,7 @@ mod tests { .expect("install failure injection"); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin caller transaction"); let error = db @@ -1079,7 +1105,7 @@ mod tests { .expect("soft-delete duplicate row"); let mut seed_tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin seed transaction"); let (_, was_inserted) = @@ -1090,7 +1116,7 @@ mod tests { seed_tx.commit().await.expect("commit older live head"); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin caller transaction"); let result = db diff --git a/crates/buzz-db/src/store/thread.rs b/crates/buzz-db/src/store/thread.rs index 7d73ace61..3d4ae1116 100644 --- a/crates/buzz-db/src/store/thread.rs +++ b/crates/buzz-db/src/store/thread.rs @@ -15,6 +15,23 @@ use chrono::{DateTime, Utc}; use sqlx::{PgPool, Row}; use uuid::Uuid; +async fn acquire_event_write_connection( + pool: &PgPool, +) -> Result> { + Ok(crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?) +} + +async fn begin_event_write_transaction( + pool: &PgPool, +) -> Result> { + let connection = acquire_event_write_connection(pool).await?; + Ok(sqlx::Transaction::begin(connection, None).await?) +} + use buzz_core::CommunityId; use crate::{error::Result, event::row_to_stored_event}; @@ -132,7 +149,7 @@ pub async fn insert_thread_metadata( depth: i32, broadcast: bool, ) -> Result<()> { - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; let result = sqlx::query( r#" @@ -260,6 +277,7 @@ pub async fn increment_reply_count( parent_event_id: &[u8], root_event_id: Option<&[u8]>, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // Always bump the parent's direct reply count and last-reply timestamp. sqlx::query( r#" @@ -271,7 +289,7 @@ pub async fn increment_reply_count( ) .bind(community_id.as_uuid()) .bind(parent_event_id) - .execute(pool) + .execute(&mut *connection) .await?; // Always bump root's descendant_count, regardless of whether root == parent. @@ -285,7 +303,7 @@ pub async fn increment_reply_count( ) .bind(community_id.as_uuid()) .bind(root_id) - .execute(pool) + .execute(&mut *connection) .await?; } @@ -301,6 +319,7 @@ pub async fn decrement_reply_count( parent_event_id: &[u8], root_event_id: Option<&[u8]>, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // Always decrement the parent's direct reply count (floor at 0). sqlx::query( r#" @@ -311,7 +330,7 @@ pub async fn decrement_reply_count( ) .bind(community_id.as_uuid()) .bind(parent_event_id) - .execute(pool) + .execute(&mut *connection) .await?; // Always decrement root's descendant_count, regardless of whether root == parent. @@ -325,7 +344,7 @@ pub async fn decrement_reply_count( ) .bind(community_id.as_uuid()) .bind(root_id) - .execute(pool) + .execute(&mut *connection) .await?; } @@ -359,7 +378,11 @@ pub async fn get_thread_replies( limit: u32, cursor: Option<&[u8]>, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; get_thread_replies_on( &mut conn, community_id, @@ -527,6 +550,11 @@ pub async fn get_thread_summary( community_id: CommunityId, event_id: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query( r#" SELECT reply_count, descendant_count, last_reply_at @@ -537,7 +565,7 @@ pub async fn get_thread_summary( ) .bind(community_id.as_uuid()) .bind(event_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; let row = match row { @@ -570,7 +598,7 @@ pub async fn get_thread_summary( ) .bind(community_id.as_uuid()) .bind(event_id) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; let participants: Vec> = participant_rows @@ -606,7 +634,11 @@ pub async fn get_channel_window( cursor: Option<(DateTime, Vec)>, kind_filter: Option<&[u32]>, ) -> Result { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; get_channel_window_on( &mut conn, community_id, @@ -818,6 +850,11 @@ pub async fn get_thread_metadata_by_event( community_id: CommunityId, event_id: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query( r#" SELECT @@ -837,7 +874,7 @@ pub async fn get_thread_metadata_by_event( ) .bind(community_id.as_uuid()) .bind(event_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; let row = match row { @@ -938,8 +975,13 @@ impl Db { ), None => ("thread_head", RoutePredicate::Bounded), }; - if let RouteDecision::Replica(mut tx, entry, reason) = - self.route_read(path, predicate).await + if let RouteDecision::Replica(mut tx, entry, reason) = self + .route_read( + path, + predicate, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await { match crate::thread::get_thread_replies_on( &mut tx, @@ -1063,6 +1105,7 @@ impl Db { .route_read( path, RoutePredicate::from_channel_cursor(channel_id, &cursor), + crate::observability::ReaderOperation::SubscriptionHistory, ) .await { diff --git a/crates/buzz-db/src/store/usage.rs b/crates/buzz-db/src/store/usage.rs index e14e75e95..a182a6667 100644 --- a/crates/buzz-db/src/store/usage.rs +++ b/crates/buzz-db/src/store/usage.rs @@ -13,7 +13,8 @@ //! to Prometheus labels and calls `metrics::gauge!(...).set(...)`. use crate::error::Result; -use crate::Db; +use crate::{observability, Db}; +use buzz_datastore_tracing::datastore_span; use sqlx::postgres::PgConnection; use sqlx::Connection; use sqlx::PgPool; @@ -21,8 +22,10 @@ use uuid::Uuid; /// Total number of communities registered on this relay. pub async fn community_count(pool: &PgPool) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let row = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM communities") - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; Ok(row) } @@ -42,6 +45,8 @@ pub struct CommunityUserCounts { /// /// Agent discriminator: `agent_owner_pubkey IS NOT NULL`. pub async fn user_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; // Single GROUP BY query; two conditional SUMs avoid two round-trips. let rows = sqlx::query_as::<_, (Uuid, i64, i64)>( r#" @@ -54,7 +59,7 @@ pub async fn user_counts(pool: &PgPool) -> Result> { GROUP BY community_id "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -80,6 +85,8 @@ pub struct CommunityChannelCount { /// Return non-deleted channel counts per community per type. pub async fn channel_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, String, i64)>( r#" SELECT community_id, channel_type::text, COUNT(*) AS count @@ -88,7 +95,7 @@ pub async fn channel_counts(pool: &PgPool) -> Result> GROUP BY community_id, channel_type "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -114,6 +121,8 @@ pub struct CommunityMessageCount { /// Return non-deleted kind=9 event counts per community. pub async fn message_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, i64)>( r#" SELECT community_id, COUNT(*) AS count @@ -122,7 +131,7 @@ pub async fn message_counts(pool: &PgPool) -> Result> GROUP BY community_id "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -147,6 +156,8 @@ pub struct CommunityMemberCount { /// Return relay-member counts per community per role. pub async fn relay_member_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, String, i64)>( r#" SELECT community_id, role::text, COUNT(*) AS count @@ -154,7 +165,7 @@ pub async fn relay_member_counts(pool: &PgPool) -> Result Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, String, i64)>( r#" SELECT community_id, status::text, COUNT(*) AS count @@ -187,7 +200,7 @@ pub async fn workflow_counts(pool: &PgPool) -> Result Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, i64)>( r#" SELECT community_id, COUNT(*) AS count @@ -218,7 +233,7 @@ pub async fn git_repo_counts(pool: &PgPool) -> Result GROUP BY community_id "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -258,6 +273,8 @@ pub async fn active_user_counts( pool: &PgPool, interval_sql: &'static str, ) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; // LEFT JOIN users: pubkeys with no row have u.* = NULL. // Three-way classification: // human — row exists (u.pubkey IS NOT NULL) and agent_owner_pubkey IS NULL @@ -282,7 +299,7 @@ pub async fn active_user_counts( "# ); let rows = sqlx::query_as::<_, (Uuid, i64, i64, i64)>(sqlx::AssertSqlSafe(sql)) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -312,6 +329,8 @@ pub async fn active_channel_counts( pool: &PgPool, interval_sql: &'static str, ) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let sql = format!( r#" SELECT community_id, COUNT(DISTINCT channel_id) AS count @@ -324,7 +343,7 @@ pub async fn active_channel_counts( "# ); let rows = sqlx::query_as::<_, (Uuid, i64)>(sqlx::AssertSqlSafe(sql)) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -348,8 +367,16 @@ pub struct CommunityHost { /// Fetch all community id → host mappings in one query. pub async fn community_hosts(pool: &PgPool) -> Result> { + community_hosts_with_operation(pool, observability::WriterOperation::Maintenance).await +} + +async fn community_hosts_with_operation( + pool: &PgPool, + operation: observability::WriterOperation, +) -> Result> { + let mut connection = observability::acquire_writer(pool, operation).await?; let rows = sqlx::query_as::<_, (Uuid, String)>("SELECT id, host FROM communities") - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows .into_iter() @@ -389,7 +416,11 @@ impl Db { &self, lock_key: i64, ) -> Result> { - let mut connection = self.pool.acquire().await?; + let mut connection = observability::acquire_writer_with_legacy_metrics( + &self.pool, + observability::WriterOperation::Maintenance, + ) + .await?; let acquired = sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)") .bind(lock_key) .fetch_one(&mut *connection) @@ -462,6 +493,12 @@ impl Db { pub async fn usage_community_hosts(&self) -> Result> { crate::usage::community_hosts(&self.pool).await } + + /// Return community host mappings during startup bootstrap work. + #[datastore_span(name = "bootstrap_community_hosts", system = "postgresql")] + pub async fn bootstrap_community_hosts(&self) -> Result> { + community_hosts_with_operation(&self.pool, observability::WriterOperation::Bootstrap).await + } } #[cfg(test)] diff --git a/crates/buzz-db/src/store/user.rs b/crates/buzz-db/src/store/user.rs index 73af6ce0a..b6e03c3ba 100644 --- a/crates/buzz-db/src/store/user.rs +++ b/crates/buzz-db/src/store/user.rs @@ -3,6 +3,7 @@ use crate::error::Result; use crate::Db; use buzz_core::CommunityId; +use buzz_datastore_tracing::datastore_span; use sqlx::PgPool; use sqlx::Row; @@ -41,6 +42,22 @@ pub struct UserSearchProfile { /// The `true` case is the reliable signal for "user was just registered" — used /// by callers to increment `buzz_users_created_total`. pub async fn ensure_user(pool: &PgPool, community_id: CommunityId, pubkey: &[u8]) -> Result { + ensure_user_with_operation( + pool, + community_id, + pubkey, + crate::observability::WriterOperation::EventWrite, + ) + .await +} + +async fn ensure_user_with_operation( + pool: &PgPool, + community_id: CommunityId, + pubkey: &[u8], + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let result = sqlx::query( r#" INSERT INTO users (community_id, pubkey) @@ -50,7 +67,7 @@ pub async fn ensure_user(pool: &PgPool, community_id: CommunityId, pubkey: &[u8] ) .bind(community_id.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -295,6 +312,24 @@ pub async fn set_agent_owner( agent_pubkey: &[u8], owner_pubkey: &[u8], ) -> Result { + set_agent_owner_with_operation( + pool, + community_id, + agent_pubkey, + owner_pubkey, + crate::observability::WriterOperation::EventWrite, + ) + .await +} + +async fn set_agent_owner_with_operation( + pool: &PgPool, + community_id: CommunityId, + agent_pubkey: &[u8], + owner_pubkey: &[u8], + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; // Conditional UPDATE: only set owner if currently NULL. This makes // "first mint wins" atomic — no TOCTOU race between concurrent mints. let result = sqlx::query( @@ -303,7 +338,7 @@ pub async fn set_agent_owner( .bind(owner_pubkey) .bind(community_id.as_uuid()) .bind(agent_pubkey) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() == 0 { @@ -312,7 +347,7 @@ pub async fn set_agent_owner( let exists = sqlx::query(r#"SELECT 1 FROM users WHERE community_id = $1 AND pubkey = $2"#) .bind(community_id.as_uuid()) .bind(agent_pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; if exists.is_none() { return Err(crate::error::DbError::NotFound( @@ -333,12 +368,17 @@ pub async fn get_agent_channel_policy( community_id: CommunityId, pubkey: &[u8], ) -> Result>)>> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#"SELECT channel_add_policy::text AS channel_add_policy, agent_owner_pubkey FROM users WHERE community_id = $1 AND pubkey = $2"#, ) .bind(community_id.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|r| -> Result<(String, Option>)> { @@ -358,13 +398,18 @@ pub async fn is_agent_owner( target_pubkey: &[u8], actor_pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query_scalar::<_, bool>( "SELECT agent_owner_pubkey = $3 FROM users WHERE community_id = $1 AND pubkey = $2 AND agent_owner_pubkey IS NOT NULL", ) .bind(community_id.as_uuid()) .bind(target_pubkey) .bind(actor_pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.unwrap_or(false)) } @@ -409,6 +454,23 @@ impl Db { ensure_user(&self.pool, community_id, pubkey).await } + /// Ensure a principal while materializing an authenticated NIP-OA + /// authorization relationship. + #[datastore_span(name = "ensure_user_for_authorization", system = "postgresql")] + pub async fn ensure_user_for_authorization( + &self, + community_id: CommunityId, + pubkey: &[u8], + ) -> Result { + ensure_user_with_operation( + &self.pool, + community_id, + pubkey, + crate::observability::WriterOperation::Authorization, + ) + .await + } + /// Get a single user record by pubkey. pub async fn get_user( &self, @@ -471,6 +533,25 @@ impl Db { set_agent_owner(&self.pool, community_id, agent_pubkey, owner_pubkey).await } + /// Materialize an authenticated NIP-OA agent-owner relationship under + /// authorization attribution. + #[datastore_span(name = "set_agent_owner_for_authorization", system = "postgresql")] + pub async fn set_agent_owner_for_authorization( + &self, + community_id: CommunityId, + agent_pubkey: &[u8], + owner_pubkey: &[u8], + ) -> Result { + set_agent_owner_with_operation( + &self.pool, + community_id, + agent_pubkey, + owner_pubkey, + crate::observability::WriterOperation::Authorization, + ) + .await + } + /// Get the channel_add_policy and agent_owner_pubkey for a user. pub async fn get_agent_channel_policy( &self, diff --git a/crates/buzz-db/src/test_support.rs b/crates/buzz-db/src/test_support.rs new file mode 100644 index 000000000..7699313d6 --- /dev/null +++ b/crates/buzz-db/src/test_support.rs @@ -0,0 +1,9 @@ +const DEFAULT_DATABASE_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 -- local test-only credentials + +/// Resolve the database URL shared by PostgreSQL-backed unit tests. +pub(crate) fn database_url() -> String { + std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("TEST_DATABASE_URL")) + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| DEFAULT_DATABASE_URL.to_owned()) +} diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs new file mode 100644 index 000000000..7c749e058 --- /dev/null +++ b/crates/buzz-db/tests/observability_source.rs @@ -0,0 +1,590 @@ +#[test] +fn database_metrics_and_slow_logs_exclude_sensitive_or_unbounded_fields() { + let implementation = include_str!("../src/runtime/observability.rs"); + let datastore_macro = include_str!("../../buzz-datastore-tracing/src/lib.rs"); + let instrumentation = format!("{implementation}\n{datastore_macro}"); + + for forbidden in [ + "\"community\" =>", + "\"event_id\" =>", + "\"event_kind\" =>", + "\"kind\" =>", + "\"sql\" =>", + "\"query\" =>", + "\"query_id\" =>", + "\"d_tag\" =>", + "\"coordinate\" =>", + "community =", + "event_id =", + "event_kind =", + "sql =", + "query_id =", + "d_tag =", + "coordinate =", + ] { + assert!( + !instrumentation.contains(forbidden), + "database instrumentation must not expose {forbidden}" + ); + } + + assert!(datastore_macro.contains("name: LitStr")); + assert!(datastore_macro.contains("\"operation\" => #name")); + assert!(datastore_macro.contains("elapsed_ms =")); + assert!( + datastore_macro.contains("parent: None"), + "slow warnings must not inherit dynamic datastore span fields" + ); + // The runtime tracing-layer assertion covers field names because a source + // search would also match ordinary local variables such as `record_error`. +} + +#[test] +fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() { + let observability = include_str!("../src/runtime/observability.rs"); + assert!(observability.contains("enum PoolOperation")); + assert!(observability.contains("pub(crate) enum WriterOperation")); + assert!(observability.contains("pub(crate) enum ReaderOperation")); + assert!(observability.contains("Self::WriterAuthentication")); + assert!(observability.contains("Self::ReaderSubscriptionHistory")); + assert!(observability.contains("pub(crate) async fn acquire_writer(")); + assert!(observability.contains("pub(super) async fn acquire_reader_with_legacy_metrics(")); + assert!(observability.contains("static POOL_WAITERS: [Mutex")); + assert!(!observability.contains("AtomicU64")); + assert!(!observability.contains("DbOperation::Other")); + assert!(!observability.contains("\"other\"")); + assert!(!observability.contains("buzz_db_pool_acquire_timeouts_total")); + assert!(!observability.contains("\"result\" =>")); + let legacy_transaction = observability + .split_once("pub(crate) async fn begin_transaction(") + .expect("observability must expose attributed transaction acquisition") + .1 + .split_once("pub(crate) async fn observe_advisory_lock") + .expect("transaction acquisition must precede advisory-lock observation") + .0; + assert!(legacy_transaction.contains("acquire_writer_with_legacy_metrics(")); + + let runtime = include_str!("../src/runtime/mod.rs"); + assert!(runtime.contains("observability::acquire_writer_until(")); + assert!(runtime.contains("WriterOperation::Readiness")); + assert!(runtime.contains("WriterOperation::EventWrite")); + assert!(runtime.contains("ReaderOperation::Bootstrap")); + assert!(runtime.contains("pub async fn begin_event_write_transaction")); + let reader_boot = runtime + .split_once("async fn read_pool_boot_ping_once(") + .expect("runtime must expose the reader boot probe") + .1 + .split_once("#[cfg(test)]") + .expect("reader boot probe must precede its test seam") + .0; + assert!(reader_boot.contains("acquire_reader_with_legacy_metrics(")); + let routed_reader = runtime + .split_once("async fn proved_reader(") + .expect("runtime must expose the routed-reader checkout") + .1 + .split_once("async fn reader_aurora_capability_on(") + .expect("routed-reader checkout must precede capability probing") + .0; + assert!(routed_reader.contains("acquire_reader_with_legacy_metrics(read_pool, operation)")); + let event_write_transaction = runtime + .split_once("pub async fn begin_event_write_transaction(") + .expect("runtime must expose the legacy event-write transaction seam") + .1 + .split_once("pub async fn insert_event_with_serving_write_guard(") + .expect("legacy event-write transaction must precede guarded writes") + .0; + assert!(event_write_transaction.contains("acquire_writer_with_legacy_metrics(")); + + let migration = include_str!("../src/runtime/migration.rs"); + let migration_lock = migration + .split_once("pub(crate) async fn with_exclusive_schema_destruction_lock") + .expect("migration must expose the schema-safety acquisition seam") + .1 + .split_once("async fn reject_legacy_nip_rs_cardinality_ambiguity") + .expect("schema-safety acquisition must precede migration validation") + .0; + assert!(migration_lock.contains("acquire_writer_with_legacy_metrics(")); + + let allowlist = include_str!("../src/store/allowlist.rs"); + assert!(allowlist.contains("WriterOperation::Authentication")); + assert!(allowlist.contains("WriterOperation::Authorization")); + assert!(!allowlist.contains("fetch_one(&self.pool)")); + + let event = include_str!("../src/store/event.rs"); + assert!(event.contains("query_events_with_operation")); + assert!(event.contains("WriterOperation::Authorization")); + assert!(event.contains("WriterOperation::SubscriptionHistory")); + assert!(event.contains("ReaderOperation::SubscriptionHistory")); + let backfill_d_tags = event + .split_once("pub async fn backfill_d_tags") + .expect("event store must expose the startup d-tag backfill") + .1 + .split_once("/// Soft-delete NIP-29 discovery events") + .expect("d-tag backfill must precede discovery deletion") + .0; + assert!(backfill_d_tags.contains("WriterOperation::Bootstrap")); + assert!(backfill_d_tags.contains("execute(&mut *connection)")); + let soft_delete_discovery = event + .split_once("pub async fn soft_delete_discovery_events") + .expect("event store must expose discovery-event deletion") + .1 + .split_once("\n}\n\n#[cfg(test)]") + .expect("discovery deletion must end the production Db implementation") + .0; + assert!(soft_delete_discovery.contains("WriterOperation::EventWrite")); + assert!(soft_delete_discovery.contains("execute(&mut *connection)")); + + let side_effects = include_str!("../../buzz-relay/src/handlers/side_effects.rs"); + assert!(side_effects.contains("query_events_for_event_write")); + assert!(side_effects.contains("query_events_for_bootstrap")); + assert!(side_effects.contains(".list_channels_for_bootstrap(")); + + let deletion = include_str!("../src/store/deletion.rs"); + let public_serving_catalog = deletion + .split_once("pub async fn validate_serving_catalog(&self)") + .expect("deletion store must preserve its public serving-catalog API") + .1 + .split_once("async fn validate_serving_catalog_on") + .expect("public serving-catalog validation must delegate to its connection helper") + .0; + assert!(public_serving_catalog.contains("WriterOperation::Bootstrap")); + assert!(public_serving_catalog.contains("observability::acquire_writer(")); + assert!(public_serving_catalog.contains("validate_serving_catalog_on")); + assert!(!public_serving_catalog.contains("self.pool.acquire()")); + + let thread = include_str!("../src/store/thread.rs"); + let thread_metadata = thread + .split_once("pub async fn get_thread_metadata_by_event(") + .expect("thread store must expose metadata lookup") + .1 + .split_once("\nimpl Db {") + .expect("metadata lookup must precede the Db wrapper section") + .0; + assert!(thread_metadata.contains("WriterOperation::EventWrite")); + assert!(thread_metadata.contains("fetch_optional(&mut *connection)")); + assert!(!thread_metadata.contains("fetch_optional(pool)")); + + let channel = include_str!("../src/store/channel.rs"); + assert!(channel.contains("async fn begin_event_write_transaction(")); + assert!(channel.contains("async fn acquire_event_write_connection(")); + for (start, end, expected) in [ + ( + "pub async fn create_channel(\n", + "/// Creates a channel with a client-supplied UUID", + "begin_event_write_transaction(pool)", + ), + ( + "pub async fn create_channel_with_id(\n", + "/// Fetches a channel record by `(community_id, id)`", + "begin_event_write_transaction(pool)", + ), + ( + "pub async fn update_channel(\n", + "/// Sets the topic for a channel", + "begin_event_write_transaction(pool)", + ), + ( + "pub async fn set_topic(\n", + "/// Sets the purpose for a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn set_purpose(\n", + "/// Archives a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn archive_channel(\n", + "/// Unarchives a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn unarchive_channel(\n", + "/// Soft-delete a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn soft_delete_channel(\n", + "/// Archive ephemeral channels", + "acquire_event_write_connection(pool)", + ), + ] { + let function = channel + .split_once(start) + .unwrap_or_else(|| panic!("missing channel seam {start}")) + .1 + .split_once(end) + .unwrap_or_else(|| panic!("channel seam {start} must precede {end}")) + .0; + assert!( + function.contains(expected), + "channel seam {start} must use {expected}" + ); + assert!(!function.contains("pool.begin().await")); + assert!(!function.contains(".execute(pool)")); + assert!(!function.contains(".fetch_optional(pool)")); + } + let get_channel = channel + .split_once("async fn get_channel_with_operation(") + .expect("channel store must route shared lookups through caller-owned intent") + .1 + .split_once("/// Lists channels in a community") + .expect("channel lookup helper must precede channel listing") + .0; + assert!(get_channel.contains("acquire_writer(pool, operation)")); + assert!(get_channel.contains("fetch_optional(&mut *connection)")); + assert!(!get_channel.contains("fetch_optional(pool)")); + assert!(channel.contains("pub async fn get_channel_for_event_write(")); + let list_channels = channel + .split_once("async fn list_channels_with_operation(") + .expect("channel listing must accept caller-owned intent") + .1 + .split_once("/// A channel archived by the ephemeral-channel reaper") + .expect("channel listing must precede ephemeral-channel types") + .0; + assert!(list_channels.contains("acquire_writer(pool, operation)")); + assert!(list_channels.contains("fetch_all(&mut *connection)")); + assert!(!list_channels.contains("fetch_all(pool)")); + assert!(channel.contains("pub async fn list_channels_for_bootstrap(")); + + let channel_members = include_str!("../src/store/channel_members.rs"); + assert!(channel_members.contains("async fn get_members_with_operation(")); + assert!(channel_members.contains("pub async fn get_members_for_event_write(")); + assert!(channel_members.contains("async fn get_users_bulk_with_operation(")); + assert!(channel_members.contains("pub async fn get_users_bulk_for_event_write(")); + + let huddle_link = event + .split_once("async fn huddle_started_link_exists_with_operation(") + .expect("huddle link lookup must accept caller-owned intent") + .1 + .split_once("/// Insert a Nostr event") + .expect("huddle link lookup must precede event insertion") + .0; + assert!(huddle_link.contains("acquire_writer(pool, operation)")); + assert!(event.contains("pub async fn huddle_started_link_exists_for_event_write(")); + // Upstream's ingest validates Huddle lifecycle events and is the second + // caller of the event-write variant. Colony's ingest has no Huddle + // lifecycle validation, so the audio handler is the only caller here. + let audio = include_str!("../../buzz-relay/src/audio/handler.rs"); + assert!(audio.contains(".huddle_started_link_exists(")); + + let workflow_sink = include_str!("../../buzz-relay/src/workflow_sink.rs"); + assert!(workflow_sink.contains(".get_members_for_event_write(")); + assert!(workflow_sink.contains(".get_users_bulk_for_event_write(")); + + for write_caller in [ + include_str!("../../buzz-relay/src/handlers/side_effects.rs"), + include_str!("../../buzz-relay/src/handlers/ingest.rs"), + include_str!("../../buzz-relay/src/handlers/command_executor.rs"), + workflow_sink, + ] { + assert!(!write_caller.contains(".get_channel(")); + assert!(write_caller.contains(".get_channel_for_event_write(")); + } + + let user = include_str!("../src/store/user.rs"); + let agent_channel_policy = user + .split_once("pub async fn get_agent_channel_policy(") + .expect("user store must expose get_agent_channel_policy") + .1 + .split_once("/// Check whether `actor_pubkey`") + .expect("agent policy lookup must precede owner lookup") + .0; + assert!(agent_channel_policy.contains("WriterOperation::Authorization")); + assert!(agent_channel_policy.contains("fetch_optional(&mut *connection)")); + assert!(!agent_channel_policy.contains("fetch_optional(pool)")); + let is_agent_owner = user + .split_once("pub async fn is_agent_owner(") + .expect("user store must expose is_agent_owner") + .1 + .split_once("/// Set the channel_add_policy") + .expect("is_agent_owner must precede set_agent_channel_policy") + .0; + assert!(is_agent_owner.contains("WriterOperation::Authorization")); + assert!(is_agent_owner.contains("acquire_writer(")); + assert!(is_agent_owner.contains("fetch_optional(&mut *connection)")); + assert!(!is_agent_owner.contains("fetch_optional(pool)")); + + let moderation = include_str!("../src/store/moderation.rs"); + let restriction_state = moderation + .split_once("pub async fn restriction_state(") + .expect("moderation store must expose restriction_state") + .1 + .split_once("/// Fetch the full ban/timeout row") + .expect("restriction state must precede full ban reads") + .0; + assert!(restriction_state.contains("WriterOperation::Authorization")); + assert!(restriction_state.contains("fetch_optional(&mut *connection)")); + assert!(!restriction_state.contains("fetch_optional(pool)")); + + let community_store = include_str!("../src/store/community.rs"); + let ensure_community = community_store + .split_once("pub async fn ensure_configured_community(") + .expect("community store must expose ensure_configured_community") + .1 + .split_once("/// Atomically creates a community") + .expect("configured-community helpers must precede community creation") + .0; + assert!(ensure_community.contains("WriterOperation::Authorization")); + assert!(ensure_community.contains("WriterOperation::Bootstrap")); + assert!(ensure_community.contains("ensure_configured_community_with_operation")); + assert!(ensure_community.contains("acquire_writer(&self.pool, operation)")); + // Colony's upsert is `ON CONFLICT DO UPDATE`, which always returns a row, + // so it reads with `fetch_one`; upstream's tombstone-filtered variant uses + // `fetch_optional`. Either way the read must run on the attributed + // connection rather than the bare pool. + assert!( + ensure_community.contains("fetch_one(&mut *connection)") + || ensure_community.contains("fetch_optional(&mut *connection)") + ); + let management_lookup = community_store + .split_once("pub async fn lookup_community_by_host_for_management(") + .expect("community store must expose management host lookup") + .1 + .split_once("/// Lists communities where") + .expect("management lookup must precede owner listing") + .0; + assert!(management_lookup.contains("WriterOperation::Authorization")); + assert!(management_lookup.contains("fetch_optional(&mut *connection)")); + assert!(!management_lookup.contains("fetch_optional(&self.pool)")); + let community_production = community_store + .split("\n#[cfg(test)]") + .next() + .expect("community production source"); + for required in [ + "WriterOperation::TenantResolution", + "WriterOperation::Authorization", + "WriterOperation::SubscriptionHistory", + "WriterOperation::EventWrite", + ] { + assert!( + community_production.contains(required), + "community P0 paths must include {required} attribution" + ); + } + assert!(!community_production.contains("self.pool.begin().await")); + assert!(!community_production.contains(".fetch_one(&self.pool)")); + assert!(!community_production.contains(".fetch_all(&self.pool)")); + assert!(!community_production.contains(".execute(&self.pool)")); + assert_eq!( + community_production + .matches(".fetch_optional(&self.pool)") + .count(), + 1, + "only the out-of-scope NIP-11 metadata read may retain a raw pool checkout" + ); + + let thread_summary = thread + .split_once("pub async fn get_thread_summary(") + .expect("thread store must expose get_thread_summary") + .1 + .split_once("/// Fetch one channel window") + .expect("thread summary must precede channel-window reads") + .0; + assert!(thread_summary.contains("WriterOperation::EventWrite")); + assert!(thread_summary.contains("fetch_optional(&mut *connection)")); + assert!(thread_summary.contains("fetch_all(&mut *connection)")); + assert!(!thread_summary.contains("fetch_optional(pool)")); + assert!(!thread_summary.contains("fetch_all(pool)")); + + let archived_identities = include_str!("../src/store/archived_identities.rs"); + let archived_identity_production = archived_identities + .split("\n#[cfg(test)]") + .next() + .expect("archived identity production source"); + assert_eq!( + archived_identity_production + .matches("WriterOperation::EventWrite") + .count(), + 4, + "all four archived identity operations must be attributed to event writes" + ); + assert!(!archived_identity_production.contains("fetch_optional(pool)")); + assert!(!archived_identity_production.contains("fetch_all(pool)")); + assert!(!archived_identity_production.contains("execute(pool)")); + + let relay_main = include_str!("../../buzz-relay/src/main.rs"); + assert!(relay_main.contains("pool_state.db.refresh_pool_waiter_metrics();")); + assert!(relay_main.contains(".ensure_configured_community_for_bootstrap(")); + + let runtime = include_str!("../src/runtime/mod.rs"); + assert!(runtime.contains("observability::refresh_pool_waiters(self.read_pool.is_some())")); + assert!(runtime.contains("self.verify_replica_fence_at_boot().await?")); + let fence_boot = runtime + .split_once("pub(crate) async fn verify_replica_fence_at_boot") + .expect("runtime must expose attributed boot fence verification") + .1 + .split_once("/// Whether a distinct read-replica pool is configured") + .expect("boot fence verification must precede read-pool plumbing") + .0; + assert!(fence_boot.contains("WriterOperation::Bootstrap")); + + let replica_fence = include_str!("../src/runtime/replica_fence.rs"); + let replica_fence_production = replica_fence + .split("\n#[cfg(test)]") + .next() + .expect("replica-fence production source"); + assert!(replica_fence_production.contains("WriterOperation::Bootstrap")); + assert!(replica_fence_production.contains("WriterOperation::Maintenance")); + assert!(!replica_fence_production.contains("pool.begin().await")); + assert!(!replica_fence_production.contains("writer.acquire().await")); + assert!(!replica_fence_production.contains("fetch_optional(writer)")); + + let usage = include_str!("../src/store/usage.rs"); + let usage_production = usage + .split("\n#[cfg(test)]") + .next() + .expect("usage production source"); + let usage_leader_lock = usage_production + .split_once("pub async fn try_lock_usage_metrics(") + .expect("usage store must expose the legacy leader-lock acquisition") + .1 + .split_once("pub async fn usage_community_count(") + .expect("usage leader lock must precede counter reads") + .0; + assert!(usage_leader_lock.contains("acquire_writer_with_legacy_metrics(")); + assert!( + usage_production + .matches("WriterOperation::Maintenance") + .count() + >= 11, + "every periodic usage checkout must be maintenance-attributed" + ); + for bypass in [ + ".fetch_one(pool)", + ".fetch_all(pool)", + ".fetch_optional(pool)", + ".execute(pool)", + ] { + assert!( + !usage_production.contains(bypass), + "usage production path bypasses operation attribution with {bypass}" + ); + } + + let channel_reaper = channel + .split_once("pub async fn reap_expired_ephemeral_channels(pool:") + .expect("channel store must expose ephemeral reaper") + .1 + .split_once("\nimpl Db {") + .expect("ephemeral reaper must precede Db wrappers") + .0; + assert!(channel_reaper.contains("WriterOperation::Maintenance")); + assert!(channel_reaper.contains("fetch_all(&mut *connection)")); + + let deletion = include_str!("../src/store/deletion.rs"); + let lease_reaper = deletion + .split_once("pub async fn reap_expired_serving_write_leases") + .expect("deletion store must expose serving-lease reaper") + .1 + .split_once("/// Return serving-lease counts") + .expect("serving-lease reaper must precede stats") + .0; + assert!(lease_reaper.contains("WriterOperation::Maintenance")); + assert!(lease_reaper.contains("execute(&mut *connection)")); + let lease_stats = deletion + .split_once("pub async fn serving_lease_stats") + .expect("deletion store must expose serving-lease stats") + .1 + .split_once("/// Whether a community remains active") + .expect("serving-lease stats must precede serving-state reads") + .0; + assert!(lease_stats.contains("WriterOperation::Maintenance")); + assert!(lease_stats.contains("fetch_one(&mut *connection)")); + for (start, end) in [ + ( + "pub async fn acquire_serving_write_lease", + "/// Renew an already-admitted external side-effect lease", + ), + ( + "pub async fn renew_serving_write_lease", + "/// Release a serving side-effect lease", + ), + ( + "pub async fn release_serving_write_lease", + "/// Check that an external side-effect lease remains current", + ), + ( + "pub async fn verify_serving_write_lease", + "/// Delete expired serving leases", + ), + ( + "pub async fn is_serving_active", + "async fn advance_with_checkpoint", + ), + ] { + let function = deletion + .split_once(start) + .unwrap_or_else(|| panic!("missing serving-write seam {start}")) + .1 + .split_once(end) + .unwrap_or_else(|| panic!("serving-write seam {start} must precede {end}")) + .0; + assert!( + function.contains("WriterOperation::EventWrite"), + "serving-write seam {start} must be event-write attributed" + ); + assert!(!function.contains("self.pool.begin().await")); + assert!(!function.contains(".execute(&self.pool)")); + assert!(!function.contains(".fetch_one(&self.pool)")); + } + + let ensure_authorization = user + .split_once("pub async fn ensure_user_for_authorization(") + .expect("user store must expose NIP-OA authorization ensure") + .1 + .split_once("/// Get a single user record") + .expect("authorization ensure must precede generic user reads") + .0; + assert!(ensure_authorization.contains("WriterOperation::Authorization")); + let set_owner_authorization = user + .split_once("pub async fn set_agent_owner_for_authorization(") + .expect("user store must expose NIP-OA authorization owner write") + .1 + .split_once("/// Get the channel_add_policy") + .expect("authorization owner write must precede policy reads") + .0; + assert!(set_owner_authorization.contains("WriterOperation::Authorization")); + let relay_api = include_str!("../../buzz-relay/src/api/mod.rs"); + assert!(relay_api.contains(".ensure_user_for_authorization(")); + assert!(relay_api.contains(".set_agent_owner_for_authorization(")); + + for (domain, source) in [ + ( + "channel_members", + include_str!("../src/store/channel_members.rs"), + ), + ("archived_identities", archived_identities), + ("event", event), + ("git_repo", include_str!("../src/store/git_repo.rs")), + ("push", include_str!("../src/store/push.rs")), + ("replica_fence", replica_fence), + ("reaction", include_str!("../src/store/reaction.rs")), + ("relay_invite", include_str!("../src/store/relay_invite.rs")), + ( + "relay_members", + include_str!("../src/store/relay_members.rs"), + ), + ("thread", thread), + ("usage", usage), + ] { + let production = source.split("\n#[cfg(test)]").next().unwrap_or(source); + for bypass in [ + "pool.begin().await", + "self.pool.begin().await", + ".fetch_one(pool)", + ".fetch_one(&self.pool)", + ".fetch_all(pool)", + ".fetch_all(&self.pool)", + ".fetch_optional(pool)", + ".fetch_optional(&self.pool)", + ".execute(pool)", + ".execute(&self.pool)", + ] { + assert!( + !production.contains(bypass), + "{domain} production path bypasses operation attribution with {bypass}" + ); + } + } +} diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 4e27b85fe..c0d2a89e8 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -526,11 +526,14 @@ fn resolve_submit_host(host: Option<&str>, relay_url: Option<&str>) -> Result Result { let database_url = required_env("DATABASE_URL")?; - let db = Db::new(&DbConfig { - database_url, - max_connections: env_parse("BUZZ_DB_POOL_SIZE", 20), - ..DbConfig::default() - }) + let db = Db::new( + &DbConfig { + database_url, + max_connections: env_parse("BUZZ_DB_POOL_SIZE", 20), + ..DbConfig::default() + } + .with_session_timeouts_from_env(), + ) .await?; Ok(store(&db)) } diff --git a/crates/buzz-media/src/error.rs b/crates/buzz-media/src/error.rs index 14ce4afe1..5abbea6f5 100644 --- a/crates/buzz-media/src/error.rs +++ b/crates/buzz-media/src/error.rs @@ -72,8 +72,10 @@ pub enum MediaError { /// Video duration exceeds the 600-second limit. #[error("video too long: duration exceeds 600 seconds")] DurationTooLong, - /// Video resolution exceeds 3840×2160. - #[error("video resolution too high: maximum is 3840x2160")] + /// Video resolution exceeds the 2160 short-edge / 3840 long-edge envelope. + #[error( + "video resolution too high: maximum is 2160 on the short edge and 3840 on the long edge" + )] ResolutionTooHigh, /// MP4 moov atom appears after mdat — not fast-start. #[error("moov atom not at front of file (not fast-start)")] diff --git a/crates/buzz-media/src/validation.rs b/crates/buzz-media/src/validation.rs index 5997e0874..160d605ff 100644 --- a/crates/buzz-media/src/validation.rs +++ b/crates/buzz-media/src/validation.rs @@ -307,7 +307,7 @@ pub fn validate_content(bytes: &[u8], config: &MediaConfig) -> Result Result 3840 || height > 2160 { + let short_edge = width.min(height); + let long_edge = width.max(height); + if short_edge > 2160 || long_edge > 3840 { return Err(MediaError::ResolutionTooHigh); } @@ -2620,6 +2624,43 @@ mod tests { ); } + #[test] + fn test_validate_video_accepts_portrait_resolution() { + let mp4_bytes = build_mp4_bytes(true, b"avc1", 1_000, 2160, 3840, false); + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), &mp4_bytes).unwrap(); + + let meta = validate_video_file(tmp.path(), &test_config()) + .expect("portrait video within the 2160x3840 envelope should be accepted"); + assert_eq!((meta.width, meta.height), (2160, 3840)); + } + + #[test] + fn test_validate_video_rejects_resolution_above_short_edge_limit() { + let mp4_bytes = build_mp4_bytes(true, b"avc1", 1_000, 2161, 3840, false); + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), &mp4_bytes).unwrap(); + + let result = validate_video_file(tmp.path(), &test_config()); + assert!( + matches!(result, Err(MediaError::ResolutionTooHigh)), + "expected ResolutionTooHigh, got {result:?}" + ); + } + + #[test] + fn test_validate_video_rejects_resolution_above_long_edge_limit() { + let mp4_bytes = build_mp4_bytes(true, b"avc1", 1_000, 2160, 3841, false); + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), &mp4_bytes).unwrap(); + + let result = validate_video_file(tmp.path(), &test_config()); + assert!( + matches!(result, Err(MediaError::ResolutionTooHigh)), + "expected ResolutionTooHigh, got {result:?}" + ); + } + #[test] fn test_validate_video_resolution_too_high() { let config = test_config(); diff --git a/crates/buzz-relay/src/api/mod.rs b/crates/buzz-relay/src/api/mod.rs index 085cfe8ff..3c2cd36e7 100644 --- a/crates/buzz-relay/src/api/mod.rs +++ b/crates/buzz-relay/src/api/mod.rs @@ -227,7 +227,7 @@ pub mod relay_members { for (role, pubkey) in [("agent", agent), ("owner", owner)] { match state .db - .ensure_user(tenant.community(), pubkey.as_bytes()) + .ensure_user_for_authorization(tenant.community(), pubkey.as_bytes()) .await { Ok(true) => { @@ -247,7 +247,11 @@ pub mod relay_members { let materialized = match state .db - .set_agent_owner(tenant.community(), agent.as_bytes(), owner.as_bytes()) + .set_agent_owner_for_authorization( + tenant.community(), + agent.as_bytes(), + owner.as_bytes(), + ) .await { Ok(true) => true, diff --git a/crates/buzz-relay/src/connection.rs b/crates/buzz-relay/src/connection.rs index 3ff985eb1..b3df196bd 100644 --- a/crates/buzz-relay/src/connection.rs +++ b/crates/buzz-relay/src/connection.rs @@ -14,12 +14,13 @@ use tracing::Instrument as _; use tracing::{debug, info, trace, warn}; use uuid::Uuid; -use buzz_auth::{generate_challenge, AuthContext, LimitType}; +use buzz_auth::{generate_challenge, AuthContext}; use buzz_core::tenant::TenantContext; use nostr::Filter; use crate::handlers; use crate::protocol::{ClientMessage, RelayMessage}; +use crate::rejection::{enforce_ws_admission, send_request_rejection, RejectionTarget}; use crate::state::{ run_registered_community_connection, AppState, CommunityConnectionControl, CommunityDisconnectReason, @@ -564,13 +565,13 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar let permit = match state.handler_semaphore.clone().try_acquire_owned() { Ok(p) => p, Err(_) => { - for message in early_rejection_messages( - None, - Some(&event.id), + // Correlate to the event id: a bare NOTICE here strands the + // client's pending publish exactly as an over-quota one did. + send_request_rejection( + &conn, + RejectionTarget::Event(event.id), "rate-limited: too many concurrent requests", - ) { - conn.send(message); - } + ); return; } }; @@ -596,10 +597,11 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar let permit = match state.handler_semaphore.clone().try_acquire_owned() { Ok(p) => p, Err(_) => { - conn.send(request_rejection_message( - Some(&sub_id), + send_request_rejection( + &conn, + RejectionTarget::Subscription(&sub_id), "rate-limited: too many concurrent requests", - )); + ); return; } }; @@ -618,9 +620,11 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar let permit = match state.handler_semaphore.clone().try_acquire_owned() { Ok(p) => p, Err(_) => { - conn.send(RelayMessage::notice( + send_request_rejection( + &conn, + RejectionTarget::Subscription(&sub_id), "rate-limited: too many concurrent requests", - )); + ); return; } }; @@ -639,133 +643,146 @@ async fn handle_text_message(text: String, conn: Arc, state: Ar } } -fn request_rejection_message(sub_id: Option<&str>, reason: &str) -> String { - match sub_id { - Some(sub_id) => RelayMessage::closed(sub_id, reason), - None => RelayMessage::notice(reason), +fn topic_for_subscription(channel_id: Option) -> EventTopic { + match channel_id { + Some(channel_id) => EventTopic::Channel(channel_id), + None => EventTopic::Global, } } -fn early_rejection_messages( - sub_id: Option<&str>, - event_id: Option<&nostr::EventId>, - reason: &str, -) -> impl Iterator { - // Preserve the existing backoff signal, then settle this specific publish. - // A NOTICE alone cannot identify the EVENT that admission refused. - std::iter::once(request_rejection_message(sub_id, reason)) - .chain(event_id.map(|id| RelayMessage::ok(&id.to_hex(), false, reason))) -} +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::sync::{Arc, Mutex}; -async fn enforce_ws_admission( - msg: &ClientMessage, - conn: &ConnectionState, - state: &AppState, -) -> bool { - let is_event = matches!(msg, ClientMessage::Event(_)); - if !is_event && !matches!(msg, ClientMessage::Req { .. } | ClientMessage::Count { .. }) { - return true; - } + use buzz_auth::AuthMethod; + use nostr::{EventBuilder, Keys, Kind}; - let (pubkey, is_agent) = { - let auth = conn.auth_state.read().await; - match &*auth { - AuthState::Authenticated(ctx) => (ctx.pubkey, ctx.agent_owner_pubkey.is_some()), - _ => return true, - } - }; + /// A connection whose outbound frames a test can read back. + /// + /// Lives here, next to `ConnectionState`, so the crate has one place that + /// knows how to build one. Shared with `crate::rejection`'s tests. + pub(crate) fn test_conn_with_auth( + auth: AuthState, + ) -> (Arc, mpsc::Receiver) { + let (send_tx, send_rx) = mpsc::channel(4); + let (ctrl_tx, _ctrl_rx) = mpsc::channel(4); + let conn = ConnectionState { + conn_id: Uuid::new_v4(), + tenant: TenantContext::resolved( + buzz_core::tenant::CommunityId::from_uuid(Uuid::nil()), + "test.local".to_string(), + ), + remote_addr: "127.0.0.1:1234".parse().expect("socket addr"), + auth_state: RwLock::new(auth), + subscriptions: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + send_tx, + ctrl_tx, + cancel: CancellationToken::new(), + backpressure_count: Arc::new(AtomicU8::new(0)), + grace_limit: 3, + }; + (Arc::new(conn), send_rx) + } - let limits = &state.auth.config().rate_limits; - let (ws_window_secs, ws_limit) = - crate::admission::ws_admission_budget(limits.human_ws_events_per_sec); - let ws_result = crate::admission::check_principal( - state.admission_rate_limiter.as_ref(), - &conn.tenant, - &pubkey, - LimitType::WsEvents, - ws_window_secs, - ws_limit, - ) - .await; - let sub_id = match msg { - ClientMessage::Req { sub_id, .. } => Some(sub_id.as_str()), - _ => None, - }; - let event_id = match msg { - ClientMessage::Event(event) => Some(&event.id), - _ => None, - }; - if !send_admission_result(conn, ws_result, sub_id, event_id) { - return false; + /// An authenticated connection — the only state admission quotas apply to. + pub(crate) fn authenticated_state() -> AuthState { + AuthState::Authenticated(AuthContext { + pubkey: Keys::generate().public_key(), + scopes: Vec::new(), + channel_ids: None, + auth_method: AuthMethod::Nip42, + agent_owner_pubkey: None, + }) } - if is_event { - let message_limit = if is_agent { - limits.agent_standard_messages_per_min - } else { - limits.human_messages_per_min - }; - let message_result = crate::admission::check_principal( - state.admission_rate_limiter.as_ref(), - &conn.tenant, - &pubkey, - LimitType::Messages, - 60, - message_limit, - ) - .await; - if !send_admission_result(conn, message_result, None, event_id) { - return false; + pub(crate) fn read_frame(rx: &mut mpsc::Receiver) -> serde_json::Value { + match rx.try_recv().expect("a frame was sent") { + WsMessage::Text(text) => serde_json::from_str(&text).expect("valid JSON frame"), + other => panic!("unexpected websocket message: {other:?}"), } } - true -} + /// Drives the real `handle_text_message` with every handler permit held, so + /// the EVENT saturation branch is reached through production dispatch rather + /// than by calling its helpers directly. + /// + /// This must go through `handle_text_message`: a test that renders the + /// rejection frame itself stays green when the call site inside the match + /// arm is reverted to a bare `NOTICE`. + #[tokio::test] + async fn saturated_handler_rejects_an_event_on_the_ok_channel() { + let state = crate::state::tests::test_state().await; + // An unauthenticated connection skips the admission quotas, so the + // semaphore is the only gate the frame can trip. + let (conn, mut rx) = test_conn_with_auth(AuthState::Failed); + + let permits = state.handler_semaphore.available_permits(); + let _held = Arc::clone(&state.handler_semaphore) + .acquire_many_owned(permits as u32) + .await + .expect("hold every handler permit"); -fn send_admission_result( - conn: &ConnectionState, - result: Result<(), crate::admission::AdmissionError>, - sub_id: Option<&str>, - event_id: Option<&nostr::EventId>, -) -> bool { - match result { - Ok(()) => true, - Err(crate::admission::AdmissionError::Exceeded { reset_in_secs }) => { - metrics::counter!("buzz_admission_rejections_total", "transport" => "websocket", "reason" => "quota").increment(1); - for message in early_rejection_messages( - sub_id, - event_id, - &format!("rate-limited: quota exceeded; retry in {reset_in_secs}s"), - ) { - conn.send(message); - } - false - } - Err(crate::admission::AdmissionError::Unavailable) => { - metrics::counter!("buzz_admission_rejections_total", "transport" => "websocket", "reason" => "unavailable").increment(1); - for message in early_rejection_messages( - sub_id, - event_id, - "rate-limited: shared admission unavailable", - ) { - conn.send(message); - } - false - } + let event = EventBuilder::new(Kind::TextNote, "hello") + .sign_with_keys(&Keys::generate()) + .expect("sign event"); + let event_id = event.id.to_hex(); + let raw = serde_json::json!(["EVENT", event]).to_string(); + + handle_text_message(raw, Arc::clone(&conn), Arc::clone(&state)).await; + + let frame = read_frame(&mut rx); + assert_eq!( + frame[0], "OK", + "an EVENT turned away for handler saturation must be rejected on the \ + OK channel — a NOTICE carries no event id, so the client's pending \ + publish cannot be settled and the send only times out" + ); + assert_eq!(frame[1], event_id); + assert_eq!(frame[2], false); + assert_eq!(frame[3], "rate-limited: too many concurrent requests"); } -} -fn topic_for_subscription(channel_id: Option) -> EventTopic { - match channel_id { - Some(channel_id) => EventTopic::Channel(channel_id), - None => EventTopic::Global, + /// The REQ arm of the same branch still settles on CLOSED. + #[tokio::test] + async fn saturated_handler_rejects_a_req_on_the_closed_channel() { + let state = crate::state::tests::test_state().await; + let (conn, mut rx) = test_conn_with_auth(AuthState::Failed); + + let permits = state.handler_semaphore.available_permits(); + let _held = Arc::clone(&state.handler_semaphore) + .acquire_many_owned(permits as u32) + .await + .expect("hold every handler permit"); + + let raw = serde_json::json!(["REQ", "history-abc", {"kinds": [1]}]).to_string(); + handle_text_message(raw, Arc::clone(&conn), Arc::clone(&state)).await; + + let frame = read_frame(&mut rx); + assert_eq!(frame[0], "CLOSED"); + assert_eq!(frame[1], "history-abc"); } -} -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{Arc, Mutex}; + /// COUNT refusals follow NIP-45 and close the named query. + #[tokio::test] + async fn saturated_handler_rejects_a_count_on_the_closed_channel() { + let state = crate::state::tests::test_state().await; + let (conn, mut rx) = test_conn_with_auth(AuthState::Failed); + + let permits = state.handler_semaphore.available_permits(); + let _held = Arc::clone(&state.handler_semaphore) + .acquire_many_owned(permits as u32) + .await + .expect("hold every handler permit"); + + let raw = serde_json::json!(["COUNT", "count-abc", {"kinds": [1]}]).to_string(); + handle_text_message(raw, Arc::clone(&conn), Arc::clone(&state)).await; + + let frame = read_frame(&mut rx); + assert_eq!(frame[0], "CLOSED"); + assert_eq!(frame[1], "count-abc"); + assert_eq!(frame[2], "rate-limited: too many concurrent requests"); + } #[derive(Debug, Default)] struct MockSinkState { @@ -860,59 +877,6 @@ mod tests { .collect() } - #[test] - fn req_rejections_are_subscription_scoped() { - let reason = "rate-limited: too many concurrent requests"; - let closed: serde_json::Value = - serde_json::from_str(&request_rejection_message(Some("history-123"), reason)) - .expect("parse CLOSED"); - assert_eq!(closed, serde_json::json!(["CLOSED", "history-123", reason])); - - let notice: serde_json::Value = - serde_json::from_str(&request_rejection_message(None, reason)).expect("parse NOTICE"); - assert_eq!(notice, serde_json::json!(["NOTICE", reason])); - } - - #[test] - fn early_event_rejections_identify_the_event_and_preserve_the_notice() { - let event_id = nostr::EventId::from_hex(&"ab".repeat(32)).expect("event ID"); - for reason in [ - "rate-limited: quota exceeded; retry in 1s", - "rate-limited: shared admission unavailable", - "rate-limited: too many concurrent requests", - ] { - let messages: Vec = - early_rejection_messages(None, Some(&event_id), reason) - .map(|message| serde_json::from_str(&message).expect("rejection JSON")) - .collect(); - assert_eq!( - messages, - vec![ - serde_json::json!(["NOTICE", reason]), - serde_json::json!(["OK", event_id.to_hex(), false, reason]), - ], - "every early EVENT rejection must settle the exact pending publish" - ); - } - } - - #[test] - fn early_non_event_rejections_do_not_emit_publish_acknowledgements() { - let reason = "rate-limited: quota exceeded; retry in 1s"; - for (sub_id, expected) in [ - ( - Some("history-123"), - serde_json::json!(["CLOSED", "history-123", reason]), - ), - (None, serde_json::json!(["NOTICE", reason])), - ] { - let messages: Vec = early_rejection_messages(sub_id, None, reason) - .map(|message| serde_json::from_str(&message).expect("rejection JSON")) - .collect(); - assert_eq!(messages, vec![expected]); - } - } - #[tokio::test] async fn send_loop_batches_queued_data_frames_into_one_flush() { let (data_tx, data_rx) = mpsc::channel(MAX_WS_SEND_BATCH); diff --git a/crates/buzz-relay/src/handlers/command_executor.rs b/crates/buzz-relay/src/handlers/command_executor.rs index b973f9510..c87f9ca09 100644 --- a/crates/buzz-relay/src/handlers/command_executor.rs +++ b/crates/buzz-relay/src/handlers/command_executor.rs @@ -124,7 +124,7 @@ async fn persist_command_event( let channel_id = channel_id_override.or_else(|| extract_channel_id(event)); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .map_err(|e| IngestError::Internal(format!("error: begin transaction: {e}")))?; buzz_deletion::store(db) @@ -503,7 +503,7 @@ async fn handle_dm_add_member( // 3. Validate channel is type "dm" let existing_channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| IngestError::Rejected("invalid: DM not found".into()))?; if existing_channel.channel_type != "dm" { @@ -513,7 +513,7 @@ async fn handle_dm_add_member( // 4. Get existing members, merge with new let existing_members = state .db - .get_members(tenant.community(), channel_id) + .get_members_for_event_write(tenant.community(), channel_id) .await .map_err(|e| IngestError::Internal(format!("error: get members: {e}")))?; @@ -634,7 +634,7 @@ async fn handle_dm_hide( // 3. Validate channel is type "dm" let channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| IngestError::Rejected("invalid: DM not found".into()))?; if channel.channel_type != "dm" { @@ -804,7 +804,7 @@ async fn handle_workflow_def( let community_id = tenant.community(); state .db - .get_channel(community_id, channel_id) + .get_channel_for_event_write(community_id, channel_id) .await .map_err(|_| IngestError::Rejected("invalid: workflow channel not found".into()))?; @@ -1624,7 +1624,10 @@ mod tests { "legacy-malformed", ); - let mut tx = db.begin_transaction().await.expect("begin legacy seed"); + let mut tx = db + .begin_event_write_transaction() + .await + .expect("begin legacy seed"); let (_, was_inserted) = buzz_db::event::insert_event_in_transaction( &mut tx, tenant.community(), diff --git a/crates/buzz-relay/src/handlers/community_provisioning.rs b/crates/buzz-relay/src/handlers/community_provisioning.rs index c58a4d1e3..b55dd9cdd 100644 --- a/crates/buzz-relay/src/handlers/community_provisioning.rs +++ b/crates/buzz-relay/src/handlers/community_provisioning.rs @@ -444,7 +444,7 @@ pub async fn provision_community( if let Some(owner_hex) = &initial_owner { state .db - .bootstrap_owner(record.id, owner_hex) + .provision_owner(record.id, owner_hex) .await .map_err(|e| format!("community provisioned but owner bootstrap failed: {e}"))?; publish_membership_snapshot_if_required(state, record.id, &record.host).await; diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index 92eeec3dc..bb52a773a 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -725,7 +725,10 @@ pub(crate) async fn derive_reaction_channel( _ => return ReactionChannelResult::NoTarget, }; - match db.get_event_by_id(community_id, &id_bytes).await { + match db + .get_event_by_id_for_event_write(community_id, &id_bytes) + .await + { Ok(Some(target)) => match target.channel_id { Some(ch_id) => ReactionChannelResult::Channel(ch_id), None => ReactionChannelResult::NoChannel, @@ -950,7 +953,7 @@ pub(crate) async fn check_channel_membership( Some(ch) => ch.visibility == "open", None => state .db - .get_channel(tenant.community(), ch_id) + .get_channel_for_event_write(tenant.community(), ch_id) .await .map(|ch| ch.visibility == "open") .unwrap_or(false), @@ -1039,7 +1042,9 @@ pub(crate) async fn resolve_nip10_thread_meta( hex::decode(&parent_hex).map_err(|_| "invalid parent event ID hex".to_string())?; let (parent_event_result, parent_meta_result) = tokio::join!( - state.db.get_event_by_id(community_id, &parent_bytes), + state + .db + .get_event_by_id_for_event_write(community_id, &parent_bytes), state .db .get_thread_metadata_by_event(community_id, &parent_bytes), @@ -1075,7 +1080,7 @@ pub(crate) async fn resolve_nip10_thread_meta( } let root_ts = if let Ok(Some(root_ev)) = state .db - .get_event_by_id(community_id, &effective_root) + .get_event_by_id_for_event_write(community_id, &effective_root) .await { chrono::DateTime::from_timestamp(root_ev.event.created_at.as_secs() as i64, 0) @@ -1119,8 +1124,10 @@ pub(crate) async fn resolve_nip10_thread_meta( } let depth = if parent_root == parent_bytes { 1 } else { 2 }; let root_created = if parent_root != parent_bytes { - if let Ok(Some(root_ev)) = - state.db.get_event_by_id(community_id, &parent_root).await + if let Ok(Some(root_ev)) = state + .db + .get_event_by_id_for_event_write(community_id, &parent_root) + .await { chrono::DateTime::from_timestamp(root_ev.event.created_at.as_secs() as i64, 0) .unwrap_or(parent_created) @@ -1227,7 +1234,7 @@ async fn validate_edit_ownership( hex::decode(&target_hex).map_err(|_| "invalid target event ID".to_string())?; let target_event = state .db - .get_event_by_id(community_id, &target_bytes) + .get_event_by_id_for_event_write(community_id, &target_bytes) .await .map_err(|e| format!("db error: {e}"))? .ok_or_else(|| "edit target event not found".to_string())?; @@ -1257,7 +1264,7 @@ async fn validate_edit_ownership( if !is_member { let is_open = state .db - .get_channel(community_id, ch_id) + .get_channel_for_event_write(community_id, ch_id) .await .map(|ch| ch.visibility == "open") .unwrap_or(false); @@ -1308,7 +1315,7 @@ async fn validate_forum_vote_target( hex::decode(&target_hex).map_err(|_| "invalid target event ID".to_string())?; let target_event = state .db - .get_event_by_id(community_id, &target_bytes) + .get_event_by_id_for_event_write(community_id, &target_bytes) .await .map_err(|e| format!("db error: {e}"))? .ok_or_else(|| "vote target event not found".to_string())?; @@ -2169,7 +2176,7 @@ async fn validate_canvas_event( let root_event = state .db - .get_event_by_id(tenant.community(), &root_bytes) + .get_event_by_id_for_event_write(tenant.community(), &root_bytes) .await .map_err(|e| format!("db error looking up canvas thread root: {e}"))? .ok_or_else(|| "invalid: canvas e tag points at an unknown event".to_string())?; @@ -2903,7 +2910,7 @@ async fn ingest_event_inner( })?; match state .db - .get_event_by_id(tenant.community(), &target_bytes) + .get_event_by_id_for_event_write(tenant.community(), &target_bytes) .await { Ok(Some(target)) => target.channel_id, @@ -2951,7 +2958,11 @@ async fn ingest_event_inner( // it later in this request); each gate keeps its existing missing-row // behavior. let channel_row = match channel_id { - Some(ch_id) => state.db.get_channel(tenant.community(), ch_id).await.ok(), + Some(ch_id) => state + .db + .get_channel_for_event_write(tenant.community(), ch_id) + .await + .ok(), None => None, }; // E1 phase-2 (§4.8 phase-2 addendum): resolve the fan-out visibility once, diff --git a/crates/buzz-relay/src/handlers/side_effects.rs b/crates/buzz-relay/src/handlers/side_effects.rs index e17626141..6365b16ff 100644 --- a/crates/buzz-relay/src/handlers/side_effects.rs +++ b/crates/buzz-relay/src/handlers/side_effects.rs @@ -146,7 +146,10 @@ async fn evict_non_member_channel_subscriptions( state: &Arc, channel_id: Uuid, ) -> anyhow::Result<()> { - let members = state.db.get_members(tenant.community(), channel_id).await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; let member_pubkeys: std::collections::HashSet> = members.into_iter().map(|m| m.pubkey).collect(); @@ -294,7 +297,7 @@ pub async fn validate_standard_deletion_event( for target_id in target_ids { let target_event = state .db - .get_event_by_id_including_deleted(tenant.community(), &target_id) + .get_event_by_id_including_deleted_for_event_write(tenant.community(), &target_id) .await? .ok_or_else(|| anyhow::anyhow!("target event not found"))?; @@ -359,7 +362,7 @@ pub async fn validate_admin_event( // (unarchive), which must be allowed through so the channel can be restored. let channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| anyhow::anyhow!("channel not found"))?; let is_unarchive_request = kind == 9002 @@ -618,7 +621,7 @@ pub async fn validate_admin_event( // BEFORE storage. Fail closed: missing target → reject. let target_event = state .db - .get_event_by_id(tenant.community(), &target_id) + .get_event_by_id_for_event_write(tenant.community(), &target_id) .await .map_err(|e| anyhow::anyhow!("db error looking up target: {e}"))? .ok_or_else(|| anyhow::anyhow!("target event not found"))?; @@ -650,7 +653,7 @@ pub async fn validate_admin_event( } let is_open = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map(|ch| ch.visibility == "open") .unwrap_or(false); @@ -957,7 +960,7 @@ async fn emit_addressable_discovery_event( let min_ts = { let existing = state .db - .query_events(&buzz_db::event::EventQuery { + .query_events_for_event_write(&buzz_db::event::EventQuery { kinds: Some(vec![kind as i32]), channel_id: Some(channel_id), limit: Some(1), @@ -989,6 +992,67 @@ async fn emit_addressable_discovery_event( Ok(()) } +fn group_members_tags(group_id: &str, members: &[MemberRecord]) -> anyhow::Result> { + let mut tags: Vec = Vec::with_capacity(members.len() + 1); + tags.push(Tag::parse(["d", group_id])?); + for member in members { + let pubkey_hex = hex::encode(&member.pubkey); + // NIP-29 convention: ["p", pubkey, relay_url, role]. Empty relay_url + // because the canonical relay is implicit (this event is signed by it). + tags.push(Tag::parse(["p", &pubkey_hex, "", &member.role])?); + } + Ok(tags) +} + +async fn store_group_members_event( + tenant: &TenantContext, + state: &Arc, + channel_id: Uuid, + member_snapshot: &mut buzz_db::channel_members::LockedMemberSnapshot, +) -> anyhow::Result> { + let group_id = channel_id.to_string(); + let tags = group_members_tags(&group_id, &member_snapshot.members)?; + let relay_pubkey = state.relay_keypair.public_key().to_bytes(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let ts = member_snapshot + .latest_member_event_timestamp(tenant.community(), channel_id, &relay_pubkey) + .await? + .map(|timestamp| timestamp + 1) + .unwrap_or(now) + .max(now); + let event = EventBuilder::new(Kind::Custom(KIND_NIP29_GROUP_MEMBERS as u16), "") + .tags(tags) + .custom_created_at(nostr::Timestamp::from(ts)) + .sign_with_keys(&state.relay_keypair) + .map_err(|error| anyhow::anyhow!("failed to sign member snapshot: {error}"))?; + let (stored, inserted) = member_snapshot + .replace_member_event(tenant.community(), channel_id, &event) + .await?; + Ok(inserted.then_some(stored)) +} + +async fn dispatch_group_members_event( + tenant: &TenantContext, + state: &Arc, + stored: Option, + relay_pubkey_hex: &str, +) { + if let Some(stored) = stored { + dispatch_persistent_event( + tenant, + state, + &stored, + KIND_NIP29_GROUP_MEMBERS, + relay_pubkey_hex, + None, + ) + .await; + } +} + /// Emit NIP-29 group discovery events (39000, 39001, 39002) signed by the relay keypair. /// Called after group creation, metadata changes, or membership changes. /// Events are stored channel-scoped (`channel_id = Some(...)`) so that existing @@ -1002,8 +1066,14 @@ pub async fn emit_group_discovery_events( state: &Arc, channel_id: Uuid, ) -> anyhow::Result<()> { - let channel = state.db.get_channel(tenant.community(), channel_id).await?; - let members = state.db.get_members(tenant.community(), channel_id).await?; + let channel = state + .db + .get_channel_for_event_write(tenant.community(), channel_id) + .await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; let relay_pubkey_hex = hex::encode(state.relay_keypair.public_key().to_bytes()); let group_id = channel_id.to_string(); @@ -1091,24 +1161,18 @@ pub async fn emit_group_discovery_events( .await?; } - { - let mut tags: Vec = vec![Tag::parse(["d", &group_id])?]; - for m in &members { - let pubkey_hex = hex::encode(&m.pubkey); - // NIP-29 convention: ["p", pubkey, relay_url, role]. Empty relay_url - // because the canonical relay is implicit (this event is signed by it). - tags.push(Tag::parse(["p", &pubkey_hex, "", &m.role])?); - } - emit_addressable_discovery_event( - tenant, - state, - channel_id, - KIND_NIP29_GROUP_MEMBERS, - tags, - &relay_pubkey_hex, - ) + // Re-capture membership behind the writer lock immediately before the + // authoritative 39002 replacement. Metadata/admin snapshots retain their + // existing behavior; only membership publication needs this freshness fence. + let relay_pubkey = state.relay_keypair.public_key().to_bytes(); + let mut member_snapshot = state + .db + .lock_member_snapshot(tenant.community(), channel_id, &relay_pubkey) .await?; - } + let stored_members = + store_group_members_event(tenant, state, channel_id, &mut member_snapshot).await?; + member_snapshot.release().await?; + dispatch_group_members_event(tenant, state, stored_members, &relay_pubkey_hex).await; Ok(()) } @@ -1255,7 +1319,7 @@ async fn handle_put_user( .map_err(|_| anyhow::anyhow!("invalid role: {role_str}"))?, None => state .db - .get_members(tenant.community(), channel_id) + .get_members_for_event_write(tenant.community(), channel_id) .await? .iter() .find(|m| m.pubkey == target_pubkey) @@ -1324,7 +1388,10 @@ async fn handle_remove_user( // Guard: prevent last-owner orphaning on self-removal (kind 9001). if target_pubkey == actor_bytes { - let members = state.db.get_members(tenant.community(), channel_id).await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; if channel_authz::is_sole_owner(&members, &actor_bytes) { return Err(ChannelAuthzError::LastOwnerRemovalTransferFirst.into()); } @@ -1450,7 +1517,7 @@ async fn handle_edit_metadata( "visibility" => { let was_open = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map(|c| c.visibility == "open") .unwrap_or(false); @@ -1566,8 +1633,10 @@ async fn handle_edit_metadata( // same channel by the same actor could collide ids and skip a fan-out. // Not reachable in practice — unarchive has a single human-driven caller; // the reaper only auto-archives — so we don't engineer around it. - for member in - state.db.get_members(tenant.community(), channel_id).await? + for member in state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await? { if let Err(e) = emit_membership_notification( tenant, @@ -1635,7 +1704,7 @@ async fn handle_delete_event_side_effect( // by sending h=A, e=. if let Some(target_event) = state .db - .get_event_by_id_including_deleted(tenant.community(), &target_id) + .get_event_by_id_including_deleted_for_event_write(tenant.community(), &target_id) .await .map_err(|e| anyhow::anyhow!("get_event_by_id failed: {e}"))? { @@ -1737,7 +1806,11 @@ async fn handle_create_group( // no-h-tag path, ingest never creates the channel, so this is the sole // increment. let channel = if let Some(client_uuid) = extract_h_tag_channel(event) { - match state.db.get_channel(tenant.community(), client_uuid).await { + match state + .db + .get_channel_for_event_write(tenant.community(), client_uuid) + .await + { Ok(ch) => ch, Err(_) => { // Channel not found — shouldn't happen (ingest_event pre-created it), @@ -1889,7 +1962,7 @@ async fn handle_join_request( // Only open channels allow self-join via kind:9021. let channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| anyhow::anyhow!("channel not found"))?; if channel.visibility != "open" { @@ -1966,7 +2039,10 @@ async fn handle_leave_request( let actor_bytes = event.pubkey.to_bytes().to_vec(); // Guard: prevent last-owner orphaning on leave. - let members = state.db.get_members(tenant.community(), channel_id).await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; if channel_authz::is_sole_owner(&members, &actor_bytes) { return Err(ChannelAuthzError::LastOwnerRemovalTransferFirst.into()); } @@ -2170,7 +2246,7 @@ async fn handle_standard_deletion_event( for target_id in target_ids { let target_event = match state .db - .get_event_by_id_including_deleted(tenant.community(), &target_id) + .get_event_by_id_including_deleted_for_event_write(tenant.community(), &target_id) .await? { Some(target) => target, @@ -2248,7 +2324,7 @@ async fn handle_standard_deletion_event( if let Ok(react_target_id) = hex::decode(&react_target_hex) { if let Ok(Some(react_target_event)) = state .db - .get_event_by_id(tenant.community(), &react_target_id) + .get_event_by_id_for_event_write(tenant.community(), &react_target_id) .await { let react_target_ts = chrono::DateTime::from_timestamp( @@ -2821,22 +2897,61 @@ async fn emit_initial_ref_state( /// safe to run at startup and periodically without producing an event stream /// when nothing changed. A failure in one community is logged and counted but /// does not prevent the remaining communities from being repaired. +/// Attribution for a NIP-43 membership reconciliation sweep. +#[derive(Clone, Copy)] +pub enum Nip43ReconciliationPurpose { + /// Before listener admission opens. + Bootstrap, + /// Periodic background repair after startup. + Maintenance, +} + +/// Preserve the original maintenance reconciliation API for downstream callers. +#[deprecated(note = "use reconcile_nip43_membership_snapshots_with_purpose")] pub async fn reconcile_nip43_membership_snapshots(state: &Arc) -> anyhow::Result { - let communities = state.db.usage_community_hosts().await?; + reconcile_nip43_membership_snapshots_with_purpose( + state, + Nip43ReconciliationPurpose::Maintenance, + ) + .await +} + +/// Reconcile NIP-43 snapshots with explicit startup or maintenance attribution. +pub async fn reconcile_nip43_membership_snapshots_with_purpose( + state: &Arc, + purpose: Nip43ReconciliationPurpose, +) -> anyhow::Result { + let communities = match purpose { + Nip43ReconciliationPurpose::Bootstrap => state.db.bootstrap_community_hosts().await?, + Nip43ReconciliationPurpose::Maintenance => state.db.usage_community_hosts().await?, + }; let mut reconciled = 0usize; for community in communities { let community_id = buzz_core::CommunityId::from_uuid(community.id); let host = community.host; let result = async { - if !state - .db - .nip43_membership_snapshot_needs_reconciliation( - community_id, - &state.relay_keypair.public_key(), - ) - .await? - { + let needs_reconciliation = match purpose { + Nip43ReconciliationPurpose::Bootstrap => { + state + .db + .nip43_membership_snapshot_needs_reconciliation_for_bootstrap( + community_id, + &state.relay_keypair.public_key(), + ) + .await? + } + Nip43ReconciliationPurpose::Maintenance => { + state + .db + .nip43_membership_snapshot_needs_reconciliation_for_maintenance( + community_id, + &state.relay_keypair.public_key(), + ) + .await? + } + }; + if !needs_reconciliation { return Ok::(false); } @@ -2985,6 +3100,68 @@ pub async fn publish_nip43_member_removed( publish_nip43_delta(tenant, state, 8001, target_pubkey_hex, "member-removed").await } +/// Repair legacy kind:39002 snapshots truncated by the former 1,000-member +/// database cap. +/// +/// The scan is deliberately limited to canonical rosters above that boundary, +/// so normal-sized channels and already-correct large snapshots incur no +/// rewrites. Community identity travels with every candidate; a shared relay +/// never resolves a channel against a neighboring tenant. +pub async fn reconcile_large_channel_member_snapshots( + state: &Arc, +) -> anyhow::Result { + const LEGACY_ROSTER_LIMIT: i64 = 1_000; + + let relay_pubkey = state.relay_keypair.public_key(); + let candidates = state + .db + .list_large_channel_rosters_needing_reconciliation( + LEGACY_ROSTER_LIMIT, + &relay_pubkey.to_bytes(), + ) + .await?; + let relay_pubkey_hex = relay_pubkey.to_hex(); + let mut reconciled = 0usize; + + for candidate in candidates { + let result = async { + let channel_id = candidate.channel_id; + // Hold the membership-writer lock from roster capture through + // replacement. Otherwise a rolling deployment can publish stale + // roster A after another relay commits and publishes roster B. + let mut member_snapshot = state + .db + .lock_member_snapshot(candidate.community_id, channel_id, &relay_pubkey.to_bytes()) + .await?; + let tenant = TenantContext::resolved(candidate.community_id, candidate.host.clone()); + let stored_members = + store_group_members_event(&tenant, state, channel_id, &mut member_snapshot).await?; + member_snapshot.release().await?; + dispatch_group_members_event(&tenant, state, stored_members, &relay_pubkey_hex).await; + Ok::(true) + } + .await; + + match result { + Ok(true) => reconciled += 1, + Ok(false) => {} + Err(error) => { + metrics::counter!("buzz_channel_roster_reconciliation_failures_total").increment(1); + warn!( + community_id = %candidate.community_id, + host = %candidate.host, + channel_id = %candidate.channel_id, + %error, + "large channel roster reconciliation failed" + ); + } + } + } + + metrics::counter!("buzz_channel_roster_reconciliations_total").increment(reconciled as u64); + Ok(reconciled) +} + /// Reconcile channels that exist in the DB but don't have kind:39000 events. /// /// This handles the case where channels were created via direct SQL inserts @@ -2999,7 +3176,10 @@ pub async fn reconcile_channel_events( ) -> anyhow::Result<()> { use buzz_db::event::EventQuery; - let channels = state.db.list_channels(tenant.community(), None).await?; + let channels = state + .db + .list_channels_for_bootstrap(tenant.community(), None) + .await?; if channels.is_empty() { return Ok(()); } @@ -3010,7 +3190,7 @@ pub async fn reconcile_channel_events( let channel_id_str = channel.id.to_string(); let existing = match state .db - .query_events(&EventQuery { + .query_events_for_bootstrap(&EventQuery { kinds: Some(vec![39000]), d_tag: Some(channel_id_str.clone()), limit: Some(1), @@ -3141,7 +3321,7 @@ pub async fn publish_dm_visibility_snapshot( let ts = { let existing = state .db - .query_events(&buzz_db::event::EventQuery { + .query_events_for_event_write(&buzz_db::event::EventQuery { kinds: Some(vec![KIND_DM_VISIBILITY as i32]), pubkey: Some(state.relay_keypair.public_key().to_bytes().to_vec()), d_tag: Some(viewer_hex.clone()), @@ -3315,6 +3495,33 @@ fn topic_for_subscription(channel_id: Option) -> EventTopic { mod tests { use super::*; + #[test] + fn group_members_snapshot_keeps_members_past_one_thousand() { + let channel_id = Uuid::new_v4(); + let members: Vec = (0_u16..1_501) + .map(|index| MemberRecord { + channel_id, + pubkey: vec![(index >> 8) as u8, index as u8], + role: if index == 1_500 { "owner" } else { "member" }.to_string(), + joined_at: chrono::Utc::now(), + invited_by: None, + removed_at: None, + }) + .collect(); + + let tags = group_members_tags(&channel_id.to_string(), &members).expect("build tags"); + assert_eq!(tags.len(), 1_502, "d tag plus every member p tag"); + + let late_pubkey = hex::encode(&members[1_500].pubkey); + assert!(tags.iter().any(|tag| { + let fields = tag.as_slice(); + fields.len() == 4 + && fields[0] == "p" + && fields[1] == late_pubkey + && fields[3] == "owner" + })); + } + #[test] fn delete_tombstone_omits_absent_moderation_metadata() { let content = diff --git a/crates/buzz-relay/src/lib.rs b/crates/buzz-relay/src/lib.rs index fe1fe92db..50ae8e091 100644 --- a/crates/buzz-relay/src/lib.rs +++ b/crates/buzz-relay/src/lib.rs @@ -28,6 +28,7 @@ pub mod gateway; mod job_broker; mod ledger_broker; mod party_broker; +mod rejection; /// REST API route handlers. pub mod api; @@ -61,6 +62,8 @@ pub mod invite_token; /// Colony job queue: reclaiming lapsed leases and escalating jobs that are /// going nowhere. pub mod job_runtime; +/// Fixed-schema evidence for the relay's earliest startup steps. +pub mod lifecycle; /// Inter-relay mesh startup wiring (`BUZZ_MESH` seam). pub mod mesh_boot; /// Prometheus metrics: recorder, upkeep, HTTP middleware. @@ -84,6 +87,7 @@ pub mod price_feed; pub mod protocol; /// Durable NIP-PL matcher and delivery worker. pub mod push_runtime; +mod readiness; /// Axum router construction. pub mod router; /// Shared application state. @@ -95,6 +99,8 @@ pub mod subscription; pub mod telemetry; /// Row-zero host binding: resolve the request community from the connection host. pub mod tenant; +#[cfg(test)] +mod test_support; /// One open task per thread: attach-or-open, completion reports, cascade close. pub(crate) mod thread_task_broker; /// Relay-side tunnel session directory and routing. diff --git a/crates/buzz-relay/src/lifecycle.rs b/crates/buzz-relay/src/lifecycle.rs new file mode 100644 index 000000000..bc9d51062 --- /dev/null +++ b/crates/buzz-relay/src/lifecycle.rs @@ -0,0 +1,591 @@ +//! Fixed-schema evidence for the relay's earliest startup steps. +//! +//! These events are written directly to stderr because crypto, tracing, +//! configuration, and metrics setup can fail before the normal telemetry +//! stack exists. Values are closed enums; raw errors and secrets never enter +//! the lifecycle schema. + +use std::{ + io::Write as _, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use serde::Serialize; +use uuid::Uuid; + +const EVENT_NAME: &str = "buzz_process_lifecycle"; +const SCHEMA_VERSION: u8 = 1; + +/// A bounded early-startup phase. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StartupPhase { + /// Process entry through a usable metrics listener. + ProcessTelemetry, + /// Install the process-wide rustls provider. + CryptoInit, + /// Install structured logging and optional OTLP tracing. + TracingInit, + /// Parse environment-backed configuration. + ConfigLoad, + /// Load and validate relay key material. + KeyLoad, + /// Install the Prometheus recorder and bind its listener. + MetricsBind, +} + +impl StartupPhase { + /// The complete wire vocabulary. + pub const ALL: [Self; 6] = [ + Self::ProcessTelemetry, + Self::CryptoInit, + Self::TracingInit, + Self::ConfigLoad, + Self::KeyLoad, + Self::MetricsBind, + ]; + + /// Stable wire value. + pub const fn as_str(self) -> &'static str { + match self { + Self::ProcessTelemetry => "process_telemetry", + Self::CryptoInit => "crypto_init", + Self::TracingInit => "tracing_init", + Self::ConfigLoad => "config_load", + Self::KeyLoad => "key_load", + Self::MetricsBind => "metrics_bind", + } + } +} + +/// A bounded terminal status. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LifecycleStatus { + /// Required work completed. + Succeeded, + /// Optional work failed and startup may continue. + Degraded, + /// Required work failed. + Failed, + /// Control flow dropped the phase without an explicit terminal. + Abandoned, +} + +impl LifecycleStatus { + #[cfg(test)] + const ALL: [Self; 4] = [ + Self::Succeeded, + Self::Degraded, + Self::Failed, + Self::Abandoned, + ]; + + const fn as_str(self) -> &'static str { + match self { + Self::Succeeded => "succeeded", + Self::Degraded => "degraded", + Self::Failed => "failed", + Self::Abandoned => "abandoned", + } + } +} + +/// A secret-safe terminal reason. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LifecycleReason { + /// Tokio runtime construction failed. + RuntimeBuild, + /// Another rustls provider was already installed. + ProviderConflict, + /// The optional OTLP exporter could not be built. + ExporterBuild, + /// Required configuration was missing, malformed, or unusable. + ConfigInvalid, + /// A required value was missing. + Missing, + /// A required value was invalid. + RequiredInvalid, + /// A required listener could not bind. + Bind, + /// A global metrics recorder already existed. + RecorderConflict, + /// A phase owner disappeared without a terminal. + OwnerDropped, + /// A panic unwound through the phase. + Panic, +} + +impl LifecycleReason { + #[cfg(test)] + const ALL: [Self; 10] = [ + Self::RuntimeBuild, + Self::ProviderConflict, + Self::ExporterBuild, + Self::ConfigInvalid, + Self::Missing, + Self::RequiredInvalid, + Self::Bind, + Self::RecorderConflict, + Self::OwnerDropped, + Self::Panic, + ]; + + /// Stable wire value. + pub const fn as_str(self) -> &'static str { + match self { + Self::RuntimeBuild => "runtime_build", + Self::ProviderConflict => "provider_conflict", + Self::ExporterBuild => "exporter_build", + Self::ConfigInvalid => "config_invalid", + Self::Missing => "missing", + Self::RequiredInvalid => "required_invalid", + Self::Bind => "bind", + Self::RecorderConflict => "recorder_conflict", + Self::OwnerDropped => "owner_dropped", + Self::Panic => "panic", + } + } +} + +#[derive(Clone, Debug, Serialize)] +struct LifecycleEvent { + event_name: &'static str, + schema_version: u8, + process_boot_id: Uuid, + sequence: u64, + track: &'static str, + phase: &'static str, + edge: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option<&'static str>, + process_started_at_unix_ms: u64, + observed_at_unix_ms: u64, + process_elapsed_ms: u64, + #[serde(skip_serializing_if = "Option::is_none")] + phase_elapsed_ms: Option, +} + +trait EventWriter: Send + Sync { + fn emit(&self, event: &LifecycleEvent); +} + +struct StderrWriter; + +impl EventWriter for StderrWriter { + fn emit(&self, event: &LifecycleEvent) { + // Best effort: reporting a startup error must never create another + // panic. This sink intentionally ignores RUST_LOG filters. + let mut stderr = std::io::stderr().lock(); + if serde_json::to_writer(&mut stderr, event).is_ok() { + let _ = stderr.write_all(b"\n"); + } + } +} + +struct ProcessLifecycle { + boot_id: Uuid, + sequence: AtomicU64, + wall_origin: SystemTime, + monotonic_origin: Instant, + writer: Arc, +} + +impl ProcessLifecycle { + fn new(writer: Arc) -> Arc { + let wall_origin = SystemTime::now(); + let monotonic_origin = Instant::now(); + Arc::new(Self { + boot_id: Uuid::new_v4(), + sequence: AtomicU64::new(1), + wall_origin, + monotonic_origin, + writer, + }) + } + + fn start(self: &Arc, phase: StartupPhase) -> PhaseGuard { + let started_at = if phase == StartupPhase::ProcessTelemetry { + self.monotonic_origin + } else { + Instant::now() + }; + self.emit(phase, "started", None, None, None); + PhaseGuard { + lifecycle: Arc::clone(self), + phase, + started_at, + finished: false, + } + } + + fn emit( + &self, + phase: StartupPhase, + edge: &'static str, + status: Option, + reason: Option, + elapsed: Option, + ) { + self.writer.emit(&LifecycleEvent { + event_name: EVENT_NAME, + schema_version: SCHEMA_VERSION, + process_boot_id: self.boot_id, + sequence: self.sequence.fetch_add(1, Ordering::Relaxed), + track: "startup", + phase: phase.as_str(), + edge, + status: status.map(LifecycleStatus::as_str), + reason: reason.map(LifecycleReason::as_str), + process_started_at_unix_ms: millis_since_epoch(self.wall_origin), + observed_at_unix_ms: millis_since_epoch(SystemTime::now()), + process_elapsed_ms: saturating_millis(self.monotonic_origin.elapsed()), + phase_elapsed_ms: elapsed.map(saturating_millis), + }); + } +} + +/// Owns one phase from its start event through exactly one terminal. +pub struct PhaseGuard { + lifecycle: Arc, + phase: StartupPhase, + started_at: Instant, + finished: bool, +} + +impl PhaseGuard { + /// Record successful completion. + pub fn succeed(self) { + self.finish(LifecycleStatus::Succeeded, None); + } + + /// Record an allowed degradation. + pub fn degrade(self, reason: LifecycleReason) { + self.finish(LifecycleStatus::Degraded, Some(reason)); + } + + /// Record a fatal failure. + pub fn fail(self, reason: LifecycleReason) { + self.finish(LifecycleStatus::Failed, Some(reason)); + } + + fn finish(mut self, status: LifecycleStatus, reason: Option) { + let elapsed = self.started_at.elapsed(); + self.lifecycle + .emit(self.phase, "terminal", Some(status), reason, Some(elapsed)); + self.finished = true; + } +} + +impl Drop for PhaseGuard { + fn drop(&mut self) { + if self.finished { + return; + } + let (status, reason) = if std::thread::panicking() { + (LifecycleStatus::Failed, LifecycleReason::Panic) + } else { + (LifecycleStatus::Abandoned, LifecycleReason::OwnerDropped) + }; + self.lifecycle.emit( + self.phase, + "terminal", + Some(status), + Some(reason), + Some(self.started_at.elapsed()), + ); + self.finished = true; + } +} + +/// Tracks the aggregate early-startup phase and its fixed subphases. +pub struct BootTracker { + lifecycle: Arc, + headline: PhaseGuard, + degraded: Option, +} + +impl BootTracker { + /// Start lifecycle accounting before constructing Tokio. + pub fn start_before_runtime( + build: impl FnOnce() -> Result, + ) -> Result<(Runtime, Self), Error> { + Self::start_before_runtime_with_writer(Arc::new(StderrWriter), build) + } + + fn start_before_runtime_with_writer( + writer: Arc, + build: impl FnOnce() -> Result, + ) -> Result<(Runtime, Self), Error> { + let lifecycle = ProcessLifecycle::new(writer); + let boot = Self { + headline: lifecycle.start(StartupPhase::ProcessTelemetry), + lifecycle, + degraded: None, + }; + match build() { + Ok(runtime) => Ok((runtime, boot)), + Err(error) => { + boot.fail(LifecycleReason::RuntimeBuild); + Err(error) + } + } + } + + /// Start a fixed early-startup subphase. + #[must_use = "dropping a phase guard emits an abandoned terminal"] + pub fn start(&self, phase: StartupPhase) -> PhaseGuard { + assert_ne!(phase, StartupPhase::ProcessTelemetry); + self.lifecycle.start(phase) + } + + /// Run a required phase and atomically terminalize both it and startup on failure. + pub fn run_required( + self, + phase: StartupPhase, + work: impl FnOnce() -> Result, + classify: impl FnOnce(&Error) -> LifecycleReason, + ) -> Result<(Self, T), Error> { + let phase_guard = self.start(phase); + match work() { + Ok(value) => { + phase_guard.succeed(); + Ok((self, value)) + } + Err(error) => { + let reason = classify(&error); + phase_guard.fail(reason); + self.fail(reason); + Err(error) + } + } + } + + /// Preserve the first optional degradation for the aggregate terminal. + pub fn mark_degraded(&mut self, reason: LifecycleReason) { + self.degraded.get_or_insert(reason); + } + + /// Finish early startup with a structured lifecycle terminal. + pub fn finish(self) { + let status = if self.degraded.is_some() { + LifecycleStatus::Degraded + } else { + LifecycleStatus::Succeeded + }; + self.headline.finish(status, self.degraded); + } + + fn fail(self, reason: LifecycleReason) { + self.headline.fail(reason); + } +} + +fn millis_since_epoch(time: SystemTime) -> u64 { + time.duration_since(UNIX_EPOCH) + .map(saturating_millis) + .unwrap_or(0) +} + +fn saturating_millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{panic::AssertUnwindSafe, sync::Mutex}; + + #[derive(Default)] + struct CapturingWriter(Mutex>); + + impl EventWriter for CapturingWriter { + fn emit(&self, event: &LifecycleEvent) { + self.0.lock().expect("capturing writer").push(event.clone()); + } + } + + fn recorder() -> (Arc, Arc) { + let writer = Arc::new(CapturingWriter::default()); + (ProcessLifecycle::new(writer.clone()), writer) + } + + fn events(writer: &CapturingWriter) -> Vec { + writer.0.lock().expect("capturing writer").clone() + } + + #[test] + fn explicit_and_dropped_terminals_are_exactly_once() { + let (lifecycle, writer) = recorder(); + lifecycle.start(StartupPhase::ConfigLoad).succeed(); + drop(lifecycle.start(StartupPhase::KeyLoad)); + + let events = events(&writer); + assert_eq!(events.len(), 4); + assert_eq!(events[0].sequence, 1); + assert_eq!(events[1].status, Some("succeeded")); + assert_eq!(events[3].status, Some("abandoned")); + assert_eq!(events[3].reason, Some("owner_dropped")); + } + + #[test] + fn panic_unwind_is_bounded() { + let (lifecycle, writer) = recorder(); + let panic = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _phase = lifecycle.start(StartupPhase::CryptoInit); + panic!("controlled test panic"); + })); + assert!(panic.is_err()); + let events = events(&writer); + assert_eq!(events[1].status, Some("failed")); + assert_eq!(events[1].reason, Some("panic")); + } + + #[test] + fn runtime_failure_terminalizes_the_headline() { + let writer = Arc::new(CapturingWriter::default()); + let result = BootTracker::start_before_runtime_with_writer( + writer.clone(), + || -> Result<(), &'static str> { Err("controlled") }, + ); + assert!(matches!(result, Err("controlled"))); + let events = events(&writer); + assert_eq!(events.len(), 2); + assert_eq!(events[1].phase, "process_telemetry"); + assert_eq!(events[1].reason, Some("runtime_build")); + } + + #[test] + fn aggregate_preserves_optional_degradation() { + let (lifecycle, writer) = recorder(); + let mut boot = BootTracker { + headline: lifecycle.start(StartupPhase::ProcessTelemetry), + lifecycle, + degraded: None, + }; + boot.mark_degraded(LifecycleReason::ExporterBuild); + boot.finish(); + let events = events(&writer); + assert_eq!(events[1].status, Some("degraded")); + assert_eq!(events[1].reason, Some("exporter_build")); + } + + #[test] + fn required_failure_terminalizes_subphase_and_headline() { + let (lifecycle, writer) = recorder(); + let boot = BootTracker { + headline: lifecycle.start(StartupPhase::ProcessTelemetry), + lifecycle, + degraded: None, + }; + let result = boot.run_required( + StartupPhase::MetricsBind, + || -> Result<(), &'static str> { Err("controlled") }, + |_error| LifecycleReason::RecorderConflict, + ); + assert!(matches!(result, Err("controlled"))); + + let events = events(&writer); + assert_eq!(events.len(), 4); + assert_eq!(events[2].phase, "metrics_bind"); + assert_eq!(events[2].status, Some("failed")); + assert_eq!(events[2].reason, Some("recorder_conflict")); + assert_eq!(events[3].phase, "process_telemetry"); + assert_eq!(events[3].status, Some("failed")); + assert_eq!(events[3].reason, Some("recorder_conflict")); + } + + #[test] + fn schema_and_vocabulary_are_frozen() { + assert_eq!( + StartupPhase::ALL.map(StartupPhase::as_str), + [ + "process_telemetry", + "crypto_init", + "tracing_init", + "config_load", + "key_load", + "metrics_bind", + ] + ); + let (lifecycle, writer) = recorder(); + drop(lifecycle.start(StartupPhase::ConfigLoad)); + let values: Vec<_> = events(&writer) + .iter() + .map(|event| serde_json::to_value(event).expect("serialize lifecycle event")) + .collect(); + assert_eq!(values[0]["schema_version"], SCHEMA_VERSION); + assert_eq!(values[0]["event_name"], EVENT_NAME); + assert_eq!(values[1]["status"], "abandoned"); + let mut started_keys: Vec<_> = values[0] + .as_object() + .expect("started event object") + .keys() + .map(String::as_str) + .collect(); + started_keys.sort_unstable(); + assert_eq!( + started_keys, + [ + "edge", + "event_name", + "observed_at_unix_ms", + "phase", + "process_boot_id", + "process_elapsed_ms", + "process_started_at_unix_ms", + "schema_version", + "sequence", + "track", + ] + ); + let mut terminal_keys: Vec<_> = values[1] + .as_object() + .expect("terminal event object") + .keys() + .map(String::as_str) + .collect(); + terminal_keys.sort_unstable(); + assert_eq!( + terminal_keys, + [ + "edge", + "event_name", + "observed_at_unix_ms", + "phase", + "phase_elapsed_ms", + "process_boot_id", + "process_elapsed_ms", + "process_started_at_unix_ms", + "reason", + "schema_version", + "sequence", + "status", + "track", + ] + ); + assert_eq!( + LifecycleStatus::ALL.map(LifecycleStatus::as_str), + ["succeeded", "degraded", "failed", "abandoned",] + ); + assert_eq!( + LifecycleReason::ALL.map(LifecycleReason::as_str), + [ + "runtime_build", + "provider_conflict", + "exporter_build", + "config_invalid", + "missing", + "required_invalid", + "bind", + "recorder_conflict", + "owner_dropped", + "panic", + ] + ); + } +} diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 58a6ddeae..29d4a606d 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -1,5 +1,4 @@ use std::collections::{HashMap, HashSet}; -use std::sync::atomic::Ordering; use std::sync::Arc; use tracing::{debug, error, info, warn}; @@ -18,6 +17,7 @@ use buzz_pubsub::PubSubManager; use buzz_search::SearchService; use buzz_relay::config::{Config, MAX_DRAIN_JITTER_MS}; +use buzz_relay::lifecycle::{BootTracker, LifecycleReason, StartupPhase}; use buzz_relay::metrics as relay_metrics; use buzz_relay::router::{build_health_router, build_router}; use buzz_relay::state::AppState; @@ -35,6 +35,18 @@ fn buzz_auto_migrate_enabled(value: Option<&str>) -> bool { }) } +async fn connect_audit_pool(config: &DbConfig) -> anyhow::Result { + let audit_config = DbConfig { + read_database_url: None, + max_connections: 5, + min_connections: 1, + ..config.clone() + }; + Db::connect_writer_pool(&audit_config) + .await + .map_err(Into::into) +} + fn relay_keypair_from_config(relay_private_key: Option<&str>) -> anyhow::Result { let hex = relay_private_key.ok_or_else(|| { anyhow::anyhow!( @@ -93,15 +105,36 @@ impl EmissionScope { const USAGE_METRICS_LOCK_KEY: i64 = 0x4255_5A5A_4D45_5452; -#[tokio::main] -async fn main() -> anyhow::Result<()> { +fn main() -> anyhow::Result<()> { + let (runtime, boot) = BootTracker::start_before_runtime(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + }) + .map_err(|error| anyhow::anyhow!("failed to build Tokio runtime: {error}"))?; + runtime.block_on(run_relay_main(boot)) +} + +async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { // Install the ring CryptoProvider for rustls. Required before any rustls // TLS connection (rediss:// to ElastiCache, wss://, S3 over TLS): both // aws-lc-rs and ring are compiled in transitively, so rustls can't // auto-select a provider and would panic at first use without this. - rustls::crypto::ring::default_provider() - .install_default() - .expect("failed to install rustls crypto provider"); + let (mut boot, ()) = boot + .run_required( + StartupPhase::CryptoInit, + || { + rustls::crypto::ring::default_provider() + .install_default() + .map_err(|_provider| ()) + }, + |_error| LifecycleReason::ProviderConflict, + ) + .map_err(|()| { + anyhow::anyhow!( + "failed to install rustls crypto provider: another provider is already installed" + ) + })?; // JSON-only structured logs — simple, machine-parseable, CAKE-compatible. // If OTEL_EXPORTER_OTLP_ENDPOINT is set, also attach an OpenTelemetry tracing @@ -110,6 +143,7 @@ async fn main() -> anyhow::Result<()> { // Build a single shared Resource (service.name=buzz-relay by default, overridable // via OTEL_SERVICE_NAME) for the trace provider so that Datadog can identify // spans under the correct service identity. + let tracing_init = boot.start(StartupPhase::TracingInit); let resource = telemetry::service_resource(); let tracer_init = telemetry::try_init_tracer(resource.clone()); let otel_enabled = matches!(&tracer_init, telemetry::TracerInit::Enabled(_)); @@ -143,17 +177,43 @@ async fn main() -> anyhow::Result<()> { .init(); // Log any exporter-build failure now that the subscriber is installed. - if let telemetry::TracerInit::ExporterBuildFailed(ref e) = tracer_init { - warn!(error = %e, "Failed to build OTLP trace exporter; distributed tracing disabled"); + match &tracer_init { + telemetry::TracerInit::Enabled(_) => tracing_init.succeed(), + // Structured logging is installed regardless of whether optional OTLP + // export is configured, so the phase itself completed successfully. + telemetry::TracerInit::Disabled => tracing_init.succeed(), + telemetry::TracerInit::ExporterBuildFailed(_) => { + tracing_init.degrade(LifecycleReason::ExporterBuild); + boot.mark_degraded(LifecycleReason::ExporterBuild); + // Do not log the raw exporter error: OTLP endpoint URLs can carry + // credentials. The bounded lifecycle reason is sufficient here. + warn!("Failed to build OTLP trace exporter; distributed tracing disabled"); + } } info!("Starting buzz-relay"); - let config = Config::from_env().map_err(|e| { - error!("Invalid configuration: {e}"); - anyhow::anyhow!("Configuration error: {e}") - })?; - let relay_keypair = relay_keypair_from_config(config.relay_private_key.as_deref())?; + let (next_boot, config) = boot + .run_required(StartupPhase::ConfigLoad, Config::from_env, |_error| { + LifecycleReason::ConfigInvalid + }) + .map_err(|error| { + error!("Invalid configuration: {error}"); + anyhow::anyhow!("Configuration error: {error}") + })?; + boot = next_boot; + + let key_failure = if config.relay_private_key.is_some() { + LifecycleReason::RequiredInvalid + } else { + LifecycleReason::Missing + }; + let (next_boot, relay_keypair) = boot.run_required( + StartupPhase::KeyLoad, + || relay_keypair_from_config(config.relay_private_key.as_deref()), + |_error| key_failure, + )?; + boot = next_boot; info!( bind_addr = %config.bind_addr, relay_url = %config.relay_url, @@ -166,7 +226,18 @@ async fn main() -> anyhow::Result<()> { let usage_interval_secs = usage_metrics_interval_secs(); let usage_idle_timeout_secs = usage_metrics_idle_timeout_secs(usage_interval_secs); - relay_metrics::install(config.metrics_port, usage_idle_timeout_secs); + let (boot, ()) = boot.run_required( + StartupPhase::MetricsBind, + || relay_metrics::try_install(config.metrics_port, usage_idle_timeout_secs), + |error| match error.failure() { + relay_metrics::MetricsInstallFailure::Bind => LifecycleReason::Bind, + relay_metrics::MetricsInstallFailure::RecorderConflict => { + LifecycleReason::RecorderConflict + } + relay_metrics::MetricsInstallFailure::ExporterBuild => LifecycleReason::ExporterBuild, + }, + )?; + boot.finish(); metrics::gauge!("buzz_audit_enabled").set(if config.audit_enabled { 1.0 } else { 0.0 }); info!( port = config.metrics_port, @@ -181,7 +252,8 @@ async fn main() -> anyhow::Result<()> { max_connections: config.db_pool_size, read_max_connections: config.db_read_pool_size, ..DbConfig::default() - }; + } + .with_session_timeouts_from_env(); let db = Db::new(&db_config).await.map_err(|e| { error!("Failed to connect to Postgres: {e}"); anyhow::anyhow!("DB connection failed: {e}") @@ -288,7 +360,7 @@ async fn main() -> anyhow::Result<()> { ); None } else { - match db.ensure_configured_community(&host).await { + match db.ensure_configured_community_for_bootstrap(&host).await { Ok(record) => { info!(host = %record.host, community = %record.id, "Deployment community ensured"); Some(record.id) @@ -388,10 +460,7 @@ async fn main() -> anyhow::Result<()> { } let audit = if config.audit_enabled { - let audit_pool = sqlx::postgres::PgPoolOptions::new() - .max_connections(5) - .min_connections(1) - .connect(&config.database_url) + let audit_pool = connect_audit_pool(&db_config) .await .map_err(|e| anyhow::anyhow!("Audit DB connection failed: {e}"))?; info!("Audit service ready"); @@ -698,12 +767,41 @@ async fn main() -> anyhow::Result<()> { ); } + match state.db.verify_channel_roster_fence().await { + Ok(()) => { + info!("Channel roster fence verified"); + } + Err(error) => { + error!(%error, "Channel roster fence validation failed"); + return Err(anyhow::anyhow!( + "Channel roster fence is unsafe; apply or repair migration 0032 before starting this relay: {error}" + )); + } + } + + // Repair legacy NIP-29 channel rosters that were persisted while the + // canonical member query still truncated at 1,000 rows. Validation above + // makes migration 0032 a code/schema compatibility gate before the new + // replacement protocol or listener can serve traffic. + match buzz_relay::handlers::side_effects::reconcile_large_channel_member_snapshots(&state).await + { + Ok(count) if count > 0 => info!(count, "large channel member snapshots repaired"), + Ok(_) => {} + Err(error) => { + tracing::warn!(%error, "large channel member snapshot startup reconciliation failed") + } + } + // NIP-43: reconcile the event-backed roster for every provisioned // community before opening the listener. `relay_members` is canonical; // this repairs pre-snapshot communities and any publication that failed // after a membership transaction committed. if config.require_relay_membership { - match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots(&state).await + match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots_with_purpose( + &state, + buzz_relay::handlers::side_effects::Nip43ReconciliationPurpose::Bootstrap, + ) + .await { Ok(count) => info!(count, "NIP-43 membership snapshots reconciled on startup"), Err(error) => { @@ -722,8 +820,9 @@ async fn main() -> anyhow::Result<()> { interval.tick().await; loop { interval.tick().await; - match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots( + match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots_with_purpose( &reconcile_state, + buzz_relay::handlers::side_effects::Nip43ReconciliationPurpose::Maintenance, ) .await { @@ -1305,6 +1404,7 @@ async fn main() -> anyhow::Result<()> { metrics::gauge!("buzz_db_pool_idle").set(db_stats.idle as f64); metrics::gauge!("buzz_db_pool_active").set(active as f64); metrics::gauge!("buzz_db_pool_max").set(db_stats.max as f64); + pool_state.db.refresh_pool_waiter_metrics(); if let Some(read_stats) = pool_state.db.read_pool_stats() { let read_active = read_stats.size.saturating_sub(read_stats.idle); @@ -1584,7 +1684,7 @@ async fn serve( }); let (shutdown_tx, _) = tokio::sync::watch::channel(false); - let shutdown_flag = Arc::clone(&state.shutting_down); + let shutdown_state = Arc::clone(&state); let drain_conn_manager = Arc::clone(&state.conn_manager); let drain_jitter_ms = state.config.drain_jitter_ms; let tx = shutdown_tx.clone(); @@ -1617,7 +1717,7 @@ async fn serve( // sleeps. Not implemented here. This comment records the plan only. let shutdown_handle = tokio::spawn(async move { shutdown_signal().await; - shutdown_flag.store(true, Ordering::Relaxed); + shutdown_state.begin_shutdown(); info!("Shutdown signal received — readiness now returns 503"); // 5s grace: let K8s stop routing new traffic before we close listeners. tokio::time::sleep(std::time::Duration::from_secs(5)).await; @@ -2334,10 +2434,11 @@ mod tests { use uuid::Uuid; use super::{ - buzz_auto_migrate_enabled, dropped_in_memory_keys, idle_timeout_secs, + buzz_auto_migrate_enabled, connect_audit_pool, dropped_in_memory_keys, idle_timeout_secs, refresh_legacy_active_gauge_recency, relay_keypair_from_config, run_periodic_until_cancelled, EmissionScope, InMemoryMetricKey, }; + use buzz_db::DbConfig; use metrics::GaugeFn; use metrics_util::{ debugging::DebugValue, @@ -2369,6 +2470,67 @@ mod tests { assert!(tick_count.load(std::sync::atomic::Ordering::Relaxed) <= 1); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn audit_writer_pool_installs_timeouts_and_bounds_advisory_lock_waits() { + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL"); + let pool = connect_audit_pool(&DbConfig { + database_url, + max_connections: 2, + min_connections: 0, + lock_timeout_ms: 500, + idle_txn_timeout_ms: 60_000, + statement_timeout_ms: 0, + ..DbConfig::default() + }) + .await + .expect("connect audit writer pool"); + + let (lock, idle, statement): (String, String, String) = sqlx::query_as( + "SELECT current_setting('lock_timeout'), \ + current_setting('idle_in_transaction_session_timeout'), \ + current_setting('statement_timeout')", + ) + .fetch_one(&pool) + .await + .expect("read effective audit writer GUCs"); + assert_eq!(lock, "500ms"); + assert_eq!(idle, "1min"); + assert_eq!(statement, "0"); + + let lock_key = i64::from_be_bytes( + Uuid::new_v4().as_bytes()[..8] + .try_into() + .expect("eight UUID bytes"), + ); + let mut holder = pool.acquire().await.expect("audit lock holder"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *holder) + .await + .expect("hold audit advisory lock"); + + let started = std::time::Instant::now(); + let mut waiter = pool.acquire().await.expect("audit lock waiter"); + let error = sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *waiter) + .await + .expect_err("audit advisory-lock waiter must time out"); + let code = match &error { + sqlx::Error::Database(db_error) => db_error.code().map(|code| code.to_string()), + other => panic!("expected database error, got {other:?}"), + }; + assert_eq!(code.as_deref(), Some("55P03")); + assert!(started.elapsed() < Duration::from_secs(5)); + + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *holder) + .await + .expect("release audit advisory lock"); + } + #[test] fn buzz_auto_migrate_is_opt_in() { assert!(!buzz_auto_migrate_enabled(None)); diff --git a/crates/buzz-relay/src/metrics.rs b/crates/buzz-relay/src/metrics.rs index 16e521a44..7dbdbb2c7 100644 --- a/crates/buzz-relay/src/metrics.rs +++ b/crates/buzz-relay/src/metrics.rs @@ -21,7 +21,7 @@ use axum::{ middleware::Next, response::Response, }; -use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; +use metrics_exporter_prometheus::{BuildError, Matcher, PrometheusBuilder}; use metrics_util::MetricKindMask; /// HTTP latency buckets (milliseconds) — only for `http_request_latency_ms`. @@ -32,6 +32,17 @@ const LATENCY_BUCKETS_MS: [f64; 11] = [ /// Seconds-scale buckets for internal processing histograms (event, search, audit). const DURATION_BUCKETS_S: [f64; 10] = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0]; +/// Readiness buckets concentrate resolution near the two-second failure budget. +const READINESS_DURATION_BUCKETS_S: [f64; 15] = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, +]; + +/// Pool checkout buckets: dense around normal sub-100ms waits, with explicit +/// coverage of the reader's 150ms and writer's default three-second budgets. +const DB_POOL_ACQUIRE_DURATION_BUCKETS_S: [f64; 9] = + [0.001, 0.005, 0.01, 0.025, 0.05, 0.15, 0.5, 1.0, 3.0]; +const DB_POOL_ACQUIRE_DURATION_UNIT: metrics::Unit = metrics::Unit::Seconds; + /// Seconds-scale buckets for Git hydration and pack streams. const GIT_DURATION_BUCKETS_S: [f64; 13] = [ 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, @@ -56,16 +67,8 @@ const GIT_PACK_BUCKETS: [f64; 9] = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 1 /// Integer-count buckets for fan-out recipient histograms. const FANOUT_BUCKETS: [f64; 9] = [0.0, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 500.0, 1000.0]; -/// Install the global metrics recorder and spawn the Prometheus HTTP exporter. -/// -/// `build()` returns the recorder + exporter future and internally spawns -/// the upkeep task, so no separate upkeep call is needed. -/// -/// Must be called from within a Tokio runtime. -/// Panics if a recorder is already installed or the port is in use. -pub fn install(port: u16, gauge_idle_timeout_secs: u64) { - let (recorder, exporter) = PrometheusBuilder::new() - .with_http_listener(([0, 0, 0, 0], port)) +fn configured_prometheus_builder(gauge_idle_timeout_secs: u64) -> PrometheusBuilder { + PrometheusBuilder::new() // Remove gauge series that the relay intentionally stops emitting. .idle_timeout( MetricKindMask::GAUGE, @@ -102,6 +105,16 @@ pub fn install(port: u16, gauge_idle_timeout_secs: u64) { &GIT_DURATION_BUCKETS_S, ) .expect("valid git compaction duration bucket boundaries") + .set_buckets_for_metric( + Matcher::Full("buzz_readiness_check_duration_seconds".to_owned()), + &READINESS_DURATION_BUCKETS_S, + ) + .expect("valid readiness duration bucket boundaries") + .set_buckets_for_metric( + Matcher::Full("buzz_db_pool_acquire_duration_seconds".to_owned()), + &DB_POOL_ACQUIRE_DURATION_BUCKETS_S, + ) + .expect("valid DB pool acquisition duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_hydrate_bytes".to_owned()), &GIT_BYTES_BUCKETS, @@ -139,11 +152,119 @@ pub fn install(port: u16, gauge_idle_timeout_secs: u64) { &FANOUT_BUCKETS, ) .expect("valid fanout bucket boundaries") +} + +/// A bounded class of metrics installation failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MetricsInstallFailure { + /// The Prometheus listener could not bind. + Bind, + /// Another component already installed a global recorder. + RecorderConflict, + /// The exporter could not be built for another reason. + ExporterBuild, +} + +/// An error returned while installing Prometheus metrics. +#[derive(Debug, thiserror::Error)] +pub enum MetricsInstallError { + /// Prometheus exporter construction failed. + #[error("failed to build Prometheus exporter: {0}")] + Build(#[source] BuildError), + /// Another component already installed the process-global recorder. + #[error("the global metrics recorder is already installed")] + RecorderConflict, +} + +impl MetricsInstallError { + /// Return the secret-safe lifecycle classification. + pub const fn failure(&self) -> MetricsInstallFailure { + match self { + Self::Build(BuildError::FailedToCreateHTTPListener(_)) => MetricsInstallFailure::Bind, + Self::Build(_) => MetricsInstallFailure::ExporterBuild, + Self::RecorderConflict => MetricsInstallFailure::RecorderConflict, + } + } +} + +/// Try to install the global metrics recorder and spawn the Prometheus HTTP exporter. +/// +/// `build()` returns the recorder + exporter future and internally spawns +/// the upkeep task, so no separate upkeep call is needed. +/// +/// Must be called from within a Tokio runtime. +/// Listener and global-recorder failures are returned rather than panicking. +/// A later exporter exit remains detached from relay service; external scrape +/// coverage is authoritative for exporter availability. +pub fn try_install(port: u16, gauge_idle_timeout_secs: u64) -> Result<(), MetricsInstallError> { + let (recorder, exporter) = configured_prometheus_builder(gauge_idle_timeout_secs) + .with_http_listener(([0, 0, 0, 0], port)) .build() - .expect("metrics exporter must build exactly once"); + .map_err(MetricsInstallError::Build)?; - metrics::set_global_recorder(recorder).expect("global recorder must be set exactly once"); + metrics::set_global_recorder(recorder) + .map_err(|_error| MetricsInstallError::RecorderConflict)?; + describe_readiness_metrics(); + describe_db_pool_metrics(); tokio::spawn(exporter); + Ok(()) +} + +/// Install the global metrics recorder and spawn the Prometheus HTTP exporter. +/// +/// This compatibility entry point preserves the original panic-on-failure API. +/// New startup code should use [`try_install`] to report typed failures. +pub fn install(port: u16, gauge_idle_timeout_secs: u64) { + try_install(port, gauge_idle_timeout_secs) + .unwrap_or_else(|error| panic!("metrics exporter must install exactly once: {error}")); +} + +/// Register the frozen operation-aware pool-acquisition contract. +pub(crate) fn describe_db_pool_metrics() { + metrics::describe_histogram!( + "buzz_db_pool_acquire_duration_seconds", + DB_POOL_ACQUIRE_DURATION_UNIT, + "Database pool checkout duration by valid pool role and operation" + ); + metrics::describe_counter!( + "buzz_db_pool_acquire_attempts_total", + "Database pool checkout terminals by valid pool role, operation, and outcome" + ); + metrics::describe_gauge!( + "buzz_db_pool_waiters", + "Current tracked-operation database pool checkout attempts in progress by valid pool role and operation" + ); +} + +/// Register the frozen readiness metric descriptions with the active recorder. +pub(crate) fn describe_readiness_metrics() { + metrics::describe_counter!( + "buzz_readiness_checks_total", + "Kubernetes health-listener readiness probes by terminal bounded reason" + ); + metrics::describe_counter!( + "buzz_readiness_dependency_checks_total", + "Completed readiness dependency attempts by dependency and bounded outcome" + ); + metrics::describe_histogram!( + "buzz_readiness_check_duration_seconds", + metrics::Unit::Seconds, + "Completed readiness check duration without outcome label multiplication" + ); + metrics::describe_gauge!( + "buzz_readiness_state", + "Latest publishable readiness state by check, where 1 is ready and 0 is not ready" + ); +} + +#[cfg(test)] +pub(crate) fn readiness_test_recorder() -> ( + metrics_exporter_prometheus::PrometheusRecorder, + metrics_exporter_prometheus::PrometheusHandle, +) { + let recorder = configured_prometheus_builder(300).build_recorder(); + let handle = recorder.handle(); + (recorder, handle) } /// Axum middleware that records CAKE framework HTTP metrics. @@ -205,3 +326,139 @@ pub async fn track_metrics(req: Request, next: Next) -> Response { response } +#[cfg(test)] +mod contract_tests { + use std::collections::BTreeSet; + + const OUTCOMES: [&str; 4] = ["success", "timeout", "error", "cancelled"]; + + fn label_keys(line: &str) -> BTreeSet<&str> { + line.split_once('{') + .and_then(|(_, rest)| rest.split_once('}')) + .map(|(labels, _)| { + labels + .split(',') + .filter_map(|label| label.split_once('=').map(|(key, _)| key)) + .collect() + }) + .unwrap_or_default() + } + + #[test] + fn production_builder_exports_frozen_db_pool_contract_and_187_series_budget() { + let (recorder, handle) = super::readiness_test_recorder(); + metrics::with_local_recorder(&recorder, || { + super::describe_db_pool_metrics(); + for (pool_role, operation) in buzz_db::DB_POOL_ACQUIRE_VALID_PAIRS { + metrics::histogram!( + "buzz_db_pool_acquire_duration_seconds", + "pool_role" => pool_role, + "operation" => operation, + ) + .record(0.02); + metrics::gauge!( + "buzz_db_pool_waiters", + "pool_role" => pool_role, + "operation" => operation, + ) + .set(0.0); + for outcome in OUTCOMES { + metrics::counter!( + "buzz_db_pool_acquire_attempts_total", + "pool_role" => pool_role, + "operation" => operation, + "outcome" => outcome, + ) + .increment(1); + } + } + }); + + let scrape = handle.render(); + assert!(scrape.contains("# TYPE buzz_db_pool_acquire_duration_seconds histogram")); + assert!(scrape.contains("# TYPE buzz_db_pool_acquire_attempts_total counter")); + assert!(scrape.contains("# TYPE buzz_db_pool_waiters gauge")); + assert!(scrape.contains("# HELP buzz_db_pool_acquire_duration_seconds Database pool checkout duration by valid pool role and operation")); + assert!(scrape.contains("# HELP buzz_db_pool_acquire_attempts_total Database pool checkout terminals by valid pool role, operation, and outcome")); + assert!(scrape.contains("# HELP buzz_db_pool_waiters Current tracked-operation database pool checkout attempts in progress by valid pool role and operation")); + assert_eq!(super::DB_POOL_ACQUIRE_DURATION_UNIT, metrics::Unit::Seconds); + let readiness_buckets = scrape + .lines() + .filter(|line| { + line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket{") + && line.contains("pool_role=\"writer\"") + && line.contains("operation=\"readiness\"") + }) + .map(|line| { + line.split(",le=\"") + .nth(1) + .and_then(|rest| rest.split_once('"').map(|(bucket, _)| bucket)) + .expect("duration bucket carries le label") + }) + .collect::>(); + assert_eq!( + readiness_buckets, + ["0.001", "0.005", "0.01", "0.025", "0.05", "0.15", "0.5", "1", "3", "+Inf",], + "duration bucket contract drifted:\n{scrape}" + ); + + let raw_series = scrape + .lines() + .filter(|line| { + line.starts_with("buzz_db_pool_acquire_duration_seconds") + || line.starts_with("buzz_db_pool_acquire_attempts_total") + || line.starts_with("buzz_db_pool_waiters{") + }) + .collect::>(); + assert_eq!( + raw_series.len(), + buzz_db::DB_POOL_ACQUIRE_RAW_SERIES_PER_POD, + "unexpected raw scrape:\n{scrape}" + ); + + for line in raw_series { + let keys = label_keys(line); + if line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket") { + assert_eq!(keys, BTreeSet::from(["le", "operation", "pool_role"])); + } else if line.starts_with("buzz_db_pool_acquire_duration_seconds") { + assert_eq!(keys, BTreeSet::from(["operation", "pool_role"])); + } else if line.starts_with("buzz_db_pool_acquire_attempts_total") { + assert_eq!(keys, BTreeSet::from(["operation", "outcome", "pool_role"])); + } else { + assert_eq!(keys, BTreeSet::from(["operation", "pool_role"])); + } + assert!(!line.contains("operation=\"other\"")); + assert!(!line.contains("result=")); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn occupied_listener_is_classified_as_bind() { + let listener = std::net::TcpListener::bind(("0.0.0.0", 0)).expect("bind occupied port"); + let port = listener.local_addr().expect("occupied address").port(); + let error = try_install(port, 300).expect_err("occupied listener must fail"); + assert_eq!(error.failure(), MetricsInstallFailure::Bind); + } + + #[tokio::test] + async fn recorder_conflict_is_typed_in_an_isolated_process() { + const CHILD_ENV: &str = "BUZZ_TEST_METRICS_RECORDER_CONFLICT"; + if std::env::var_os(CHILD_ENV).is_some() { + let recorder = configured_prometheus_builder(300).build_recorder(); + metrics::set_global_recorder(recorder).expect("install first recorder"); + let error = try_install(0, 300).expect_err("second recorder must fail"); + assert_eq!(error.failure(), MetricsInstallFailure::RecorderConflict); + return; + } + + crate::test_support::run_exact_test_child( + "metrics::tests::recorder_conflict_is_typed_in_an_isolated_process", + CHILD_ENV, + ); + } +} diff --git a/crates/buzz-relay/src/readiness.rs b/crates/buzz-relay/src/readiness.rs new file mode 100644 index 000000000..79a1a9857 --- /dev/null +++ b/crates/buzz-relay/src/readiness.rs @@ -0,0 +1,855 @@ +//! Readiness dependency evaluation and ordered metrics publication. +//! +//! [`ReadinessCoordinator`] is process-owned. Its mutex is the linearization +//! point shared by health-probe commits and terminal shutdown, so an older +//! evaluation can never overwrite newer gauges or publish ready after shutdown. + +use std::future::Future; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::Duration; + +use buzz_db::{Db, DbError, DbReadinessOutcome}; +use tokio::time::Instant; + +const READINESS_TIMEOUT: Duration = Duration::from_secs(2); + +/// Closed label set exported by `buzz_readiness_checks_total{reason}`. +#[cfg(test)] +pub(crate) const READINESS_REASON_LABELS: [&str; 12] = [ + "ready", + "shutting_down", + "postgres_pool_timeout", + "postgres_pool_error", + "postgres_query_timeout", + "postgres_query_error", + "redis_pool_timeout", + "redis_pool_error", + "deletion_catalog_timeout", + "deletion_catalog_error", + "overall_timeout", + "multiple_dependencies_failed", +]; + +/// Maximum raw Prometheus series emitted by readiness for one pod. +/// +/// - 12 overall reasons +/// - 11 valid dependency/outcome pairs (Postgres 5, Redis 3, catalog 3) +/// - 4 histograms x (15 configured buckets + `+Inf` + count + sum) = 72 +/// - 4 current-state gauges +#[cfg(test)] +pub(crate) const READINESS_RAW_SERIES_PER_POD: usize = 12 + 11 + (4 * 18) + 4; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PostgresOutcome { + Success, + PoolTimeout, + PoolError, + QueryTimeout, + QueryError, +} + +impl PostgresOutcome { + fn label(self) -> &'static str { + match self { + Self::Success => "success", + Self::PoolTimeout => "pool_timeout", + Self::PoolError => "pool_error", + Self::QueryTimeout => "operation_timeout", + Self::QueryError => "operation_error", + } + } + + fn is_success(self) -> bool { + self == Self::Success + } + + fn is_timeout(self) -> bool { + matches!(self, Self::PoolTimeout | Self::QueryTimeout) + } +} + +impl From for PostgresOutcome { + fn from(outcome: DbReadinessOutcome) -> Self { + match outcome { + DbReadinessOutcome::Success => Self::Success, + DbReadinessOutcome::PoolTimeout => Self::PoolTimeout, + DbReadinessOutcome::PoolError => Self::PoolError, + DbReadinessOutcome::QueryTimeout => Self::QueryTimeout, + DbReadinessOutcome::QueryError => Self::QueryError, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RedisOutcome { + Success, + PoolTimeout, + PoolError, +} + +impl RedisOutcome { + fn label(self) -> &'static str { + match self { + Self::Success => "success", + Self::PoolTimeout => "pool_timeout", + Self::PoolError => "pool_error", + } + } + + fn is_success(self) -> bool { + self == Self::Success + } + + fn is_timeout(self) -> bool { + self == Self::PoolTimeout + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DeletionCatalogOutcome { + Success, + OperationTimeout, + OperationError, +} + +impl DeletionCatalogOutcome { + fn label(self) -> &'static str { + match self { + Self::Success => "success", + Self::OperationTimeout => "operation_timeout", + Self::OperationError => "operation_error", + } + } + + fn is_success(self) -> bool { + self == Self::Success + } + + fn is_timeout(self) -> bool { + self == Self::OperationTimeout + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ReadinessReason { + Ready, + ShuttingDown, + PostgresPoolTimeout, + PostgresPoolError, + PostgresQueryTimeout, + PostgresQueryError, + RedisPoolTimeout, + RedisPoolError, + DeletionCatalogTimeout, + DeletionCatalogError, + OverallTimeout, + MultipleDependenciesFailed, +} + +impl ReadinessReason { + pub(crate) fn label(self) -> &'static str { + match self { + Self::Ready => "ready", + Self::ShuttingDown => "shutting_down", + Self::PostgresPoolTimeout => "postgres_pool_timeout", + Self::PostgresPoolError => "postgres_pool_error", + Self::PostgresQueryTimeout => "postgres_query_timeout", + Self::PostgresQueryError => "postgres_query_error", + Self::RedisPoolTimeout => "redis_pool_timeout", + Self::RedisPoolError => "redis_pool_error", + Self::DeletionCatalogTimeout => "deletion_catalog_timeout", + Self::DeletionCatalogError => "deletion_catalog_error", + Self::OverallTimeout => "overall_timeout", + Self::MultipleDependenciesFailed => "multiple_dependencies_failed", + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct TimedOutcome { + outcome: O, + duration: Duration, +} + +impl TimedOutcome { + #[cfg(test)] + pub(crate) fn new(outcome: O, duration: Duration) -> Self { + Self { outcome, duration } + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ReadinessEvaluation { + postgres: Option>, + redis: Option>, + deletion_catalog: Option>, + pub(crate) reason: ReadinessReason, + total_duration: Duration, +} + +impl ReadinessEvaluation { + pub(crate) fn shutting_down() -> Self { + Self { + postgres: None, + redis: None, + deletion_catalog: None, + reason: ReadinessReason::ShuttingDown, + total_duration: Duration::ZERO, + } + } + + #[cfg(test)] + pub(crate) fn from_results( + postgres: TimedOutcome, + redis: TimedOutcome, + deletion_catalog: TimedOutcome, + total_duration: Duration, + ) -> Self { + Self::for_dependencies(postgres, redis, deletion_catalog, total_duration) + } + + fn for_dependencies( + postgres: TimedOutcome, + redis: TimedOutcome, + deletion_catalog: TimedOutcome, + total_duration: Duration, + ) -> Self { + let reason = final_reason(postgres.outcome, redis.outcome, deletion_catalog.outcome); + Self { + postgres: Some(postgres), + redis: Some(redis), + deletion_catalog: Some(deletion_catalog), + reason, + total_duration, + } + } + + pub(crate) fn is_ready(self) -> bool { + self.reason == ReadinessReason::Ready + } + + pub(crate) fn postgres_ready(self) -> bool { + self.postgres + .is_some_and(|result| result.outcome.is_success()) + } + + pub(crate) fn redis_ready(self) -> bool { + self.redis.is_some_and(|result| result.outcome.is_success()) + } + + pub(crate) fn deletion_catalog_ready(self) -> bool { + self.deletion_catalog + .is_some_and(|result| result.outcome.is_success()) + } + + fn dependencies_ran(self) -> bool { + self.postgres.is_some() || self.redis.is_some() || self.deletion_catalog.is_some() + } +} + +fn final_reason( + postgres: PostgresOutcome, + redis: RedisOutcome, + deletion_catalog: DeletionCatalogOutcome, +) -> ReadinessReason { + let failure_count = usize::from(!postgres.is_success()) + + usize::from(!redis.is_success()) + + usize::from(!deletion_catalog.is_success()); + + if failure_count == 0 { + return ReadinessReason::Ready; + } + if failure_count > 1 { + let all_failures_are_timeouts = (postgres.is_success() || postgres.is_timeout()) + && (redis.is_success() || redis.is_timeout()) + && (deletion_catalog.is_success() || deletion_catalog.is_timeout()); + return if all_failures_are_timeouts { + ReadinessReason::OverallTimeout + } else { + ReadinessReason::MultipleDependenciesFailed + }; + } + + match postgres { + PostgresOutcome::PoolTimeout => ReadinessReason::PostgresPoolTimeout, + PostgresOutcome::PoolError => ReadinessReason::PostgresPoolError, + PostgresOutcome::QueryTimeout => ReadinessReason::PostgresQueryTimeout, + PostgresOutcome::QueryError => ReadinessReason::PostgresQueryError, + PostgresOutcome::Success => match redis { + RedisOutcome::PoolTimeout => ReadinessReason::RedisPoolTimeout, + RedisOutcome::PoolError => ReadinessReason::RedisPoolError, + RedisOutcome::Success => match deletion_catalog { + DeletionCatalogOutcome::OperationTimeout => ReadinessReason::DeletionCatalogTimeout, + DeletionCatalogOutcome::OperationError => ReadinessReason::DeletionCatalogError, + DeletionCatalogOutcome::Success => ReadinessReason::Ready, + }, + }, + } +} + +async fn timed(future: F) -> TimedOutcome +where + F: Future, +{ + let started_at = Instant::now(); + let outcome = future.await; + TimedOutcome { + outcome, + duration: started_at.elapsed(), + } +} + +async fn evaluate_dependencies( + postgres: P, + redis: R, + deletion_catalog: D, +) -> ReadinessEvaluation +where + P: Future, + R: Future, + D: Future, +{ + let started_at = Instant::now(); + let (postgres, redis, deletion_catalog) = + tokio::join!(timed(postgres), timed(redis), timed(deletion_catalog),); + ReadinessEvaluation::for_dependencies(postgres, redis, deletion_catalog, started_at.elapsed()) +} + +async fn redis_check(pool: &deadpool_redis::Pool, deadline: Instant) -> RedisOutcome { + match tokio::time::timeout_at(deadline, pool.get()).await { + Err(_) => RedisOutcome::PoolTimeout, + Ok(Err(error)) => { + tracing::debug!(error = %error, "Redis readiness pool acquisition failed"); + RedisOutcome::PoolError + } + Ok(Ok(_connection)) => RedisOutcome::Success, + } +} + +async fn deletion_catalog_check(db: &Db, deadline: Instant) -> DeletionCatalogOutcome { + classify_deletion_catalog_result( + db.validate_deletion_serving_catalog_for_readiness(deadline) + .await, + ) +} + +fn classify_deletion_catalog_result(result: buzz_db::Result<()>) -> DeletionCatalogOutcome { + match result { + Err(DbError::Sqlx(sqlx::Error::PoolTimedOut)) => DeletionCatalogOutcome::OperationTimeout, + Err(error) => { + tracing::debug!(error = %error, "Deletion catalog readiness validation failed"); + DeletionCatalogOutcome::OperationError + } + Ok(()) => DeletionCatalogOutcome::Success, + } +} + +#[async_trait::async_trait] +pub(crate) trait ReadinessEvaluator: Send + Sync { + async fn evaluate(&self, db: &Db, redis_pool: &deadpool_redis::Pool) -> ReadinessEvaluation; +} + +struct ProductionReadinessEvaluator; + +#[async_trait::async_trait] +impl ReadinessEvaluator for ProductionReadinessEvaluator { + async fn evaluate(&self, db: &Db, redis_pool: &deadpool_redis::Pool) -> ReadinessEvaluation { + let deadline = Instant::now() + READINESS_TIMEOUT; + evaluate_dependencies( + async { db.readiness_check(deadline).await.into() }, + redis_check(redis_pool, deadline), + deletion_catalog_check(db, deadline), + ) + .await + } +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ProbeTicket { + generation: u64, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) enum ProbeStart { + Evaluate(ProbeTicket), + ShuttingDown, +} + +#[derive(Debug, Default)] +struct PublicationState { + next_generation: u64, + latest_published_generation: u64, + shutdown_generation: Option, +} + +/// Serializes readiness result publication with terminal process shutdown. +pub(crate) struct ReadinessCoordinator { + state: Mutex, + evaluator: Arc, +} + +impl Default for ReadinessCoordinator { + fn default() -> Self { + Self { + state: Mutex::new(PublicationState::default()), + evaluator: Arc::new(ProductionReadinessEvaluator), + } + } +} + +impl ReadinessCoordinator { + #[cfg(test)] + pub(crate) fn with_evaluator(evaluator: Arc) -> Self { + Self { + state: Mutex::new(PublicationState::default()), + evaluator, + } + } + + fn lock_state(&self) -> MutexGuard<'_, PublicationState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(crate) async fn evaluate( + &self, + db: &Db, + redis_pool: &deadpool_redis::Pool, + ) -> ReadinessEvaluation { + self.evaluator.evaluate(db, redis_pool).await + } + + /// Allocates a health-probe generation or records a truthful shutdown fast path. + pub(crate) fn begin_probe(&self) -> ProbeStart { + let mut state = self.lock_state(); + if state.shutdown_generation.is_some() { + let evaluation = ReadinessEvaluation::shutting_down(); + record_attempt_metrics(&evaluation, ReadinessReason::ShuttingDown); + record_overall_state(false); + return ProbeStart::ShuttingDown; + } + + state.next_generation = state.next_generation.saturating_add(1); + ProbeStart::Evaluate(ProbeTicket { + generation: state.next_generation, + }) + } + + /// Commits one completed health probe through the shared publication fence. + pub(crate) fn finish_probe( + &self, + ticket: ProbeTicket, + evaluation: ReadinessEvaluation, + ) -> ReadinessEvaluation { + let mut state = self.lock_state(); + if state.shutdown_generation.is_some() { + record_attempt_metrics(&evaluation, ReadinessReason::ShuttingDown); + return ReadinessEvaluation::shutting_down(); + } + + record_attempt_metrics(&evaluation, evaluation.reason); + if ticket.generation > state.latest_published_generation { + record_current_state(&evaluation); + state.latest_published_generation = ticket.generation; + } + evaluation + } + + /// Returns whether a compatibility/public readiness evaluation may start. + pub(crate) fn public_evaluation_allowed(&self) -> bool { + self.lock_state().shutdown_generation.is_none() + } + + /// Makes shutdown dominate a public request that was already in flight. + pub(crate) fn finish_public_evaluation( + &self, + evaluation: ReadinessEvaluation, + ) -> ReadinessEvaluation { + if self.lock_state().shutdown_generation.is_some() { + ReadinessEvaluation::shutting_down() + } else { + evaluation + } + } + + /// Commits terminal shutdown and immediately publishes overall not-ready. + pub(crate) fn begin_shutdown(&self) { + let mut state = self.lock_state(); + if state.shutdown_generation.is_none() { + let generation = state.next_generation.saturating_add(1); + state.shutdown_generation = Some(generation); + record_overall_state(false); + } + } +} + +fn record_attempt_metrics(evaluation: &ReadinessEvaluation, reason: ReadinessReason) { + metrics::counter!( + "buzz_readiness_checks_total", + "reason" => reason.label(), + ) + .increment(1); + + if !evaluation.dependencies_ran() { + return; + } + + metrics::histogram!( + "buzz_readiness_check_duration_seconds", + "check" => "overall", + ) + .record(evaluation.total_duration.as_secs_f64()); + + if let Some(result) = evaluation.postgres { + record_dependency_attempt("postgres", result.outcome.label(), result.duration); + } + if let Some(result) = evaluation.redis { + record_dependency_attempt("redis", result.outcome.label(), result.duration); + } + if let Some(result) = evaluation.deletion_catalog { + record_dependency_attempt("deletion_catalog", result.outcome.label(), result.duration); + } +} + +fn record_dependency_attempt(dependency: &'static str, outcome: &'static str, duration: Duration) { + metrics::counter!( + "buzz_readiness_dependency_checks_total", + "dependency" => dependency, + "outcome" => outcome, + ) + .increment(1); + metrics::histogram!( + "buzz_readiness_check_duration_seconds", + "check" => dependency, + ) + .record(duration.as_secs_f64()); +} + +fn record_current_state(evaluation: &ReadinessEvaluation) { + record_overall_state(evaluation.is_ready()); + if let Some(result) = evaluation.postgres { + record_dependency_state("postgres", result.outcome.is_success()); + } + if let Some(result) = evaluation.redis { + record_dependency_state("redis", result.outcome.is_success()); + } + if let Some(result) = evaluation.deletion_catalog { + record_dependency_state("deletion_catalog", result.outcome.is_success()); + } +} + +fn record_overall_state(ready: bool) { + metrics::gauge!("buzz_readiness_state", "check" => "overall").set(if ready { + 1.0 + } else { + 0.0 + }); +} + +fn record_dependency_state(dependency: &'static str, ready: bool) { + metrics::gauge!("buzz_readiness_state", "check" => dependency).set(if ready { + 1.0 + } else { + 0.0 + }); +} + +#[cfg(test)] +mod tests { + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + use metrics_util::CompositeKey; + + use super::*; + + fn ready_evaluation() -> ReadinessEvaluation { + ReadinessEvaluation::from_results( + TimedOutcome::new(PostgresOutcome::Success, Duration::from_millis(35)), + TimedOutcome::new(RedisOutcome::Success, Duration::from_millis(10)), + TimedOutcome::new(DeletionCatalogOutcome::Success, Duration::from_millis(20)), + Duration::from_millis(35), + ) + } + + fn redis_failure_evaluation() -> ReadinessEvaluation { + ReadinessEvaluation::from_results( + TimedOutcome::new(PostgresOutcome::Success, Duration::from_millis(35)), + TimedOutcome::new(RedisOutcome::PoolTimeout, Duration::from_secs(2)), + TimedOutcome::new(DeletionCatalogOutcome::Success, Duration::from_millis(20)), + Duration::from_secs(2), + ) + } + + fn exact_metric<'a>( + snapshot: &'a [( + CompositeKey, + Option, + Option, + DebugValue, + )], + name: &str, + labels: &[(&str, &str)], + ) -> Option<&'a DebugValue> { + snapshot.iter().find_map(|(key, _, _, value)| { + let actual = key + .key() + .labels() + .map(|label| (label.key(), label.value())) + .collect::>(); + (key.key().name() == name + && actual.len() == labels.len() + && labels.iter().all(|expected| actual.contains(expected))) + .then_some(value) + }) + } + + fn gauge_value( + snapshot: &[( + CompositeKey, + Option, + Option, + DebugValue, + )], + check: &str, + ) -> f64 { + let value = exact_metric(snapshot, "buzz_readiness_state", &[("check", check)]) + .expect("readiness gauge"); + let DebugValue::Gauge(value) = value else { + panic!("readiness state must be a gauge"); + }; + value.into_inner() + } + + #[tokio::test(start_paused = true)] + async fn evaluation_preserves_a_completed_check_when_another_times_out() { + let evaluation = evaluate_dependencies( + async { + tokio::time::sleep(Duration::from_millis(35)).await; + PostgresOutcome::Success + }, + async { + tokio::time::sleep(Duration::from_secs(2)).await; + RedisOutcome::PoolTimeout + }, + async { + tokio::time::sleep(Duration::from_millis(10)).await; + DeletionCatalogOutcome::Success + }, + ) + .await; + + assert_eq!(evaluation.reason, ReadinessReason::RedisPoolTimeout); + assert_eq!( + evaluation.postgres.map(|result| result.duration), + Some(Duration::from_millis(35)) + ); + assert_eq!( + evaluation.redis.map(|result| result.duration), + Some(Duration::from_secs(2)) + ); + } + + #[test] + fn simultaneous_dependency_timeouts_are_an_overall_timeout() { + assert_eq!( + final_reason( + PostgresOutcome::PoolTimeout, + RedisOutcome::PoolTimeout, + DeletionCatalogOutcome::Success, + ), + ReadinessReason::OverallTimeout + ); + } + + #[test] + fn dependency_types_expose_only_valid_outcome_pairs() { + assert_eq!( + [ + PostgresOutcome::Success, + PostgresOutcome::PoolTimeout, + PostgresOutcome::PoolError, + PostgresOutcome::QueryTimeout, + PostgresOutcome::QueryError, + ] + .map(PostgresOutcome::label), + [ + "success", + "pool_timeout", + "pool_error", + "operation_timeout", + "operation_error", + ] + ); + assert_eq!( + [ + RedisOutcome::Success, + RedisOutcome::PoolTimeout, + RedisOutcome::PoolError, + ] + .map(RedisOutcome::label), + ["success", "pool_timeout", "pool_error"] + ); + assert_eq!( + [ + DeletionCatalogOutcome::Success, + DeletionCatalogOutcome::OperationTimeout, + DeletionCatalogOutcome::OperationError, + ] + .map(DeletionCatalogOutcome::label), + ["success", "operation_timeout", "operation_error"] + ); + assert_eq!(READINESS_RAW_SERIES_PER_POD, 99); + } + + #[test] + fn deletion_catalog_deadline_is_a_timeout_not_an_operation_error() { + assert_eq!( + classify_deletion_catalog_result(Err(DbError::Sqlx(sqlx::Error::PoolTimedOut))), + DeletionCatalogOutcome::OperationTimeout + ); + assert_eq!( + classify_deletion_catalog_result(Err(DbError::InvalidData("catalog".into()))), + DeletionCatalogOutcome::OperationError + ); + } + + #[test] + fn slow_older_failure_cannot_overwrite_newer_success_gauges() { + let coordinator = ReadinessCoordinator::default(); + let ProbeStart::Evaluate(slow_a) = coordinator.begin_probe() else { + panic!("serving probe A"); + }; + let ProbeStart::Evaluate(fast_b) = coordinator.begin_probe() else { + panic!("serving probe B"); + }; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + coordinator.finish_probe(fast_b, ready_evaluation()); + coordinator.finish_probe(slow_a, redis_failure_evaluation()); + }); + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!(gauge_value(&snapshot, "overall"), 1.0); + assert_eq!(gauge_value(&snapshot, "redis"), 1.0); + assert!(matches!( + exact_metric( + &snapshot, + "buzz_readiness_checks_total", + &[("reason", "ready")] + ), + Some(DebugValue::Counter(1)) + )); + assert!(matches!( + exact_metric( + &snapshot, + "buzz_readiness_checks_total", + &[("reason", "redis_pool_timeout")] + ), + Some(DebugValue::Counter(1)) + )); + } + + #[test] + fn slow_older_success_cannot_overwrite_newer_failure_gauges() { + let coordinator = ReadinessCoordinator::default(); + let ProbeStart::Evaluate(slow_a) = coordinator.begin_probe() else { + panic!("serving probe A"); + }; + let ProbeStart::Evaluate(fast_b) = coordinator.begin_probe() else { + panic!("serving probe B"); + }; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + coordinator.finish_probe(fast_b, redis_failure_evaluation()); + coordinator.finish_probe(slow_a, ready_evaluation()); + }); + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!(gauge_value(&snapshot, "overall"), 0.0); + assert_eq!(gauge_value(&snapshot, "postgres"), 1.0); + assert_eq!(gauge_value(&snapshot, "redis"), 0.0); + assert_eq!(gauge_value(&snapshot, "deletion_catalog"), 1.0); + } + + #[test] + fn shutdown_fast_path_preserves_dependency_state_and_histograms() { + let coordinator = ReadinessCoordinator::default(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + let ProbeStart::Evaluate(ticket) = coordinator.begin_probe() else { + panic!("initial serving probe"); + }; + coordinator.finish_probe(ticket, ready_evaluation()); + coordinator.begin_shutdown(); + assert!(matches!( + coordinator.begin_probe(), + ProbeStart::ShuttingDown + )); + }); + let after = snapshotter.snapshot().into_vec(); + + for dependency in ["postgres", "redis", "deletion_catalog"] { + assert_eq!( + gauge_value(&after, dependency), + 1.0, + "shutdown must not fabricate {dependency} state" + ); + } + for check in ["overall", "postgres", "redis", "deletion_catalog"] { + assert!( + matches!( + exact_metric( + &after, + "buzz_readiness_check_duration_seconds", + &[("check", check)] + ), + Some(DebugValue::Histogram(values)) if values.len() == 1 + ), + "shutdown fast path must not add a {check} duration" + ); + } + assert_eq!(gauge_value(&after, "overall"), 0.0); + assert!(matches!( + exact_metric( + &after, + "buzz_readiness_checks_total", + &[("reason", "shutting_down")] + ), + Some(DebugValue::Counter(1)) + )); + } + + #[test] + fn shutdown_dominates_an_in_flight_success_without_resurrecting_gauges() { + let coordinator = ReadinessCoordinator::default(); + let ProbeStart::Evaluate(ticket) = coordinator.begin_probe() else { + panic!("serving probe"); + }; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + let response = metrics::with_local_recorder(&recorder, || { + coordinator.begin_shutdown(); + coordinator.finish_probe(ticket, ready_evaluation()) + }); + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!(response.reason, ReadinessReason::ShuttingDown); + assert_eq!(gauge_value(&snapshot, "overall"), 0.0); + assert!( + exact_metric(&snapshot, "buzz_readiness_state", &[("check", "postgres")]).is_none() + ); + assert!(matches!( + exact_metric( + &snapshot, + "buzz_readiness_dependency_checks_total", + &[("dependency", "postgres"), ("outcome", "success")] + ), + Some(DebugValue::Counter(1)) + )); + } +} diff --git a/crates/buzz-relay/src/rejection.rs b/crates/buzz-relay/src/rejection.rs new file mode 100644 index 000000000..84d6cd1e2 --- /dev/null +++ b/crates/buzz-relay/src/rejection.rs @@ -0,0 +1,415 @@ +//! How a rejected client frame is addressed back to the client. +//! +//! NIP-01 gives every request type its own acknowledgement channel, and a +//! rejection is only actionable if it travels on the same one: a REQ or COUNT +//! refusal settles on `CLOSED`, an EVENT on `OK`. Rejecting an EVENT with a bare +//! `NOTICE` leaves a client that tracks pending publishes by event id with +//! nothing to key on, so the send cannot fail — it can only time out. + +use crate::admission::AdmissionError; +use crate::connection::{AuthState, ConnectionState}; +use crate::protocol::{ClientMessage, RelayMessage}; +use crate::state::AppState; +use buzz_auth::LimitType; + +/// What a rejected client frame is correlated back to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RejectionTarget<'a> { + /// A REQ or COUNT names the query it opened. + Subscription(&'a str), + /// An EVENT names the event it submitted. + Event(nostr::EventId), + /// No per-request correlation exists — connection-scoped notice. + Connection, +} + +/// Picks the acknowledgement channel a rejection of `msg` must travel on. +pub(crate) fn rejection_target_for(msg: &ClientMessage) -> RejectionTarget<'_> { + match msg { + ClientMessage::Req { sub_id, .. } | ClientMessage::Count { sub_id, .. } => { + RejectionTarget::Subscription(sub_id.as_str()) + } + ClientMessage::Event(event) => RejectionTarget::Event(event.id), + _ => RejectionTarget::Connection, + } +} + +/// Renders `reason` as the rejection frame `target`'s acknowledgement channel +/// expects. +pub(crate) fn request_rejection_message(target: RejectionTarget<'_>, reason: &str) -> String { + match target { + RejectionTarget::Subscription(sub_id) => RelayMessage::closed(sub_id, reason), + RejectionTarget::Event(event_id) => RelayMessage::ok(&event_id.to_hex(), false, reason), + RejectionTarget::Connection => RelayMessage::notice(reason), + } +} + +/// Sends `reason` on `target`'s acknowledgement channel, and — for an EVENT — +/// repeats it as a `NOTICE`. +/// +/// The `OK` is what a current client settles its pending publish from, and it +/// goes first so nothing waits on the compatibility frame. The `NOTICE` is +/// there for clients shipped before Colony started arming back-pressure from +/// the `OK` reason: the relay outlives the app versions talking to it, and a +/// Desktop or Canary install from before that change arms its rate-limit gate +/// from `NOTICE` alone. Drop the extra frame once every channel has been on an +/// OK-arming client for a release. +pub(crate) fn send_request_rejection( + conn: &ConnectionState, + target: RejectionTarget<'_>, + reason: &str, +) { + conn.send(request_rejection_message(target, reason)); + if matches!(target, RejectionTarget::Event(_)) { + conn.send(RelayMessage::notice(reason)); + } +} + +/// Applies the WebSocket admission quotas to `msg`, returning whether it may be +/// handled. A rejection is addressed to the frame's own acknowledgement channel. +pub(crate) async fn enforce_ws_admission( + msg: &ClientMessage, + conn: &ConnectionState, + state: &AppState, +) -> bool { + let is_event = matches!(msg, ClientMessage::Event(_)); + if !is_event && !matches!(msg, ClientMessage::Req { .. } | ClientMessage::Count { .. }) { + return true; + } + + let (pubkey, is_agent) = { + let auth = conn.auth_state.read().await; + match &*auth { + AuthState::Authenticated(ctx) => (ctx.pubkey, ctx.agent_owner_pubkey.is_some()), + _ => return true, + } + }; + + let limits = &state.auth.config().rate_limits; + let (ws_window_secs, ws_limit) = + crate::admission::ws_admission_budget(limits.human_ws_events_per_sec); + let ws_result = crate::admission::check_principal( + state.admission_rate_limiter.as_ref(), + &conn.tenant, + &pubkey, + LimitType::WsEvents, + ws_window_secs, + ws_limit, + ) + .await; + if !send_admission_result(conn, ws_result, msg) { + return false; + } + + if is_event { + let message_limit = if is_agent { + limits.agent_standard_messages_per_min + } else { + limits.human_messages_per_min + }; + let message_result = crate::admission::check_principal( + state.admission_rate_limiter.as_ref(), + &conn.tenant, + &pubkey, + LimitType::Messages, + 60, + message_limit, + ) + .await; + // The per-minute message quota only applies to EVENTs, and its + // rejection must be as correlatable as the burst quota's. + if !send_admission_result(conn, message_result, msg) { + return false; + } + } + + true +} + +/// Forwards an admission verdict to the client, returning whether the frame was +/// admitted. +/// +/// The rejection target is derived from `msg` here rather than supplied by the +/// caller: every quota check in this module must address its rejection to the +/// rejected frame's own acknowledgement channel, so there is deliberately no way +/// for a call site to name a different one. +fn send_admission_result( + conn: &ConnectionState, + result: Result<(), AdmissionError>, + msg: &ClientMessage, +) -> bool { + let target = rejection_target_for(msg); + match result { + Ok(()) => true, + Err(AdmissionError::Exceeded { reset_in_secs }) => { + metrics::counter!("buzz_admission_rejections_total", "transport" => "websocket", "reason" => "quota").increment(1); + send_request_rejection( + conn, + target, + &format!("rate-limited: quota exceeded; retry in {reset_in_secs}s"), + ); + false + } + Err(AdmissionError::Unavailable) => { + metrics::counter!("buzz_admission_rejections_total", "transport" => "websocket", "reason" => "unavailable").increment(1); + send_request_rejection(conn, target, "rate-limited: shared admission unavailable"); + false + } + } +} + +#[cfg(test)] +mod tests { + //! A rejected frame must be answerable on the acknowledgement channel the + //! client is actually waiting on. + //! + //! History: an over-quota EVENT used to be rejected with a bare + //! `["NOTICE", reason]`. A NOTICE carries no event id, and desktop/mobile + //! settle pending publishes only from an `OK` keyed by event id, so the + //! rejection was unaddressable: the send could not fail, it could only time + //! out (25s in Desktop, `PUBLISH_TIMEOUT_MS`) and surface as a message stuck + //! on "Sending…". Startup quota exhaustion made it routine in the first + //! seconds after launch. + //! + //! These tests drive the production rejection path — a real parsed + //! `ClientMessage` through `enforce_ws_admission` and + //! `send_admission_result` — and assert on the frame that reaches the + //! connection's outbound channel. + + use std::sync::Arc; + + use axum::extract::ws::Message as WsMessage; + use nostr::{EventBuilder, Keys, Kind}; + use tokio::sync::mpsc; + + use crate::connection::tests::{authenticated_state, read_frame, test_conn_with_auth}; + use crate::connection::AuthState; + + use super::*; + + fn sent_frame(rx: &mut mpsc::Receiver) -> serde_json::Value { + read_frame(rx) + } + + fn test_conn() -> (Arc, mpsc::Receiver) { + test_conn_with_auth(AuthState::Failed) + } + + /// Parses a real EVENT frame exactly as the recv loop does, so the test is + /// coupled to production parsing and not to a hand-built target. + fn parsed_event_message() -> (ClientMessage, String) { + let event = EventBuilder::new(Kind::TextNote, "hello") + .sign_with_keys(&Keys::generate()) + .expect("sign event"); + let event_id = event.id.to_hex(); + let frame = serde_json::json!(["EVENT", event]).to_string(); + (ClientMessage::parse(&frame).expect("parse EVENT"), event_id) + } + + /// A refused EVENT must still carry the legacy `NOTICE` behind its `OK`. + /// + /// Desktop and Canary builds shipped before Colony armed back-pressure from + /// the `OK` reason arm their rate-limit gate from `NOTICE` alone, and the + /// relay outlives the client versions connected to it. Deleting the second + /// frame makes those installs retry straight back into the same quota. + #[test] + fn a_refused_event_keeps_the_notice_behind_the_correlated_ok() { + let (conn, mut rx) = test_conn(); + let (msg, event_id) = parsed_event_message(); + let reason = "rate-limited: quota exceeded; retry in 7s"; + + let admitted = send_admission_result( + &conn, + Err(AdmissionError::Exceeded { reset_in_secs: 7 }), + &msg, + ); + + assert!(!admitted, "an over-quota frame is not admitted"); + assert_eq!( + sent_frame(&mut rx), + serde_json::json!(["OK", event_id, false, reason]), + "the correlated OK goes first so a current client settles its \ + pending publish without waiting on the compatibility frame" + ); + assert_eq!( + sent_frame(&mut rx), + serde_json::json!(["NOTICE", reason]), + "and the NOTICE follows for clients that only arm backoff from it" + ); + } + + /// A refused REQ or COUNT must NOT gain the compatibility NOTICE: `CLOSED` + /// already names the subscription, and a second frame would arm the gate + /// twice for one refusal. + #[test] + fn a_refused_subscription_does_not_gain_a_compatibility_notice() { + let (conn, mut rx) = test_conn(); + let reason = "rate-limited: shared admission unavailable"; + let msg = ClientMessage::parse( + &serde_json::json!(["REQ", "history-1", { "kinds": [1] }]).to_string(), + ) + .expect("parse REQ"); + + assert!(!send_admission_result( + &conn, + Err(AdmissionError::Unavailable), + &msg + )); + + assert_eq!( + sent_frame(&mut rx), + serde_json::json!(["CLOSED", "history-1", reason]) + ); + assert!( + rx.try_recv().is_err(), + "a subscription refusal is one frame: CLOSED already correlates it" + ); + } + + /// The regression: an over-quota EVENT must be rejected with + /// `OK(event_id, false, reason)` so the client can settle the exact pending + /// publish it belongs to. A NOTICE here reintroduces the 25s send stall. + #[test] + fn over_quota_event_is_rejected_with_a_correlated_ok() { + let (conn, mut rx) = test_conn(); + let (msg, event_id) = parsed_event_message(); + + let admitted = send_admission_result( + &conn, + Err(AdmissionError::Exceeded { reset_in_secs: 7 }), + &msg, + ); + + assert!(!admitted, "an over-quota frame is not admitted"); + let frame = sent_frame(&mut rx); + assert_eq!( + frame[0], "OK", + "an EVENT rejection must travel on the OK channel — a NOTICE cannot \ + be correlated to a pending publish, so the send hangs until the \ + client's publish timeout instead of failing" + ); + assert_eq!( + frame[1], event_id, + "the OK must name the rejected event id, which is what the client's \ + pending-publish map is keyed by" + ); + assert_eq!(frame[2], false, "and must be an explicit rejection"); + assert_eq!( + frame[3], "rate-limited: quota exceeded; retry in 7s", + "the retry hint must survive so the client can arm its gate" + ); + } + + /// The same correlation is required when admission is unavailable rather + /// than exceeded — both branches strand a send if they emit a NOTICE. + #[test] + fn event_rejected_for_unavailable_admission_is_also_correlated() { + let (conn, mut rx) = test_conn(); + let (msg, event_id) = parsed_event_message(); + + send_admission_result(&conn, Err(AdmissionError::Unavailable), &msg); + + let frame = sent_frame(&mut rx); + assert_eq!(frame[0], "OK"); + assert_eq!(frame[1], event_id); + assert_eq!(frame[2], false); + } + + /// A REQ still settles on CLOSED, which carries the subscription id. This + /// pins the pre-existing behavior the fix must not disturb. + #[test] + fn over_quota_req_still_closes_the_subscription() { + let (conn, mut rx) = test_conn(); + let raw = serde_json::json!(["REQ", "history-abc", {"kinds": [1]}]).to_string(); + let msg = ClientMessage::parse(&raw).expect("parse REQ"); + + send_admission_result( + &conn, + Err(AdmissionError::Exceeded { reset_in_secs: 7 }), + &msg, + ); + + let frame = sent_frame(&mut rx); + assert_eq!(frame[0], "CLOSED"); + assert_eq!( + frame[1], "history-abc", + "a REQ rejection must name the subscription it rejected" + ); + assert_eq!(frame[2], "rate-limited: quota exceeded; retry in 7s"); + } + + /// NIP-45 uses `CLOSED(query_id, reason)` when a relay refuses a COUNT. + #[test] + fn over_quota_count_closes_the_query() { + let (conn, mut rx) = test_conn(); + let raw = serde_json::json!(["COUNT", "count-abc", {"kinds": [1]}]).to_string(); + let msg = ClientMessage::parse(&raw).expect("parse COUNT"); + + send_admission_result( + &conn, + Err(AdmissionError::Exceeded { reset_in_secs: 7 }), + &msg, + ); + + let frame = sent_frame(&mut rx); + assert_eq!(frame[0], "CLOSED"); + assert_eq!(frame[1], "count-abc"); + assert_eq!(frame[2], "rate-limited: quota exceeded; retry in 7s"); + } + + /// Drives the real entry point `handle_text_message` calls, so the wiring + /// between `enforce_ws_admission` and the target choice is under test and + /// not just the leaf renderer. + /// + /// The state's Redis is deliberately unreachable, which makes admission + /// return `Unavailable` — a production rejection path that needs no live + /// quota burst to reach. + async fn enforce_against_unreachable_admission(raw: &str) -> serde_json::Value { + let state = crate::state::tests::test_state().await; + let (conn, mut rx) = test_conn_with_auth(authenticated_state()); + let msg = ClientMessage::parse(raw).expect("parse client frame"); + + let admitted = enforce_ws_admission(&msg, &conn, &state).await; + assert!(!admitted, "an unadmitted frame must not be handled"); + sent_frame(&mut rx) + } + + #[tokio::test] + async fn enforce_ws_admission_rejects_an_event_on_the_ok_channel() { + let event = EventBuilder::new(Kind::TextNote, "hello") + .sign_with_keys(&Keys::generate()) + .expect("sign event"); + let event_id = event.id.to_hex(); + let raw = serde_json::json!(["EVENT", event]).to_string(); + + let frame = enforce_against_unreachable_admission(&raw).await; + + assert_eq!( + frame[0], "OK", + "the admission gate must reject an EVENT on the channel the client's \ + pending publish is keyed by, or the send can only time out" + ); + assert_eq!(frame[1], event_id); + assert_eq!(frame[2], false); + } + + #[tokio::test] + async fn enforce_ws_admission_rejects_a_count_on_the_closed_channel() { + let raw = serde_json::json!(["COUNT", "count-abc", {"kinds": [1]}]).to_string(); + + let frame = enforce_against_unreachable_admission(&raw).await; + + assert_eq!(frame[0], "CLOSED"); + assert_eq!(frame[1], "count-abc"); + } + + #[tokio::test] + async fn enforce_ws_admission_rejects_a_req_on_the_closed_channel() { + let raw = serde_json::json!(["REQ", "history-abc", {"kinds": [1]}]).to_string(); + + let frame = enforce_against_unreachable_admission(&raw).await; + + assert_eq!(frame[0], "CLOSED"); + assert_eq!(frame[1], "history-abc"); + } +} diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index b53d3be49..03f60966c 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -24,6 +24,7 @@ use crate::audio; use crate::connection::handle_connection; use crate::metrics::track_metrics; use crate::nip11::{nip11_document, relay_info_handler}; +use crate::readiness::{self, ReadinessEvaluation, ReadinessReason}; use crate::state::AppState; /// Build the axum [`Router`] with all relay routes, middleware, and CORS configuration. @@ -67,7 +68,7 @@ pub fn build_router(state: Arc) -> Router { // Health endpoints .route("/health", get(health_handler)) .route("/_liveness", get(liveness_handler)) - .route("/_readiness", get(readiness_handler)) + .route("/_readiness", get(public_readiness_handler)) // Nostr HTTP bridge (NIP-98 auth) .route("/events", post(api::bridge::submit_event)) .route("/query", post(api::bridge::query_events)) @@ -329,7 +330,7 @@ async fn read_spa_index(index: &std::path::Path) -> axum::response::Response { pub fn build_health_router(state: Arc) -> Router { Router::new() .route("/_liveness", get(liveness_handler)) - .route("/_readiness", get(readiness_handler)) + .route("/_readiness", get(kubernetes_readiness_handler)) .route("/_status", get(status_handler)) .route("/_mesh", get(mesh_status_handler)) .with_state(state) @@ -453,11 +454,36 @@ async fn liveness_handler() -> impl IntoResponse { (StatusCode::OK, "ok") } -/// Readiness probe — checks shutdown flag, Postgres, and Redis connectivity. -async fn readiness_handler(State(state): State>) -> impl IntoResponse { - use std::time::Duration; +/// Compatibility endpoint on the public listener. It evaluates dependencies +/// and preserves the existing response contract but never records rollout +/// telemetry. +async fn public_readiness_handler(State(state): State>) -> impl IntoResponse { + if !state.readiness.public_evaluation_allowed() { + return readiness_response(ReadinessEvaluation::shutting_down(), false); + } + + let evaluation = state.readiness.evaluate(&state.db, &state.redis_pool).await; + let evaluation = state.readiness.finish_public_evaluation(evaluation); + readiness_response(evaluation, false) +} + +/// Kubernetes health-listener endpoint. All rollout metrics flow through the +/// process-owned coordinator so shutdown and probe generations are ordered. +async fn kubernetes_readiness_handler(State(state): State>) -> impl IntoResponse { + let readiness::ProbeStart::Evaluate(ticket) = state.readiness.begin_probe() else { + return readiness_response(ReadinessEvaluation::shutting_down(), true); + }; + + let evaluation = state.readiness.evaluate(&state.db, &state.redis_pool).await; + let evaluation = state.readiness.finish_probe(ticket, evaluation); + readiness_response(evaluation, true) +} - if state.shutting_down.load(Ordering::Relaxed) { +fn readiness_response( + evaluation: ReadinessEvaluation, + include_reason: bool, +) -> axum::response::Response { + if evaluation.reason == ReadinessReason::ShuttingDown { return ( StatusCode::SERVICE_UNAVAILABLE, Json(json!({"status": "shutting_down"})), @@ -465,33 +491,23 @@ async fn readiness_handler(State(state): State>) -> impl IntoRespo .into_response(); } - let check = async { - let (pg_ok, redis_ok, deletion_catalog_ok) = tokio::join!( - state.db.ping(), - async { state.redis_pool.get().await.is_ok() }, - async { state.db.validate_deletion_serving_catalog().await.is_ok() }, - ); - (pg_ok, redis_ok, deletion_catalog_ok) - }; + let pg_ok = evaluation.postgres_ready(); + let redis_ok = evaluation.redis_ready(); + let deletion_catalog_ok = evaluation.deletion_catalog_ready(); - let (pg_ok, redis_ok, deletion_catalog_ok) = - tokio::time::timeout(Duration::from_secs(2), check) - .await - .unwrap_or((false, false, false)); - - if pg_ok && redis_ok && deletion_catalog_ok { + if evaluation.is_ready() { (StatusCode::OK, Json(json!({"status": "ready"}))).into_response() } else { - ( - StatusCode::SERVICE_UNAVAILABLE, - Json(json!({ - "status": "not_ready", - "postgres": pg_ok, - "redis": redis_ok, - "deletion_catalog": deletion_catalog_ok - })), - ) - .into_response() + let mut payload = json!({ + "status": "not_ready", + "postgres": pg_ok, + "redis": redis_ok, + "deletion_catalog": deletion_catalog_ok + }); + if include_reason { + payload["reason"] = json!(evaluation.reason.label()); + } + (StatusCode::SERVICE_UNAVAILABLE, Json(payload)).into_response() } } @@ -545,12 +561,17 @@ fn build_cors_layer(cors_origins: &[String]) -> CorsLayer { #[cfg(test)] mod tests { + use std::collections::VecDeque; + use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + use std::sync::{Mutex, PoisonError}; + use std::time::Duration; + use axum::{routing::get, Router}; use futures_util::SinkExt; use opentelemetry::trace::TracerProvider as _; use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider}; use tokio::net::TcpListener; - use tokio::sync::mpsc; + use tokio::sync::{mpsc, Notify}; use tokio_tungstenite::{connect_async, tungstenite::Message}; use tower::ServiceBuilder; use tracing::Instrument as _; @@ -558,6 +579,98 @@ mod tests { use super::*; + struct ScriptedReadinessEvaluator { + evaluations: Mutex>, + } + + impl ScriptedReadinessEvaluator { + fn new(evaluations: impl IntoIterator) -> Self { + Self { + evaluations: Mutex::new(evaluations.into_iter().collect()), + } + } + + fn push(&self, evaluation: ReadinessEvaluation) { + self.evaluations + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push_back(evaluation); + } + } + + #[async_trait::async_trait] + impl readiness::ReadinessEvaluator for ScriptedReadinessEvaluator { + async fn evaluate( + &self, + _db: &buzz_db::Db, + _redis_pool: &deadpool_redis::Pool, + ) -> ReadinessEvaluation { + self.evaluations + .lock() + .unwrap_or_else(PoisonError::into_inner) + .pop_front() + .expect("scripted readiness evaluation") + } + } + + struct BarrierReadinessEvaluator { + calls: AtomicUsize, + first_started: Notify, + release_first: Notify, + first: ReadinessEvaluation, + second: ReadinessEvaluation, + } + + impl BarrierReadinessEvaluator { + fn new(first: ReadinessEvaluation, second: ReadinessEvaluation) -> Self { + Self { + calls: AtomicUsize::new(0), + first_started: Notify::new(), + release_first: Notify::new(), + first, + second, + } + } + } + + #[async_trait::async_trait] + impl readiness::ReadinessEvaluator for BarrierReadinessEvaluator { + async fn evaluate( + &self, + _db: &buzz_db::Db, + _redis_pool: &deadpool_redis::Pool, + ) -> ReadinessEvaluation { + if self.calls.fetch_add(1, AtomicOrdering::SeqCst) == 0 { + self.first_started.notify_waiters(); + self.release_first.notified().await; + self.first + } else { + self.second + } + } + } + + fn readiness_evaluation( + postgres: readiness::PostgresOutcome, + redis: readiness::RedisOutcome, + deletion_catalog: readiness::DeletionCatalogOutcome, + ) -> ReadinessEvaluation { + ReadinessEvaluation::from_results( + readiness::TimedOutcome::new(postgres, Duration::from_millis(35)), + readiness::TimedOutcome::new(redis, Duration::from_millis(20)), + readiness::TimedOutcome::new(deletion_catalog, Duration::from_millis(15)), + Duration::from_millis(35), + ) + } + + fn ready_evaluation() -> ReadinessEvaluation { + readiness_evaluation( + readiness::PostgresOutcome::Success, + readiness::RedisOutcome::Success, + readiness::DeletionCatalogOutcome::Success, + ) + } + #[test] fn invite_landing_path_requires_exactly_one_nonempty_code_segment() { assert!(is_invite_landing_path("/invite/payload.mac")); @@ -597,6 +710,447 @@ mod tests { assert!(!should_serve_spa("/arbitrary", true)); } + async fn readiness_state(evaluator: Arc) -> Arc { + let mut config = crate::config::Config::from_env().expect("default config loads"); + config.require_relay_membership = false; + config.database_url = "postgres://buzz:buzz_dev@127.0.0.1:1/buzz".to_string(); + config.redis_url = "redis://127.0.0.1:1".to_string(); + let pool = sqlx::PgPool::connect_lazy(&config.database_url).expect("lazy pg pool"); + let db = buzz_db::Db::from_pool(pool.clone()); + let redis_pool = deadpool_redis::Config::from_url(&config.redis_url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("redis pool"); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new(&config.redis_url, redis_pool.clone()) + .await + .expect("pubsub manager"), + ); + let audit = buzz_audit::AuditService::new(pool.clone()); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let search = buzz_search::SearchService::new(pool.clone()); + let workflow_engine = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let (mut state, _audit_shutdown) = AppState::new( + config, + db, + redis_pool, + audit, + pubsub, + auth, + search, + workflow_engine, + nostr::Keys::generate(), + media_storage, + ); + state.set_readiness_evaluator(evaluator); + Arc::new(state) + } + + async fn readiness_request(router: Router) -> (StatusCode, serde_json::Value) { + let response = router + .oneshot( + Request::get("/_readiness") + .body(Body::empty()) + .expect("readiness request"), + ) + .await + .expect("readiness response"); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .expect("readiness response body"); + let payload = serde_json::from_slice(&body).expect("readiness JSON"); + (status, payload) + } + + fn readiness_metric_lines(rendered: &str) -> Vec<&str> { + rendered + .lines() + .filter(|line| line.starts_with("buzz_readiness")) + .collect() + } + + fn sorted_readiness_metric_lines(rendered: &str) -> Vec { + let mut lines = readiness_metric_lines(rendered) + .into_iter() + .map(str::to_owned) + .collect::>(); + lines.sort(); + lines + } + + fn metric_value(rendered: &str, exact_prefix: &str) -> f64 { + rendered + .lines() + .find_map(|line| { + line.strip_prefix(exact_prefix) + .and_then(|value| value.strip_prefix(' ')) + .and_then(|value| value.parse().ok()) + }) + .unwrap_or_else(|| panic!("missing metric line: {exact_prefix}")) + } + + #[test] + fn production_readiness_routes_export_the_frozen_health_only_contract() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime"); + let evaluator = Arc::new(ScriptedReadinessEvaluator::new(std::iter::repeat_n( + ready_evaluation(), + 4, + ))); + let (recorder, handle) = crate::metrics::readiness_test_recorder(); + + metrics::with_local_recorder(&recorder, || { + crate::metrics::describe_readiness_metrics(); + runtime.block_on(async { + let state = readiness_state(evaluator.clone()).await; + let public = build_router(state.clone()); + let health = build_health_router(state.clone()); + + for _ in 0..3 { + assert_eq!( + readiness_request(public.clone()).await, + (StatusCode::OK, json!({"status": "ready"})) + ); + } + assert!( + readiness_metric_lines(&handle.render()).is_empty(), + "public compatibility requests must emit no readiness series" + ); + + assert_eq!( + readiness_request(health.clone()).await, + (StatusCode::OK, json!({"status": "ready"})) + ); + let first_scrape = handle.render(); + + assert!(first_scrape.contains("# TYPE buzz_readiness_checks_total counter")); + assert!(first_scrape + .contains("# TYPE buzz_readiness_dependency_checks_total counter")); + assert!(first_scrape + .contains("# TYPE buzz_readiness_check_duration_seconds histogram")); + assert!(first_scrape.contains("# TYPE buzz_readiness_state gauge")); + assert_eq!( + metric_value( + &first_scrape, + "buzz_readiness_checks_total{reason=\"ready\"}" + ), + 1.0 + ); + assert_eq!( + metric_value( + &first_scrape, + "buzz_readiness_dependency_checks_total{dependency=\"postgres\",outcome=\"success\"}" + ), + 1.0 + ); + assert_eq!( + metric_value( + &first_scrape, + "buzz_readiness_state{check=\"overall\"}" + ), + 1.0 + ); + for bucket in ["2", "2.5", "+Inf"] { + assert!(first_scrape.contains(&format!( + "buzz_readiness_check_duration_seconds_bucket{{check=\"overall\",le=\"{bucket}\"}}" + ))); + } + assert!(!first_scrape.contains("result=")); + assert!(!first_scrape + .lines() + .filter(|line| line.starts_with("buzz_readiness_check_duration_seconds")) + .any(|line| line.contains("outcome="))); + + let before_public_failure = sorted_readiness_metric_lines(&first_scrape); + evaluator.push(readiness_evaluation( + readiness::PostgresOutcome::Success, + readiness::RedisOutcome::PoolTimeout, + readiness::DeletionCatalogOutcome::Success, + )); + assert_eq!( + readiness_request(public.clone()).await, + ( + StatusCode::SERVICE_UNAVAILABLE, + json!({ + "status": "not_ready", + "postgres": true, + "redis": false, + "deletion_catalog": true + }) + ) + ); + assert_eq!( + sorted_readiness_metric_lines(&handle.render()), + before_public_failure + ); + + let contract_evaluations = [ + readiness_evaluation( + readiness::PostgresOutcome::PoolTimeout, + readiness::RedisOutcome::Success, + readiness::DeletionCatalogOutcome::Success, + ), + readiness_evaluation( + readiness::PostgresOutcome::PoolError, + readiness::RedisOutcome::Success, + readiness::DeletionCatalogOutcome::Success, + ), + readiness_evaluation( + readiness::PostgresOutcome::QueryTimeout, + readiness::RedisOutcome::Success, + readiness::DeletionCatalogOutcome::Success, + ), + readiness_evaluation( + readiness::PostgresOutcome::QueryError, + readiness::RedisOutcome::Success, + readiness::DeletionCatalogOutcome::Success, + ), + readiness_evaluation( + readiness::PostgresOutcome::Success, + readiness::RedisOutcome::PoolTimeout, + readiness::DeletionCatalogOutcome::Success, + ), + readiness_evaluation( + readiness::PostgresOutcome::Success, + readiness::RedisOutcome::PoolError, + readiness::DeletionCatalogOutcome::Success, + ), + readiness_evaluation( + readiness::PostgresOutcome::Success, + readiness::RedisOutcome::Success, + readiness::DeletionCatalogOutcome::OperationTimeout, + ), + readiness_evaluation( + readiness::PostgresOutcome::Success, + readiness::RedisOutcome::Success, + readiness::DeletionCatalogOutcome::OperationError, + ), + readiness_evaluation( + readiness::PostgresOutcome::PoolTimeout, + readiness::RedisOutcome::PoolTimeout, + readiness::DeletionCatalogOutcome::OperationTimeout, + ), + readiness_evaluation( + readiness::PostgresOutcome::PoolError, + readiness::RedisOutcome::PoolError, + readiness::DeletionCatalogOutcome::Success, + ), + ]; + for evaluation in contract_evaluations { + evaluator.push(evaluation); + let (status, payload) = readiness_request(health.clone()).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(payload["reason"], json!(evaluation.reason.label())); + } + + let before_shutdown = handle.render(); + let histogram_counts_before = ["overall", "postgres", "redis", "deletion_catalog"] + .map(|check| { + metric_value( + &before_shutdown, + &format!( + "buzz_readiness_check_duration_seconds_count{{check=\"{check}\"}}" + ), + ) + }); + state.begin_shutdown(); + assert_eq!( + readiness_request(public).await, + ( + StatusCode::SERVICE_UNAVAILABLE, + json!({"status": "shutting_down"}) + ) + ); + let after_public_shutdown = handle.render(); + assert!(after_public_shutdown + .lines() + .all(|line| !line.contains("reason=\"shutting_down\""))); + + assert_eq!( + readiness_request(health).await, + ( + StatusCode::SERVICE_UNAVAILABLE, + json!({"status": "shutting_down"}) + ) + ); + let final_scrape = handle.render(); + let histogram_counts_after = ["overall", "postgres", "redis", "deletion_catalog"] + .map(|check| { + metric_value( + &final_scrape, + &format!( + "buzz_readiness_check_duration_seconds_count{{check=\"{check}\"}}" + ), + ) + }); + assert_eq!(histogram_counts_after, histogram_counts_before); + assert_eq!( + metric_value( + &final_scrape, + "buzz_readiness_checks_total{reason=\"shutting_down\"}" + ), + 1.0 + ); + assert_eq!( + metric_value( + &final_scrape, + "buzz_readiness_state{check=\"overall\"}" + ), + 0.0 + ); + assert!(!final_scrape.contains("sensitive-sql-or-url")); + + let exported_reasons = final_scrape + .lines() + .filter(|line| line.starts_with("buzz_readiness_checks_total{")) + .count(); + assert_eq!(exported_reasons, readiness::READINESS_REASON_LABELS.len()); + assert_eq!( + readiness_metric_lines(&final_scrape).len(), + readiness::READINESS_RAW_SERIES_PER_POD, + "readiness series contract must stay at or below its 99-series cap" + ); + }); + }); + } + + fn run_out_of_order_route_case( + first: ReadinessEvaluation, + second: ReadinessEvaluation, + ) -> (serde_json::Value, serde_json::Value, String) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime"); + let evaluator = Arc::new(BarrierReadinessEvaluator::new(first, second)); + let (recorder, handle) = crate::metrics::readiness_test_recorder(); + + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let state = readiness_state(evaluator.clone()).await; + let health = build_health_router(state); + let first_started = evaluator.first_started.notified(); + let slow_first = tokio::spawn(readiness_request(health.clone())); + first_started.await; + + let (_, second_payload) = readiness_request(health).await; + evaluator.release_first.notify_one(); + let (_, first_payload) = slow_first.await.expect("slow first probe task"); + (first_payload, second_payload, handle.render()) + }) + }) + } + + #[test] + fn real_health_route_generation_fence_covers_both_completion_orders() { + let failure = readiness_evaluation( + readiness::PostgresOutcome::Success, + readiness::RedisOutcome::PoolTimeout, + readiness::DeletionCatalogOutcome::Success, + ); + + let (older_failure, newer_success, success_scrape) = + run_out_of_order_route_case(failure, ready_evaluation()); + assert_eq!(older_failure["reason"], json!("redis_pool_timeout")); + assert_eq!(newer_success, json!({"status": "ready"})); + assert_eq!( + metric_value(&success_scrape, "buzz_readiness_state{check=\"overall\"}"), + 1.0 + ); + assert_eq!( + metric_value(&success_scrape, "buzz_readiness_state{check=\"redis\"}"), + 1.0 + ); + + let (older_success, newer_failure, failure_scrape) = + run_out_of_order_route_case(ready_evaluation(), failure); + assert_eq!(older_success, json!({"status": "ready"})); + assert_eq!(newer_failure["reason"], json!("redis_pool_timeout")); + assert_eq!( + metric_value(&failure_scrape, "buzz_readiness_state{check=\"overall\"}"), + 0.0 + ); + assert_eq!( + metric_value(&failure_scrape, "buzz_readiness_state{check=\"redis\"}"), + 0.0 + ); + for scrape in [&success_scrape, &failure_scrape] { + assert_eq!( + metric_value(scrape, "buzz_readiness_checks_total{reason=\"ready\"}"), + 1.0 + ); + assert_eq!( + metric_value( + scrape, + "buzz_readiness_checks_total{reason=\"redis_pool_timeout\"}" + ), + 1.0 + ); + } + } + + #[test] + fn real_health_route_shutdown_fence_dominates_an_in_flight_success() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime"); + let evaluator = Arc::new(BarrierReadinessEvaluator::new( + ready_evaluation(), + ready_evaluation(), + )); + let (recorder, handle) = crate::metrics::readiness_test_recorder(); + + metrics::with_local_recorder(&recorder, || { + runtime.block_on(async { + let state = readiness_state(evaluator.clone()).await; + let health = build_health_router(state.clone()); + let first_started = evaluator.first_started.notified(); + let in_flight = tokio::spawn(readiness_request(health)); + first_started.await; + + state.begin_shutdown(); + evaluator.release_first.notify_one(); + assert_eq!( + in_flight.await.expect("in-flight readiness task"), + ( + StatusCode::SERVICE_UNAVAILABLE, + json!({"status": "shutting_down"}) + ) + ); + + let scrape = handle.render(); + assert_eq!( + metric_value(&scrape, "buzz_readiness_state{check=\"overall\"}"), + 0.0 + ); + assert!(scrape + .lines() + .all(|line| !line.starts_with("buzz_readiness_state{check=\"postgres\"}"))); + assert_eq!( + metric_value( + &scrape, + "buzz_readiness_checks_total{reason=\"shutting_down\"}" + ), + 1.0 + ); + assert_eq!( + metric_value( + &scrape, + "buzz_readiness_dependency_checks_total{dependency=\"postgres\",outcome=\"success\"}" + ), + 1.0 + ); + }); + }); + } + #[tokio::test(flavor = "current_thread")] async fn http_and_datastore_spans_are_exported_in_the_same_trace() { let exporter = InMemorySpanExporter::default(); diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 51e6e8489..230fdc911 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -720,6 +720,8 @@ pub struct AppState { pub audio_rooms: Arc, /// Set to `true` on SIGTERM — readiness probe returns 503. pub shutting_down: Arc, + /// Orders readiness gauge publication against terminal shutdown. + pub(crate) readiness: Arc, /// Process start time — used by `/_status` endpoint. pub started_at: Instant, /// Shared, community-scoped NIP-98 replay prevention. @@ -950,6 +952,7 @@ impl AppState { git_pack_cache, audio_rooms: Arc::new(AudioRoomManager::new()), shutting_down: Arc::new(AtomicBool::new(false)), + readiness: Arc::new(crate::readiness::ReadinessCoordinator::default()), started_at: Instant::now(), nip98_replay, admission_rate_limiter, @@ -1004,6 +1007,23 @@ impl AppState { ) } + /// Atomically closes readiness publication before exposing shutdown to + /// the relay's other fast-path lifecycle checks. + pub fn begin_shutdown(&self) { + self.readiness.begin_shutdown(); + self.shutting_down.store(true, Ordering::Release); + } + + #[cfg(test)] + pub(crate) fn set_readiness_evaluator( + &mut self, + evaluator: Arc, + ) { + self.readiness = Arc::new(crate::readiness::ReadinessCoordinator::with_evaluator( + evaluator, + )); + } + /// Inter-relay mesh handle. `None` ⇒ mesh-off / single-instance: callers /// must no-op to today's behavior. Set once by `main.rs` after boot. pub fn mesh(&self) -> Option<&crate::mesh_boot::MeshHandle> { @@ -1273,7 +1293,7 @@ impl AppState { pub async fn revalidate_live_communities(&self) -> usize { let (closed, failures) = revalidate_registered_communities(&self.community_connections, |community_id| { - self.db.is_community_active(community_id) + self.db.is_community_active_for_maintenance(community_id) }) .await; for (community_id, error) in failures { @@ -1395,11 +1415,33 @@ impl AuditShutdownHandle { /// and the post-cancel drain share the same logic. async fn log_audit_entry(audit: &buzz_audit::AuditService, entry: buzz_audit::NewAuditEntry) { let t = std::time::Instant::now(); - if let Err(e) = audit.log(entry).await { - metrics::counter!("buzz_audit_log_errors_total").increment(1); - tracing::error!("Audit log failed: {e}"); - } else { - metrics::histogram!("buzz_audit_log_seconds").record(t.elapsed().as_secs_f64()); + let mut retry_delay_ms = 50u64; + let mut retries = 0u64; + loop { + match audit.log(entry.clone()).await { + Ok(_) => { + metrics::histogram!("buzz_audit_log_seconds").record(t.elapsed().as_secs_f64()); + return; + } + Err(buzz_audit::AuditError::Database(sqlx::Error::Database(database_error))) + if database_error.code().as_deref() == Some("55P03") => + { + retries += 1; + metrics::counter!("buzz_audit_log_lock_retries_total").increment(1); + tracing::warn!( + retries, + retry_delay_ms, + "Audit advisory lock timed out; preserving entry for retry" + ); + tokio::time::sleep(std::time::Duration::from_millis(retry_delay_ms)).await; + retry_delay_ms = (retry_delay_ms * 2).min(1_000); + } + Err(error) => { + metrics::counter!("buzz_audit_log_errors_total").increment(1); + tracing::error!("Audit log failed: {error}"); + return; + } + } } } @@ -1457,6 +1499,10 @@ pub mod tests { /// Build an `AppState` from the environment (`DATABASE_URL`), with relay /// membership enforcement off and an unreachable Redis (pub/sub state is /// lazy). The returned state is ready for handlers or `build_router`. + /// + /// Also shared with `crate::rejection`'s tests, where the unreachable + /// Redis is what makes admission resolve to `AdmissionError::Unavailable` + /// without any live infrastructure. pub async fn test_state() -> Arc { test_state_with_redis("redis://127.0.0.1:1").await } @@ -1501,6 +1547,130 @@ pub mod tests { Arc::new(state) } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn audit_worker_retries_lock_timeout_until_original_entry_is_appended_once() { + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL"); + let observer = sqlx::PgPool::connect(&database_url) + .await + .expect("connect observer pool"); + let application_name = format!("audit-retry-test-{}", Uuid::new_v4()); + let hook_application_name = application_name.clone(); + let audit_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .after_connect(move |conn, _meta| { + let application_name = hook_application_name.clone(); + Box::pin(async move { + sqlx::query( + "SELECT set_config('application_name', $1, false), \ + set_config('lock_timeout', '100', false)", + ) + .bind(application_name) + .execute(&mut *conn) + .await?; + Ok(()) + }) + }) + .connect(&database_url) + .await + .expect("connect audit pool"); + + let community_id = Uuid::new_v4(); + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community_id) + .bind(format!("audit-retry-{community_id}.example")) + .execute(&observer) + .await + .expect("insert test community"); + let object_id = format!("audit-retry-object-{}", Uuid::new_v4()); + let entry = buzz_audit::NewAuditEntry { + community_id: CommunityId::from_uuid(community_id), + action: buzz_audit::AuditAction::EventCreated, + actor_pubkey: Some(vec![0xab; 32]), + object_id: Some(object_id.clone()), + detail: serde_json::json!({"test": "lock-timeout-retry"}), + }; + + // Mirrors buzz_audit::service::AUDIT_LOCK_NAMESPACE. + let lock_key = format!("buzz_audit:{community_id}"); + let mut holder = observer.acquire().await.expect("acquire lock holder"); + sqlx::query("SELECT pg_advisory_lock(hashtextextended($1, 0))") + .bind(&lock_key) + .execute(&mut *holder) + .await + .expect("hold community audit lock"); + + let audit = Arc::new(AuditService::new(audit_pool)); + let worker = tokio::spawn({ + let audit = Arc::clone(&audit); + async move { log_audit_entry(&audit, entry).await } + }); + + // Observe one timed-out advisory-lock attempt and then a second wait. + // Releasing during the first wait would not prove that the worker + // preserved and retried the original queue entry. + tokio::time::timeout(std::time::Duration::from_secs(3), async { + let mut saw_first_wait = false; + let mut saw_retry_gap = false; + loop { + let waiting: bool = sqlx::query_scalar( + "SELECT EXISTS (\ + SELECT 1 FROM pg_stat_activity \ + WHERE application_name = $1 \ + AND query LIKE 'SELECT pg_advisory_lock%' \ + AND wait_event = 'advisory'\ + )", + ) + .bind(&application_name) + .fetch_one(&observer) + .await + .expect("inspect audit lock waiter"); + if waiting { + if saw_retry_gap { + break; + } + saw_first_wait = true; + } else if saw_first_wait { + saw_retry_gap = true; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .expect("audit worker never retried after lock_timeout"); + + sqlx::query("SELECT pg_advisory_unlock(hashtextextended($1, 0))") + .bind(&lock_key) + .execute(&mut *holder) + .await + .expect("release community audit lock"); + tokio::time::timeout(std::time::Duration::from_secs(3), worker) + .await + .expect("audit worker did not finish after lock release") + .expect("audit worker task panicked"); + + let rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM audit_log WHERE community_id = $1 AND object_id = $2", + ) + .bind(community_id) + .bind(&object_id) + .fetch_one(&observer) + .await + .expect("count retried audit rows"); + assert_eq!(rows, 1, "the preserved entry must be appended exactly once"); + + sqlx::query("DELETE FROM audit_log WHERE community_id = $1 AND object_id = $2") + .bind(community_id) + .bind(&object_id) + .execute(&observer) + .await + .expect("remove test audit row"); + // Community rows are permanent tombstones, and deleting one cascades + // into fenced child tables after the row is gone, which trips + // enforce_community_write_fence (migration 0059). Leave the host row in + // place; it is `audit-retry-.example`, so reruns never collide. + } + #[test] fn send_to_resets_grace_counter_on_success() { let (mgr, id, _rx, _ctrl_rx, _cancel, bp) = setup_conn(16); diff --git a/crates/buzz-relay/src/telemetry.rs b/crates/buzz-relay/src/telemetry.rs index 91bd92f0f..7ffd6a733 100644 --- a/crates/buzz-relay/src/telemetry.rs +++ b/crates/buzz-relay/src/telemetry.rs @@ -223,9 +223,9 @@ pub enum TracerInit { Enabled(SdkTracerProvider), /// `OTEL_EXPORTER_OTLP_ENDPOINT` was unset — no-op, no connection. Disabled, - /// Endpoint was set but the exporter failed to build. The inner error - /// string is suitable for a `tracing::warn!` call made by the caller - /// **after** `tracing_subscriber::registry()…init()`. + /// Endpoint was set but the exporter failed to build. The inner error is + /// diagnostic data only and must not be logged: exporter errors can + /// include credential-bearing endpoint URLs. ExporterBuildFailed(String), } @@ -234,7 +234,8 @@ pub enum TracerInit { /// /// Deliberately does **not** call `tracing::warn!` internally — the subscriber /// may not be installed yet at call time, which would silently drop the event. -/// Callers are responsible for logging [`TracerInit::ExporterBuildFailed`]. +/// Callers may log a fixed, credential-free message for +/// [`TracerInit::ExporterBuildFailed`], but must not log its inner error. pub fn try_init_tracer(resource: Resource) -> TracerInit { if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_err() { return TracerInit::Disabled; diff --git a/crates/buzz-relay/src/test_support.rs b/crates/buzz-relay/src/test_support.rs new file mode 100644 index 000000000..b6d487bf0 --- /dev/null +++ b/crates/buzz-relay/src/test_support.rs @@ -0,0 +1,97 @@ +//! Test-only helpers shared by the relay's unit tests. +//! +//! Upstream also exposes a `database_url()` helper here; Colony's PostgreSQL +//! tests resolve their own URL, so only the isolated-child runner is carried. + +const CHILD_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +const MAX_CAPTURE_BYTES: u64 = 1024 * 1024; + +struct CapturedStream { + retained: Vec, + total_bytes: u64, +} + +fn capture_stream(mut stream: impl std::io::Read) -> CapturedStream { + let mut retained = Vec::new(); + let mut total_bytes = 0_u64; + let mut chunk = [0_u8; 8192]; + loop { + let read = stream.read(&mut chunk).expect("read child output pipe"); + if read == 0 { + break; + } + total_bytes = total_bytes.saturating_add(u64::try_from(read).expect("read size fits u64")); + let remaining = usize::try_from(MAX_CAPTURE_BYTES) + .expect("capture ceiling fits usize") + .saturating_sub(retained.len()); + retained.extend_from_slice(&chunk[..read.min(remaining)]); + } + CapturedStream { + retained, + total_bytes, + } +} + +fn join_capture(capture: std::thread::JoinHandle, stream: &str) -> Vec { + let capture = capture.join().expect("child capture thread must not panic"); + assert!( + capture.total_bytes <= MAX_CAPTURE_BYTES, + "child {stream} exceeded {MAX_CAPTURE_BYTES} bytes: {}", + capture.total_bytes, + ); + capture.retained +} + +/// Run exactly one unit test in an isolated, deadline-bounded child process. +pub(crate) fn run_exact_test_child(test_name: &str, child_env: &str) { + use std::{ + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, + }; + + let mut child = Command::new(std::env::current_exe().expect("test executable")) + .arg("--exact") + .arg(test_name) + .arg("--nocapture") + .env(child_env, "1") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn isolated test child"); + let stdout = child.stdout.take().expect("child stdout pipe"); + let stderr = child.stderr.take().expect("child stderr pipe"); + let stdout = thread::spawn(move || capture_stream(stdout)); + let stderr = thread::spawn(move || capture_stream(stderr)); + + let deadline = Instant::now() + CHILD_TEST_TIMEOUT; + let (status, timed_out) = loop { + if let Some(status) = child.try_wait().expect("poll isolated test child") { + break (status, false); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let status = child.wait().expect("reap timed-out test child"); + break (status, true); + } + thread::sleep(Duration::from_millis(10)); + }; + + let stdout = join_capture(stdout, "stdout"); + let stderr = join_capture(stderr, "stderr"); + let output = format!( + "{}{}", + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + + assert!( + !timed_out, + "isolated test child exceeded {CHILD_TEST_TIMEOUT:?}:\n{output}" + ); + assert!(status.success(), "isolated test child failed:\n{output}"); + assert!( + output.contains("running 1 test") && output.contains(test_name), + "exact selector did not run the intended test {test_name}:\n{output}" + ); +} diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index 04ad5f8b5..ad9f11b44 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -221,7 +221,7 @@ impl ActionSink for RelayActionSink { let channel = state .db - .get_channel(tenant.community(), channel_uuid) + .get_channel_for_event_write(tenant.community(), channel_uuid) .await .map_err(|e| match &e { buzz_db::DbError::ChannelNotFound(_) | buzz_db::DbError::NotFound(_) => { @@ -269,13 +269,13 @@ impl ActionSink for RelayActionSink { let members = state .db - .get_members(tenant.community(), channel_uuid) + .get_members_for_event_write(tenant.community(), channel_uuid) .await .map_err(|e| ActionSinkError::Database(e.to_string()))?; let member_pubkeys: Vec> = members.iter().map(|m| m.pubkey.clone()).collect(); let users = state .db - .get_users_bulk(tenant.community(), &member_pubkeys) + .get_users_bulk_for_event_write(tenant.community(), &member_pubkeys) .await .map_err(|e| ActionSinkError::Database(e.to_string()))?; let named_members: Vec<(String, String)> = users @@ -747,7 +747,7 @@ mod integration_tests { .to_vec(); let stored = state .db - .get_event_by_id(community, &id_bytes) + .get_event_by_id_for_event_write(community, &id_bytes) .await .expect("query event") .expect("event persisted"); @@ -968,7 +968,7 @@ mod integration_tests { .to_vec(); state .db - .get_event_by_id(community, &id_bytes) + .get_event_by_id_for_event_write(community, &id_bytes) .await .expect("query event") .expect("event persisted") diff --git a/crates/buzz-relay/src/workspace_tab_broker.rs b/crates/buzz-relay/src/workspace_tab_broker.rs index 29f3cc4e5..3c1ad3ec0 100644 --- a/crates/buzz-relay/src/workspace_tab_broker.rs +++ b/crates/buzz-relay/src/workspace_tab_broker.rs @@ -89,7 +89,7 @@ async fn apply_tab_action_inner( let mut tx = state .db - .begin_transaction() + .begin_event_write_transaction() .await .map_err(|error| format!("workspace tab transaction failed: {error}"))?; diff --git a/crates/buzz-relay/tests/boot_lifecycle.rs b/crates/buzz-relay/tests/boot_lifecycle.rs new file mode 100644 index 000000000..29fcf991f --- /dev/null +++ b/crates/buzz-relay/tests/boot_lifecycle.rs @@ -0,0 +1,457 @@ +use std::{ + collections::BTreeMap, + io::{Read as _, Write as _}, + net::{TcpListener, TcpStream}, + process::{Child, Command, ExitStatus, Output, Stdio}, + thread::{self, JoinHandle}, + time::{Duration, Instant}, +}; + +use serde_json::Value; + +use buzz_relay::lifecycle::StartupPhase; + +const VALID_RELAY_PRIVATE_KEY: &str = + "0000000000000000000000000000000000000000000000000000000000000001"; +const CHILD_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_CAPTURE_BYTES: u64 = 1024 * 1024; + +struct RelayProcess { + child: Option, + stdout: Option>, + stderr: Option>, + scratch_dir: std::path::PathBuf, +} + +struct CapturedStream { + retained: Vec, + total_bytes: u64, +} + +impl RelayProcess { + fn spawn(environment: &[(&str, &str)]) -> Self { + let scratch_dir = + std::env::temp_dir().join(format!("buzz-boot-lifecycle-{}", uuid::Uuid::new_v4())); + let mut command = Command::new(env!("CARGO_BIN_EXE_buzz-relay")); + command + .env_clear() + .env("RUST_BACKTRACE", "0") + .env("RUST_LOG", "buzz_relay=info") + .env("BUZZ_GIT_REPO_PATH", scratch_dir.join("repos")) + .env("BUZZ_GIT_PACK_CACHE_PATH", scratch_dir.join("pack-cache")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for (name, value) in environment { + command.env(name, value); + } + let mut child = command.spawn().expect("spawn buzz-relay child process"); + let stdout = child.stdout.take().expect("relay stdout pipe"); + let stderr = child.stderr.take().expect("relay stderr pipe"); + Self { + child: Some(child), + stdout: Some(thread::spawn(move || capture_stream(stdout))), + stderr: Some(thread::spawn(move || capture_stream(stderr))), + scratch_dir, + } + } + + fn try_wait(&mut self) -> Option { + self.child + .as_mut() + .expect("relay child") + .try_wait() + .expect("poll relay child") + } + + fn wait(mut self, timeout: Duration) -> Output { + let deadline = Instant::now() + timeout; + let status = loop { + if let Some(status) = self.try_wait() { + break status; + } + if Instant::now() >= deadline { + let child = self.child.as_mut().expect("relay child"); + let _ = child.kill(); + let _ = child.wait(); + panic!("buzz-relay child exceeded {timeout:?}"); + } + thread::sleep(Duration::from_millis(10)); + }; + self.child.take(); + let output = Output { + status, + stdout: join_capture(self.stdout.take(), "stdout"), + stderr: join_capture(self.stderr.take(), "stderr"), + }; + let _ = std::fs::remove_dir_all(&self.scratch_dir); + output + } + + fn terminate(mut self) -> Output { + self.child + .as_mut() + .expect("relay child") + .kill() + .expect("terminate exact relay child"); + self.wait(Duration::from_secs(2)) + } +} + +impl Drop for RelayProcess { + fn drop(&mut self) { + if let Some(child) = self.child.as_mut() { + let _ = child.kill(); + let _ = child.wait(); + } + let _ = std::fs::remove_dir_all(&self.scratch_dir); + } +} + +fn capture_stream(mut stream: impl std::io::Read) -> CapturedStream { + let mut retained = Vec::new(); + let mut total_bytes = 0_u64; + let mut chunk = [0_u8; 8192]; + loop { + let read = stream.read(&mut chunk).expect("read relay output pipe"); + if read == 0 { + break; + } + total_bytes = total_bytes.saturating_add(u64::try_from(read).expect("read size fits u64")); + let remaining = usize::try_from(MAX_CAPTURE_BYTES) + .expect("capture ceiling fits usize") + .saturating_sub(retained.len()); + retained.extend_from_slice(&chunk[..read.min(remaining)]); + } + CapturedStream { + retained, + total_bytes, + } +} + +fn join_capture(capture: Option>, stream: &str) -> Vec { + let capture = capture + .expect("relay capture thread") + .join() + .expect("relay capture thread must not panic"); + assert!( + capture.total_bytes <= MAX_CAPTURE_BYTES, + "relay {stream} exceeded {MAX_CAPTURE_BYTES} bytes: {}", + capture.total_bytes, + ); + capture.retained +} + +fn run_relay(environment: &[(&str, &str)]) -> Output { + RelayProcess::spawn(environment).wait(CHILD_TIMEOUT) +} + +fn scrape_metrics(port: u16) -> std::io::Result { + let address = std::net::SocketAddr::from(([127, 0, 0, 1], port)); + let mut stream = TcpStream::connect_timeout(&address, Duration::from_millis(100))?; + stream.set_read_timeout(Some(Duration::from_millis(500)))?; + stream.write_all(b"GET /metrics HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n")?; + let mut response = String::new(); + stream.read_to_string(&mut response)?; + Ok(response) +} + +fn wait_for_relay_metrics(process: &mut RelayProcess, port: u16) -> String { + let deadline = Instant::now() + Duration::from_secs(8); + loop { + assert!( + process.try_wait().is_none(), + "relay exited before its metrics endpoint became usable" + ); + if let Ok(response) = scrape_metrics(port) { + if response.contains("buzz_audit_enabled") { + return response; + } + } + assert!( + Instant::now() < deadline, + "relay metrics did not become scrapeable within 8s" + ); + thread::sleep(Duration::from_millis(20)); + } +} + +fn assert_no_startup_lifecycle_metrics(scrape: &str) { + for line in scrape.lines() { + let Some(name) = line + .strip_prefix("# HELP ") + .or_else(|| line.strip_prefix("# TYPE ")) + .and_then(|rest| rest.split_ascii_whitespace().next()) + else { + continue; + }; + assert!( + !["startup", "boot", "lifecycle"] + .iter() + .any(|term| name.contains(term)) + && !StartupPhase::ALL + .iter() + .any(|phase| name.contains(phase.as_str())), + "logs-only lifecycle contract emitted metric family {name}" + ); + } +} + +fn lifecycle_events(output: &Output) -> Vec { + let mut events: Vec = output + .stdout + .split(|byte| *byte == b'\n') + .chain(output.stderr.split(|byte| *byte == b'\n')) + .filter_map(|line| serde_json::from_slice::(line).ok()) + .filter(|event| event["event_name"] == "buzz_process_lifecycle") + .collect(); + events.sort_by_key(|event| event["sequence"].as_u64()); + events +} + +fn lifecycle_events_from(bytes: &[u8]) -> Vec { + bytes + .split(|byte| *byte == b'\n') + .filter_map(|line| serde_json::from_slice::(line).ok()) + .filter(|event| event["event_name"] == "buzz_process_lifecycle") + .collect() +} + +fn assert_accounting(events: &[Value]) { + assert!(!events.is_empty(), "child emitted no lifecycle events"); + let boot_id = events[0]["process_boot_id"] + .as_str() + .expect("process_boot_id"); + let mut counts = BTreeMap::::new(); + for (index, event) in events.iter().enumerate() { + assert_eq!(event["schema_version"], 1); + assert_eq!(event["sequence"], u64::try_from(index + 1).unwrap()); + assert_eq!(event["process_boot_id"], boot_id); + assert_eq!(event["track"], "startup"); + let count = counts + .entry(event["phase"].as_str().expect("phase").to_owned()) + .or_default(); + match event["edge"].as_str() { + Some("started") => count.0 += 1, + Some("terminal") => count.1 += 1, + other => panic!("unexpected lifecycle edge: {other:?}"), + } + } + assert!( + counts + .values() + .all(|(started, terminal)| *started == 1 && *terminal == 1), + "every started phase must have one terminal: {counts:?}" + ); +} + +fn assert_terminal(events: &[Value], phase: &str, status: &str, reason: Option<&str>) { + let terminal = events + .iter() + .find(|event| event["phase"] == phase && event["edge"] == "terminal") + .unwrap_or_else(|| panic!("missing {phase} terminal")); + assert_eq!(terminal["status"], status); + match reason { + Some(reason) => assert_eq!(terminal["reason"], reason), + None => assert!(terminal["reason"].is_null()), + } +} + +fn phases(events: &[Value]) -> Vec<&str> { + events + .iter() + .filter(|event| event["edge"] == "started") + .map(|event| event["phase"].as_str().expect("phase")) + .collect() +} + +#[test] +fn invalid_config_terminalizes_at_main_even_with_logs_disabled() { + let output = run_relay(&[ + ("RUST_LOG", "off"), + ("BUZZ_BIND_ADDR", "not-a-socket-address"), + ]); + assert!(!output.status.success()); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_eq!( + phases(&events), + [ + "process_telemetry", + "crypto_init", + "tracing_init", + "config_load" + ] + ); + assert_terminal(&events, "config_load", "failed", Some("config_invalid")); + assert_terminal( + &events, + "process_telemetry", + "failed", + Some("config_invalid"), + ); + assert_eq!(lifecycle_events_from(&output.stderr), events); + assert!(lifecycle_events_from(&output.stdout).is_empty()); +} + +#[test] +#[cfg(unix)] +fn config_filesystem_failure_has_a_bounded_terminal() { + let output = run_relay(&[ + ("RUST_LOG", "off"), + ("BUZZ_GIT_REPO_PATH", "/dev/null/not-a-directory"), + ]); + assert!(!output.status.success()); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_terminal(&events, "config_load", "failed", Some("config_invalid")); + assert_terminal( + &events, + "process_telemetry", + "failed", + Some("config_invalid"), + ); +} + +#[test] +fn invalid_config_value_has_the_same_bounded_terminal() { + let output = run_relay(&[("RUST_LOG", "off"), ("BUZZ_DRAIN_JITTER_MS", "bogus")]); + assert!(!output.status.success()); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_terminal(&events, "config_load", "failed", Some("config_invalid")); + assert_terminal( + &events, + "process_telemetry", + "failed", + Some("config_invalid"), + ); +} + +#[test] +fn configured_otlp_terminalizes_tracing_before_a_later_failure() { + let output = run_relay(&[ + ("RUST_LOG", "off"), + ("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4317"), + ("BUZZ_BIND_ADDR", "not-a-socket-address"), + ]); + assert!(!output.status.success()); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_terminal(&events, "tracing_init", "succeeded", None); + assert_terminal(&events, "config_load", "failed", Some("config_invalid")); +} + +#[test] +fn missing_key_stops_before_metrics_bind() { + let output = run_relay(&[]); + assert!(!output.status.success()); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_eq!( + phases(&events), + [ + "process_telemetry", + "crypto_init", + "tracing_init", + "config_load", + "key_load" + ] + ); + assert_terminal(&events, "key_load", "failed", Some("missing")); + assert_terminal(&events, "process_telemetry", "failed", Some("missing")); +} + +#[test] +fn invalid_key_uses_a_bounded_reason_without_leaking_the_value() { + let secret = "private-key-material-that-must-not-appear"; + let output = run_relay(&[("BUZZ_RELAY_PRIVATE_KEY", secret)]); + assert!(!output.status.success()); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_terminal(&events, "key_load", "failed", Some("required_invalid")); + let combined = [output.stdout, output.stderr].concat(); + assert!(!String::from_utf8_lossy(&combined).contains(secret)); +} + +#[test] +fn occupied_metrics_port_has_a_typed_bind_terminal() { + let occupied = TcpListener::bind(("0.0.0.0", 0)).expect("bind occupied port"); + let port = occupied.local_addr().expect("occupied address").port(); + let port = port.to_string(); + let output = run_relay(&[ + ("BUZZ_RELAY_PRIVATE_KEY", VALID_RELAY_PRIVATE_KEY), + ("BUZZ_METRICS_PORT", &port), + ]); + assert!(!output.status.success()); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_terminal(&events, "metrics_bind", "failed", Some("bind")); + assert_terminal(&events, "process_telemetry", "failed", Some("bind")); +} + +#[test] +fn otlp_build_failure_is_degraded_without_leaking_endpoint_credentials() { + let secret = "telemetry-secret-marker"; + let endpoint = format!("https://telemetry-user:{secret}@["); + let fake_database = TcpListener::bind(("127.0.0.1", 0)).expect("bind fake database"); + let database_url = format!( + "postgres://buzz@127.0.0.1:{}/buzz", + fake_database.local_addr().expect("database address").port() + ); + let reserved = TcpListener::bind(("127.0.0.1", 0)).expect("reserve metrics port"); + let port = reserved.local_addr().expect("metrics address").port(); + drop(reserved); + let port_value = port.to_string(); + let mut process = RelayProcess::spawn(&[ + ("OTEL_EXPORTER_OTLP_ENDPOINT", &endpoint), + ("RUST_LOG", "buzz_relay=warn"), + ("BUZZ_RELAY_PRIVATE_KEY", VALID_RELAY_PRIVATE_KEY), + ("BUZZ_METRICS_PORT", &port_value), + ("DATABASE_URL", &database_url), + ]); + let scrape = wait_for_relay_metrics(&mut process, port); + assert_no_startup_lifecycle_metrics(&scrape); + let output = process.terminate(); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_terminal(&events, "tracing_init", "degraded", Some("exporter_build")); + assert_terminal( + &events, + "process_telemetry", + "degraded", + Some("exporter_build"), + ); + let combined = [output.stdout, output.stderr].concat(); + assert!(!String::from_utf8_lossy(&combined).contains(secret)); +} + +#[test] +fn successful_main_emits_complete_lifecycle_without_startup_metrics() { + let fake_database = TcpListener::bind(("127.0.0.1", 0)).expect("bind fake database"); + let database_url = format!( + "postgres://buzz@127.0.0.1:{}/buzz", + fake_database.local_addr().expect("database address").port() + ); + let reserved = TcpListener::bind(("127.0.0.1", 0)).expect("reserve metrics port"); + let port = reserved.local_addr().expect("metrics address").port(); + drop(reserved); + let port_value = port.to_string(); + let mut process = RelayProcess::spawn(&[ + ("RUST_LOG", "off"), + ("BUZZ_RELAY_PRIVATE_KEY", VALID_RELAY_PRIVATE_KEY), + ("BUZZ_METRICS_PORT", &port_value), + ("DATABASE_URL", &database_url), + ]); + let scrape = wait_for_relay_metrics(&mut process, port); + assert_no_startup_lifecycle_metrics(&scrape); + + let output = process.terminate(); + let events = lifecycle_events(&output); + assert_accounting(&events); + assert_terminal(&events, "crypto_init", "succeeded", None); + assert_terminal(&events, "tracing_init", "succeeded", None); + assert_terminal(&events, "config_load", "succeeded", None); + assert_terminal(&events, "key_load", "succeeded", None); + assert_terminal(&events, "metrics_bind", "succeeded", None); + assert_terminal(&events, "process_telemetry", "succeeded", None); +} diff --git a/crates/buzz-search/Cargo.toml b/crates/buzz-search/Cargo.toml index e28c5b684..6c6b9ada2 100644 --- a/crates/buzz-search/Cargo.toml +++ b/crates/buzz-search/Cargo.toml @@ -14,6 +14,7 @@ sqlx = { workspace = true } uuid = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } +metrics = { workspace = true } [dev-dependencies] tokio = { workspace = true } diff --git a/crates/buzz-search/tests/fts_integration.rs b/crates/buzz-search/tests/fts_integration.rs index 559931e45..2787afe3c 100644 --- a/crates/buzz-search/tests/fts_integration.rs +++ b/crates/buzz-search/tests/fts_integration.rs @@ -36,6 +36,8 @@ const MIGRATION_0032_SQL: &str = const MIGRATION_0036_SQL: &str = include_str!("../../../migrations/0036_discovery_workspace_records.sql"); const MIGRATION_0037_SQL: &str = include_str!("../../../migrations/0037_usage_record_fts.sql"); +const MIGRATION_0074_SQL: &str = + include_str!("../../../migrations/0074_private_managed_agent_fts.sql"); async fn setup() -> (PgPool, String) { let url = std::env::var("BUZZ_TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.to_string()); @@ -101,6 +103,9 @@ async fn setup() -> (PgPool, String) { pool.execute(MIGRATION_0037_SQL) .await .expect("apply 0037 migration"); + pool.execute(MIGRATION_0074_SQL) + .await + .expect("apply 0074 migration"); (pool, schema) } diff --git a/crates/buzz-test-client/tests/e2e_relay.rs b/crates/buzz-test-client/tests/e2e_relay.rs index 70aecf734..9d9104978 100644 --- a/crates/buzz-test-client/tests/e2e_relay.rs +++ b/crates/buzz-test-client/tests/e2e_relay.rs @@ -839,16 +839,12 @@ async fn test_ws_quota_rejection_retries_identical_event_after_expiry() { .send_raw(&frame) .await .expect("send quota-blocked event"); - let notice = match client - .recv_event(Duration::from_secs(5)) - .await - .expect("quota NOTICE") - { - RelayMessage::Notice { message } => message, - other => panic!("expected quota NOTICE, got {other:?}"), - }; - assert!(notice.starts_with("rate-limited: quota exceeded; retry in ")); - match client + // A refused EVENT is answered on its own acknowledgement channel first, so + // a client keyed by event id can settle the exact pending publish without + // waiting out its publish timeout. The NOTICE follows as a compatibility + // frame for clients shipped before Colony armed back-pressure from the OK + // reason; both must arrive, in this order. + let reason = match client .recv_event(Duration::from_secs(5)) .await .expect("event-scoped quota OK") @@ -856,9 +852,27 @@ async fn test_ws_quota_rejection_retries_identical_event_after_expiry() { RelayMessage::Ok(response) => { assert_eq!(response.event_id, event_id); assert!(!response.accepted); - assert_eq!(response.message, notice); + assert!( + response + .message + .starts_with("rate-limited: quota exceeded; retry in "), + "the retry hint must survive so the client can arm its gate, got {:?}", + response.message + ); + response.message } other => panic!("expected exact event rejection, got {other:?}"), + }; + match client + .recv_event(Duration::from_secs(5)) + .await + .expect("quota NOTICE") + { + RelayMessage::Notice { message } => assert_eq!( + message, reason, + "the compatibility NOTICE must repeat the OK's reason verbatim" + ), + other => panic!("expected the compatibility quota NOTICE, got {other:?}"), } let rejected_count: i64 = sqlx::query_scalar("SELECT count(*) FROM events WHERE id = $1") .bind(event.id.to_bytes().to_vec()) diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 869896766..7ee99ca89 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -100,6 +100,79 @@ disables that probe through `relay.extraEnv`, `/_readiness` does not test object storage; configuration is still parsed strictly, but reachability and addressing errors surface on the first storage operation. +### Early-startup telemetry contract + +`buzz_process_lifecycle` JSON records are the authoritative history for the +fixed phases `crypto_init`, `tracing_init`, `config_load`, `key_load`, and +`metrics_bind`, plus the aggregate `process_telemetry` result. They use bounded +status/reason values and never contain raw configuration, keys, URLs, or errors. +These phases intentionally do not emit metrics. Most run before the Prometheus +exporter exists, and one uniform log-only contract preserves every phase's real +event time and failure without assigning an eventual scrape time to earlier work. + +### Readiness telemetry contract + +Only requests served by the private health listener (`BUZZ_HEALTH_PORT`) emit +rollout readiness telemetry. The compatibility `/_readiness` route on the public +app listener returns health but does not change these metrics. + +| Metric | Type | Labels | +|--------|------|--------| +| `buzz_readiness_checks_total` | counter | `reason` from the closed readiness-reason set | +| `buzz_readiness_dependency_checks_total` | counter | `dependency`, typed bounded `outcome` | +| `buzz_readiness_check_duration_seconds` | histogram | `check` only | +| `buzz_readiness_state` | gauge | `check` only; latest publishable generation | + +The schema has a ceiling of 99 raw Prometheus series per pod: 12 overall +reasons, 11 valid dependency/outcome pairs, 72 histogram series, and 4 gauges. +Do not add pod, ReplicaSet, version, rollout, error text, SQL, URL, tenant, +user, community, pubkey, header, query, or other request-controlled labels. +Shutdown without dependency evaluation increments only +`buzz_readiness_checks_total{reason="shutting_down"}` and sets the overall +state to zero; it does not fabricate dependency failures or latency samples. + +### Operation-aware database pool acquisition contract + +The operation-aware families separate three questions: who is waiting now, +how completed/abandoned attempts ended, and how long checkout waits took. +Outcome remains on the terminal counter for historical deployment comparison; +it is intentionally absent from the expensive duration histogram. + +These families cover the explicitly routed deployment-critical operations +listed below; they are not a count of every SQLx checkout in Buzz. In +particular, a zero operation waiter does not prove that the shared SQLx pool +has no uninstrumented waiter. Interpret it beside the pool active, idle, and +maximum gauges when diagnosing total capacity pressure. + +| Metric | Type | Labels | +|--------|------|--------| +| `buzz_db_pool_acquire_duration_seconds` | histogram | `pool_role`, `operation` | +| `buzz_db_pool_acquire_attempts_total` | counter | `pool_role`, `operation`, `outcome` | +| `buzz_db_pool_waiters` | gauge | `pool_role`, `operation`; tracked operations only, periodically refreshed including zero | + +Outcomes are `success`, `timeout`, `error`, and `cancelled`. Operations are +`bootstrap`, `readiness`, `tenant_resolution`, `authentication`, +`authorization`, `subscription_history`, `event_write`, and `maintenance`. +Only the following eleven pairs are valid: + +```text +writer/bootstrap reader/bootstrap +writer/readiness +writer/tenant_resolution +writer/authentication +writer/authorization reader/authorization +writer/subscription_history reader/subscription_history +writer/event_write +writer/maintenance +``` + +Nine finite checkout buckets plus `+Inf`, sum, and count yield 12 histogram +series per valid pair. The new contract therefore has a hard ceiling of 187 +raw Prometheus series per pod: `11 x (12 + 4 + 1)`. The two legacy acquisition +families remain temporarily for dashboard compatibility and are not part of +that new-family budget. No `other` operation or request-controlled/sensitive +label is valid. + ## Relay Pod extensions The chart exposes narrow extension points for init containers, volumes, relay @@ -205,6 +278,8 @@ default so long-lived WebSocket connections have time to drain. Schema migrations are embedded in the relay binary via `sqlx::migrate!` and run at startup, gated by `BUZZ_AUTO_MIGRATE` (default `true`). Multiple replicas race-safely behind a Postgres advisory lock. `helm upgrade` is the entire upgrade procedure. +Migration 0032 is a hard compatibility boundary for relay versions that publish repaired channel rosters. The relay verifies the roster-fence trigger catalog and behavior before opening listeners and refuses to start if 0032 is missing or inert. Apply migrations before rolling the relay; for large installations, prefer a controlled `buzz-admin migrate` job with PostgreSQL lock monitoring before the code rollout. + If you prefer decoupling migrations from serving, set `migrate.autoMigrate=false`. **In that mode the chart does not run migrations for you** — you own running `buzz-admin migrate` (separate Pod / one-shot Job) against the database before every `helm install` / `helm upgrade`. Readiness probes only verify DB connectivity, not schema freshness, so a pod will appear healthy against an unmigrated schema and fail under load. A pre-upgrade Helm Job for this is on the chart roadmap; the values knob `migrate.preUpgradeJob.enabled` is reserved. ## Backups diff --git a/desktop/src/shared/api/readOnlyRelayClient.ts b/desktop/src/shared/api/readOnlyRelayClient.ts index 7c35dfb85..16a0df2a8 100644 --- a/desktop/src/shared/api/readOnlyRelayClient.ts +++ b/desktop/src/shared/api/readOnlyRelayClient.ts @@ -8,6 +8,10 @@ import { type RelaySubscriptionFilter, } from "@/shared/api/relayClientShared"; import { closeWebSocket } from "@/shared/api/relayWebSocketClose"; +import { + activateRateLimitIfSignalled, + waitForRateLimit, +} from "@/shared/api/relayRateLimitGate"; import { AUTH_TIMEOUT_MS, HISTORY_TIMEOUT_MS, @@ -106,7 +110,10 @@ export class ReadOnlyRelayClient { async publishEvent(event: RelayEvent): Promise { await this.connect(); - if (this.wsId === null) { + const generation = this.generation; + await waitForRateLimit(); + + if (generation !== this.generation || this.wsId === null) { throw new Error("Read-only relay socket is not connected."); } @@ -278,6 +285,7 @@ export class ReadOnlyRelayClient { if (success) { publish.resolve(); } else { + activateRateLimitIfSignalled(message); publish.reject( new Error(message || "Observer relay rejected the event."), ); diff --git a/desktop/src/shared/api/readOnlyRelayClientPublishRejection.test.mjs b/desktop/src/shared/api/readOnlyRelayClientPublishRejection.test.mjs new file mode 100644 index 000000000..ea0e14629 --- /dev/null +++ b/desktop/src/shared/api/readOnlyRelayClientPublishRejection.test.mjs @@ -0,0 +1,165 @@ +// ReadOnlyRelayClient publishes to inactive communities, but it shares the +// process-wide relay rate-limit gate with the primary session. Addressed EVENT +// refusals therefore need to settle this client's pending publish, arm the +// shared gate, and defer later sends until the advertised window expires. +import assert from "node:assert/strict"; +import test from "node:test"; + +let fakeNow = 0; +const pendingTimers = new Map(); +let nextTimerId = 1; +const sends = []; + +globalThis.window = { + setTimeout: (fn, ms) => { + const id = nextTimerId++; + pendingTimers.set(id, { fn, fireAt: fakeNow + ms }); + return id; + }, + clearTimeout: (id) => pendingTimers.delete(id), +}; +Date.now = () => fakeNow; + +// Colony routes every native call through the NativeBridge indirection rather +// than reaching into `__TAURI_INTERNALS__`, so the send transport is installed +// here instead of on `window`. +const { setNativeBridge } = await import("@/shared/api/nativeBridge"); +const { createMockNativeBridge } = await import( + "@/testing/createMockNativeBridge" +); +setNativeBridge( + createMockNativeBridge((command, args) => { + if (command === "plugin:websocket|send") sends.push(args); + return undefined; + }), +); + +const { ReadOnlyRelayClient } = await import("./readOnlyRelayClient.ts"); +const { activateRateLimit, isRateLimited, resetRateLimitGate } = await import( + "./relayRateLimitGate.ts" +); + +function tickTo(ms) { + fakeNow = ms; + for (const [id, { fn, fireAt }] of Array.from(pendingTimers.entries())) { + if (fireAt <= fakeNow) { + pendingTimers.delete(id); + fn(); + } + } +} + +function reset() { + resetRateLimitGate(); + fakeNow = 0; + pendingTimers.clear(); + nextTimerId = 1; + sends.length = 0; +} + +function connectedClient() { + const client = new ReadOnlyRelayClient("wss://inactive.example"); + client.wsId = 7; + client.connect = async () => {}; + return client; +} + +function armPendingPublish(client, eventId) { + const settled = new Promise((resolve, reject) => { + client.publishes.set(eventId, { + resolve, + reject, + timeout: window.setTimeout(() => {}, 25_000), + }); + }); + return settled.then( + () => ({ status: "resolved" }), + (error) => ({ status: "rejected", error }), + ); +} + +function deliver(client, frame) { + return client.handleWsMessage( + { type: "Text", data: JSON.stringify(frame) }, + client.generation, + ); +} + +test("a rate-limited OK rejects the named publish and arms the shared gate", async () => { + reset(); + const client = connectedClient(); + const eventId = "a".repeat(64); + const settled = armPendingPublish(client, eventId); + + await deliver(client, [ + "OK", + eventId, + false, + "rate-limited: quota exceeded; retry in 4s", + ]); + + const outcome = await settled; + assert.equal(outcome.status, "rejected"); + assert.match(outcome.error.message, /rate-limited/); + assert.equal(client.publishes.has(eventId), false); + assert.equal(isRateLimited(), true); +}); + +test("an ordinary OK rejection does not arm the shared gate", async () => { + reset(); + const client = connectedClient(); + const eventId = "b".repeat(64); + const settled = armPendingPublish(client, eventId); + + await deliver(client, ["OK", eventId, false, "invalid: bad signature"]); + + assert.equal((await settled).status, "rejected"); + assert.equal(isRateLimited(), false); +}); + +test("publish waits outside its timeout and pending state, then sends and settles", async () => { + reset(); + activateRateLimit(4); + const client = connectedClient(); + const event = { id: "c".repeat(64), kind: 5 }; + + const published = client.publishEvent(event); + await Promise.resolve(); + await Promise.resolve(); + + assert.equal(sends.length, 0, "EVENT must remain unsent while gated"); + assert.equal( + client.publishes.has(event.id), + false, + "publish timeout and pending ownership start only after the gate expires", + ); + assert.equal(pendingTimers.size, 1, "only the gate timer should be armed"); + + tickTo(4_000); + await Promise.resolve(); + await Promise.resolve(); + + assert.equal(sends.length, 1); + assert.deepEqual(JSON.parse(sends[0].message.data), ["EVENT", event]); + assert.equal(client.publishes.has(event.id), true); + + await deliver(client, ["OK", event.id, true, ""]); + await published; + assert.equal(client.publishes.has(event.id), false); +}); + +test("a disconnected client does not send after the shared gate expires", async () => { + reset(); + activateRateLimit(4); + const client = connectedClient(); + const event = { id: "d".repeat(64), kind: 5 }; + + const published = client.publishEvent(event); + await Promise.resolve(); + client.disconnect(); + tickTo(4_000); + + await assert.rejects(published, /not connected/); + assert.equal(sends.length, 0); + assert.equal(client.publishes.has(event.id), false); +}); diff --git a/desktop/src/shared/api/relayRateLimitGate.ts b/desktop/src/shared/api/relayRateLimitGate.ts index 0af3eed7d..040bedae7 100644 --- a/desktop/src/shared/api/relayRateLimitGate.ts +++ b/desktop/src/shared/api/relayRateLimitGate.ts @@ -87,6 +87,24 @@ export function activateRateLimit(retryInSeconds: number | null): void { }, durationMs); } +/** + * Arms the gate if `message` is a relay back-pressure signal, and reports + * whether it was. + * + * The relay marks back-pressure with a `rate-limited:` prefix on whichever + * frame carries the rejection — `NOTICE` for connection-scoped limits, `OK` + * for one addressed to a single event, `CLOSED` for a subscription. Every + * inbound path needs the same test, so it lives here with the gate rather than + * being re-derived per call site. + */ +export function activateRateLimitIfSignalled(message: string): boolean { + if (!message.startsWith("rate-limited:")) { + return false; + } + activateRateLimit(parseRateLimitHint(message)); + return true; +} + /** Returns `true` when the relay has signalled back-pressure and the gate is active. */ export function isRateLimited(): boolean { return expiresAt !== null && Date.now() < expiresAt; diff --git a/docs/push-gateway-deployment.md b/docs/push-gateway-deployment.md index e99d0a47a..bcd8dc1c0 100644 --- a/docs/push-gateway-deployment.md +++ b/docs/push-gateway-deployment.md @@ -36,6 +36,8 @@ The gateway stores APNs tokens encrypted in PostgreSQL. Database backups therefo ## PostgreSQL and replicas +The gateway's dedicated pool does not consume the relay-oriented `BUZZ_DB_LOCK_TIMEOUT_MS`, `BUZZ_DB_IDLE_TXN_TIMEOUT_MS`, or `BUZZ_DB_STATEMENT_TIMEOUT_MS` settings. Its session-timeout policy remains separate from the `buzz-db` writer policy and must be designed and rolled out independently. + All replicas must share one PostgreSQL database. Delivery authority, replay admission, and endpoint quota reservation are transactional there, so replica count does not multiply the abuse ceiling. The gateway owns a scoped migration history under `crates/buzz-push-gateway/migrations`; it creates only the six `push_gateway_*` authority tables plus SQLx's migration-history table and never runs relay migrations. The Helm chart runs a single pre-install/pre-upgrade migration Job using `migration.existingSecret`; that secret contains a DDL-capable `DATABASE_URL`. The URL MUST name a dedicated gateway database, not the relay database: SQLx stores its `_sqlx_migrations` history in `public`, so sharing a database would collide with another application's migration history. `migration.runtimeDatabaseRole` names an existing LOGIN role (the default is `buzz_push_gateway_runtime`) used by runtime `DATABASE_URL`. After scoped migrations, the Job revokes database `CREATE` from that role and schema `CREATE` from both `PUBLIC` and the role, then grants only database `CONNECT`, schema `USAGE`, and `SELECT, INSERT, UPDATE, DELETE` on the six gateway tables. The migration role must own the database/schema objects or otherwise be allowed to issue those grants; it is never provided to runtime replicas. Readiness rejects an empty/partial schema, missing DML, or a runtime role that retains database/schema `CREATE`. Helm waits for the migration hook before updating replicas, so rolling deployments never race unconditional startup migration. Readiness must be removed from load-balancer service endpoints before terminating a pod. diff --git a/migrations/0072_replica_heartbeat_vacuum_truncate.sql b/migrations/0072_replica_heartbeat_vacuum_truncate.sql new file mode 100644 index 000000000..479b04948 --- /dev/null +++ b/migrations/0072_replica_heartbeat_vacuum_truncate.sql @@ -0,0 +1,7 @@ +-- replica_heartbeat is a single-row table updated continuously by every relay +-- pod. Autovacuum heap truncation briefly takes ACCESS EXCLUSIVE on the table; +-- replaying that lock on a hot standby can cancel concurrent heartbeat reads. +-- The table cannot reclaim meaningful disk space by truncating one page, so +-- disable only the truncation phase while retaining normal autovacuum cleanup. + +ALTER TABLE replica_heartbeat SET (vacuum_truncate = false); diff --git a/migrations/0073_channel_roster_snapshot_fence.sql b/migrations/0073_channel_roster_snapshot_fence.sql new file mode 100644 index 000000000..cdc7bc4b9 --- /dev/null +++ b/migrations/0073_channel_roster_snapshot_fence.sql @@ -0,0 +1,76 @@ +-- Prevent mixed-version relay pods from publishing a stale NIP-29 member +-- snapshot after a newer canonical roster has been committed. +-- +-- Old binaries already serialize kind 39002 replacement on the replacement +-- advisory key. This trigger adds the channel-membership key at INSERT time, +-- after that canonical key, and validates every p tag against the current +-- active membership set and roles. New binaries take both keys in the same +-- order before capture and replacement. Thus old and new writers remain +-- compatible during a rolling deploy. +CREATE OR REPLACE FUNCTION guard_channel_roster_snapshot() +RETURNS TRIGGER AS $$ +DECLARE + canonical_members TEXT[]; + snapshot_members TEXT[]; +BEGIN + IF NEW.kind <> 39002 OR NEW.channel_id IS NULL THEN + RETURN NEW; + END IF; + + PERFORM pg_advisory_xact_lock(hashtextextended( + 'buzz_channel_membership:' || NEW.community_id::text || ':' || NEW.channel_id::text, + 0 + )); + + SELECT COALESCE( + array_agg(encode(cm.pubkey, 'hex') || ':' || cm.role::text ORDER BY cm.pubkey), + ARRAY[]::TEXT[] + ) + INTO canonical_members + FROM channel_members cm + WHERE cm.community_id = NEW.community_id + AND cm.channel_id = NEW.channel_id + AND cm.removed_at IS NULL; + + -- A roster is canonical only when every p tag uses the emitted four-field + -- shape, contains a 32-byte hex pubkey and valid authoritative role, has no + -- duplicate members, and exactly matches the active membership rows. + IF EXISTS ( + SELECT 1 + FROM jsonb_array_elements(NEW.tags) AS roster_tag(tag_json) + WHERE roster_tag.tag_json->>0 = 'p' + AND ( + jsonb_array_length(roster_tag.tag_json) <> 4 + OR COALESCE(roster_tag.tag_json->>1, '') !~ '^[0-9a-fA-F]{64}$' + OR roster_tag.tag_json->>2 <> '' + OR COALESCE(roster_tag.tag_json->>3, '') NOT IN ('owner', 'admin', 'bot', 'member', 'guest') + ) + ) THEN + RAISE EXCEPTION 'kind 39002 roster contains an invalid p tag' + USING ERRCODE = '23514'; + END IF; + + SELECT COALESCE( + array_agg( + lower((roster_tag.tag_json->>1)) || ':' || (roster_tag.tag_json->>3) + ORDER BY decode((roster_tag.tag_json->>1), 'hex') + ), + ARRAY[]::TEXT[] + ) + INTO snapshot_members + FROM jsonb_array_elements(NEW.tags) AS roster_tag(tag_json) + WHERE roster_tag.tag_json->>0 = 'p'; + + IF snapshot_members IS DISTINCT FROM canonical_members THEN + RAISE EXCEPTION 'kind 39002 roster does not match canonical channel membership' + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events; +CREATE TRIGGER trg_events_guard_channel_roster_snapshot + BEFORE INSERT ON events + FOR EACH ROW EXECUTE FUNCTION guard_channel_roster_snapshot(); diff --git a/migrations/0074_private_managed_agent_fts.sql b/migrations/0074_private_managed_agent_fts.sql new file mode 100644 index 000000000..87caf54ac --- /dev/null +++ b/migrations/0074_private_managed_agent_fts.sql @@ -0,0 +1,46 @@ +-- NIP-PMA kind:30194 carries the owner's private managed-agent payload +-- (agent nsec, env vars, prompt) as NIP-44 ciphertext and is author-only. +-- Exclude it from full-text search without changing the search policy of +-- existing installations. Fresh installs are already safe via the positive +-- allowlist from migration 0008; this closes the brownfield gap where a +-- populated database still runs the legacy negative skip-set (0001/0005) +-- and would tokenize the ciphertext into search_tsv. +-- +-- Same shape as 0014 (kind:30350): PostgreSQL cannot alter a generated +-- expression in place, so capture the current expression, drop the column, +-- and re-add it wrapped with the new exclusion. Every other kind keeps +-- whatever policy the database had before. +-- +-- Operational cost: this is not free on large databases. DROP COLUMN + +-- ADD ... GENERATED ... STORED rewrites the entire events heap and then +-- rebuilds the GIN index, all under an ACCESS EXCLUSIVE lock inside the +-- migration transaction (CREATE INDEX CONCURRENTLY is not possible +-- here), with no lock_timeout. Expect relay downtime proportional to +-- the size of events. The index is recreated from the stock definition +-- below; any non-stock indexes or storage parameters on search_tsv are +-- not captured or replayed. 0014 set this precedent on smaller tables; +-- operators with large brownfield databases should schedule a window. +DO $$ +DECLARE + existing_expression TEXT; +BEGIN + SELECT pg_get_expr(d.adbin, d.adrelid) + INTO existing_expression + FROM pg_attrdef d + JOIN pg_attribute a + ON a.attrelid = d.adrelid + AND a.attnum = d.adnum + WHERE d.adrelid = 'events'::regclass + AND a.attname = 'search_tsv'; + + IF existing_expression IS NULL THEN + RAISE EXCEPTION 'events.search_tsv generated expression not found'; + END IF; + + ALTER TABLE events DROP COLUMN search_tsv; + EXECUTE format( + 'ALTER TABLE events ADD COLUMN search_tsv TSVECTOR GENERATED ALWAYS AS (CASE WHEN kind = 30194 THEN NULL::tsvector ELSE (%s) END) STORED', + existing_expression + ); + CREATE INDEX idx_events_search_tsv ON events USING GIN (search_tsv); +END $$; diff --git a/schema/schema.sql b/schema/schema.sql index c8ff683d1..b8fff5149 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1193,6 +1193,85 @@ AFTER INSERT ON events DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION refresh_channel_ttl_after_event_insert(); +-- Channel roster snapshot fence (keep in sync with migrations/0032). +-- Prevent mixed-version relay pods from publishing a stale NIP-29 member +-- snapshot after a newer canonical roster has been committed. +-- +-- Old binaries already serialize kind 39002 replacement on the replacement +-- advisory key. This trigger adds the channel-membership key at INSERT time, +-- after that canonical key, and validates every p tag against the current +-- active membership set and roles. New binaries take both keys in the same +-- order before capture and replacement. Thus old and new writers remain +-- compatible during a rolling deploy. +CREATE OR REPLACE FUNCTION guard_channel_roster_snapshot() +RETURNS TRIGGER AS $$ +DECLARE + canonical_members TEXT[]; + snapshot_members TEXT[]; +BEGIN + IF NEW.kind <> 39002 OR NEW.channel_id IS NULL THEN + RETURN NEW; + END IF; + + PERFORM pg_advisory_xact_lock(hashtextextended( + 'buzz_channel_membership:' || NEW.community_id::text || ':' || NEW.channel_id::text, + 0 + )); + + SELECT COALESCE( + array_agg(encode(cm.pubkey, 'hex') || ':' || cm.role::text ORDER BY cm.pubkey), + ARRAY[]::TEXT[] + ) + INTO canonical_members + FROM channel_members cm + WHERE cm.community_id = NEW.community_id + AND cm.channel_id = NEW.channel_id + AND cm.removed_at IS NULL; + + -- A roster is canonical only when every p tag uses the emitted four-field + -- shape, contains a 32-byte hex pubkey and valid authoritative role, has no + -- duplicate members, and exactly matches the active membership rows. + IF EXISTS ( + SELECT 1 + FROM jsonb_array_elements(NEW.tags) AS roster_tag(tag_json) + WHERE roster_tag.tag_json->>0 = 'p' + AND ( + jsonb_array_length(roster_tag.tag_json) <> 4 + OR COALESCE(roster_tag.tag_json->>1, '') !~ '^[0-9a-fA-F]{64}$' + OR roster_tag.tag_json->>2 <> '' + OR COALESCE(roster_tag.tag_json->>3, '') NOT IN ('owner', 'admin', 'bot', 'member', 'guest') + ) + ) THEN + RAISE EXCEPTION 'kind 39002 roster contains an invalid p tag' + USING ERRCODE = '23514'; + END IF; + + SELECT COALESCE( + array_agg( + lower((roster_tag.tag_json->>1)) || ':' || (roster_tag.tag_json->>3) + ORDER BY decode((roster_tag.tag_json->>1), 'hex') + ), + ARRAY[]::TEXT[] + ) + INTO snapshot_members + FROM jsonb_array_elements(NEW.tags) AS roster_tag(tag_json) + WHERE roster_tag.tag_json->>0 = 'p'; + + IF snapshot_members IS DISTINCT FROM canonical_members THEN + RAISE EXCEPTION 'kind 39002 roster does not match canonical channel membership' + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events; +CREATE TRIGGER trg_events_guard_channel_roster_snapshot + BEFORE INSERT ON events + FOR EACH ROW EXECUTE FUNCTION guard_channel_roster_snapshot(); + + -- Replica-fence floor guard (keep in sync with migrations/0021). A deferred -- constraint trigger re-checks, inside COMMIT processing, that channel-bearing -- event rows are no older than `buzz.created_at_floor` seconds before commit @@ -1323,6 +1402,8 @@ CREATE TABLE replica_heartbeat ( id smallint PRIMARY KEY CHECK (id = 1), epoch uuid NOT NULL DEFAULT gen_random_uuid(), token bigint NOT NULL DEFAULT 0 +) WITH ( + vacuum_truncate = false ); INSERT INTO replica_heartbeat (id) VALUES (1); diff --git a/scripts/check-schema-drift.mjs b/scripts/check-schema-drift.mjs index 0ecbb2eb8..28bf62ff4 100644 --- a/scripts/check-schema-drift.mjs +++ b/scripts/check-schema-drift.mjs @@ -22,7 +22,7 @@ // refused and no head was ever authored // // Partition children (`CREATE TABLE ... PARTITION OF ...`) are excluded: CI -// creates those with scripts/attach-schema-partitions.sql, not schema.sql. +// creates those with scripts/reconcile-schema-after-pgschema.sql, not schema.sql. // // KNOWN_DRIFT below is a burn-down list, not a permanent exemption. Adding an // entry means you are knowingly shipping a table CI cannot see -- do not, diff --git a/scripts/check-schema-drift.test.mjs b/scripts/check-schema-drift.test.mjs index 708fb05c4..3bf52493f 100644 --- a/scripts/check-schema-drift.test.mjs +++ b/scripts/check-schema-drift.test.mjs @@ -38,7 +38,7 @@ test("IF NOT EXISTS is matched on either side", () => { }); test("partition children are not drift", () => { - // CI creates these with scripts/attach-schema-partitions.sql, so they are + // CI creates these with scripts/reconcile-schema-after-pgschema.sql, so they are // deliberately absent from schema.sql. Counting them would produce dozens of // false positives, one per month of `events_p2026_NN`. assert.deepEqual( diff --git a/scripts/create-required-extensions.sql b/scripts/create-required-extensions.sql index 72b4d37f3..cc32faff8 100644 --- a/scripts/create-required-extensions.sql +++ b/scripts/create-required-extensions.sql @@ -15,5 +15,5 @@ -- `function digest(text, unknown) does not exist`. Nothing caught it because -- the only suite exercising that path had never run in CI. -- --- Run this after `pgschema apply`, alongside attach-schema-partitions.sql. +-- Run this after `pgschema apply`, alongside reconcile-schema-after-pgschema.sql. CREATE EXTENSION IF NOT EXISTS pgcrypto; diff --git a/scripts/e2e-large-channel-roster.sh b/scripts/e2e-large-channel-roster.sh new file mode 100755 index 000000000..9f82f9bd3 --- /dev/null +++ b/scripts/e2e-large-channel-roster.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Prove a channel member past the historical 1,000-row boundary is present in +# relay-served discovery, can write, and survives an authoritative republish. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +: "${DATABASE_URL:?set DATABASE_URL to the isolated relay database}" +: "${BUZZ_RELAY_URL:=http://localhost:3030}" +: "${RELAY_URL:=ws://localhost:3030}" +: "${BUZZ_RELAY_PRIVATE_KEY:=0000000000000000000000000000000000000000000000000000000000000001}" +export BUZZ_RELAY_URL RELAY_URL BUZZ_RELAY_PRIVATE_KEY +unset BUZZ_AUTH_TAG + +for binary in buzz buzz-admin; do + resolved="$(command -v "$binary" || true)" + [[ "$resolved" == "$REPO_ROOT/target/release/$binary" ]] || { + echo "error: $binary must resolve to $REPO_ROOT/target/release/$binary (got ${resolved:-not found})" >&2 + exit 1 + } +done +command -v jq >/dev/null || { echo "error: jq is required" >&2; exit 1; } +command -v psql >/dev/null || { echo "error: psql is required" >&2; exit 1; } + +key_field() { + awk -v label="$1" '$1 == label && $2 == "key:" { print $3 }' +} + +OWNER_GEN="$(buzz-admin generate-key)" +OWNER_SK="$(printf '%s\n' "$OWNER_GEN" | key_field Secret)" +export BUZZ_PRIVATE_KEY="$OWNER_SK" + +CHANNEL="$(buzz channels create \ + --name "roster-boundary-$$" --type stream --visibility open | jq -er '.channel_id')" + +LATE_GEN="$(buzz-admin generate-key)" +LATE_SK="$(printf '%s\n' "$LATE_GEN" | key_field Secret)" +LATE_PUBKEY="$(printf '%s\n' "$LATE_GEN" | key_field Public)" + +# The creator is roster position 1. Add 1,499 fixtures followed by the real +# late identity at position 1,501. A final API-added member forces the relay to +# emit a fresh discovery snapshot through the normal membership-change path. +psql "$DATABASE_URL" -v ON_ERROR_STOP=1 \ + --set=channel="$CHANNEL" --set=late_pubkey="$LATE_PUBKEY" <<'SQL' +WITH target AS ( + SELECT community_id, id + FROM channels + WHERE id = :'channel'::uuid +), fixtures AS ( + SELECT n, decode(lpad(to_hex(n + 65536), 64, '0'), 'hex') AS pubkey + FROM generate_series(1, 1499) AS n +) +INSERT INTO channel_members (community_id, channel_id, pubkey, role, joined_at) +SELECT target.community_id, target.id, fixtures.pubkey, 'member', + NOW() + fixtures.n * interval '1 millisecond' +FROM target CROSS JOIN fixtures; + +INSERT INTO channel_members (community_id, channel_id, pubkey, role, joined_at) +SELECT community_id, id, decode(:'late_pubkey', 'hex'), 'member', + NOW() + interval '2 seconds' +FROM channels +WHERE id = :'channel'::uuid; +SQL + +TRIGGER_GEN="$(buzz-admin generate-key)" +TRIGGER_PUBKEY="$(printf '%s\n' "$TRIGGER_GEN" | key_field Public)" +buzz channels add-member --channel "$CHANNEL" --pubkey "$TRIGGER_PUBKEY" --role member >/dev/null + +BEFORE="$(buzz channels members --channel "$CHANNEL")" +BEFORE_COUNT="$(jq 'length' <<<"$BEFORE")" +jq -e --arg pk "$LATE_PUBKEY" 'any(.[]; .pubkey == $pk and .role == "member")' \ + <<<"$BEFORE" >/dev/null +(( BEFORE_COUNT > 1000 )) +printf 'PASS discovery-before-republish channel=%s members=%s late_pubkey=%s\n' \ + "$CHANNEL" "$BEFORE_COUNT" "$LATE_PUBKEY" + +export BUZZ_PRIVATE_KEY="$LATE_SK" +ACTION="$(buzz messages send --channel "$CHANNEL" --content "member-1501-action")" +jq -e '.accepted == true' <<<"$ACTION" >/dev/null +ACTION_ID="$(jq -er '.event_id' <<<"$ACTION")" +buzz messages get --channel "$CHANNEL" --limit 10 \ + | jq -e --arg id "$ACTION_ID" 'any(.[]; .id == $id and .content == "member-1501-action")' >/dev/null +printf 'PASS late-member-action event_id=%s\n' "$ACTION_ID" + +# Targeted roster repair must not replace canonical metadata or admin events. +# Preserve both the event ID and complete tags, including fields this backfill +# command does not know how to rebuild (DM participants, topic, TTL, etc.). +DISCOVERY_BEFORE="$(psql "$DATABASE_URL" -AtX -v ON_ERROR_STOP=1 \ + --set=channel="$CHANNEL" <<'SQL' +SELECT jsonb_object_agg(kind::text, jsonb_build_object( + 'id', encode(id, 'hex'), + 'tags', tags +) ORDER BY kind)::text +FROM events +WHERE channel_id = :'channel'::uuid + AND kind IN (39000, 39001) + AND deleted_at IS NULL; +SQL +)" +jq -e 'has("39000") and has("39001")' <<<"$DISCOVERY_BEFORE" >/dev/null + +DATABASE_URL="$DATABASE_URL" RELAY_URL="$RELAY_URL" \ + buzz-admin reconcile-channels --channel "$CHANNEL" >/dev/null + +DISCOVERY_AFTER="$(psql "$DATABASE_URL" -AtX -v ON_ERROR_STOP=1 \ + --set=channel="$CHANNEL" <<'SQL' +SELECT jsonb_object_agg(kind::text, jsonb_build_object( + 'id', encode(id, 'hex'), + 'tags', tags +) ORDER BY kind)::text +FROM events +WHERE channel_id = :'channel'::uuid + AND kind IN (39000, 39001) + AND deleted_at IS NULL; +SQL +)" +[[ "$DISCOVERY_AFTER" == "$DISCOVERY_BEFORE" ]] +printf 'PASS targeted-repair-preserves-metadata-and-admin-events channel=%s\n' "$CHANNEL" + +AFTER="$(buzz channels members --channel "$CHANNEL")" +AFTER_COUNT="$(jq 'length' <<<"$AFTER")" +jq -e --arg pk "$LATE_PUBKEY" 'any(.[]; .pubkey == $pk and .role == "member")' \ + <<<"$AFTER" >/dev/null +(( AFTER_COUNT == BEFORE_COUNT )) +printf 'PASS discovery-after-republish channel=%s members=%s late_pubkey=%s\n' \ + "$CHANNEL" "$AFTER_COUNT" "$LATE_PUBKEY" diff --git a/scripts/attach-schema-partitions.sql b/scripts/reconcile-schema-after-pgschema.sql similarity index 83% rename from scripts/attach-schema-partitions.sql rename to scripts/reconcile-schema-after-pgschema.sql index 04b358df6..512dc5927 100644 --- a/scripts/attach-schema-partitions.sql +++ b/scripts/reconcile-schema-after-pgschema.sql @@ -1,11 +1,10 @@ --- Attach partition child tables after pgschema apply. +-- Reconcile schema details that pgschema does not preserve. -- --- pgschema currently emits existing partition children as standalone CREATE TABLE --- statements when applying schema/schema.sql in CI. The tables exist, but they --- are not attached to their partitioned parents, so inserts into events or --- delivery_log fail with "no partition of relation ... found for row". Keep this --- idempotent: raw psql/schema.sql already attaches these partitions, while --- pgschema-created schemas need this repair step. +-- pgschema reconciles DDL, but it does not execute seed DML or preserve every +-- table storage parameter from schema/schema.sql. It also currently emits +-- partition children as standalone CREATE TABLE statements. Every pgschema +-- apply caller must run this idempotent script so fresh bootstraps converge on +-- the same live database contract as migration-managed databases. DO $$ BEGIN @@ -16,8 +15,7 @@ BEGIN ) THEN -- pgschema may copy parent triggers onto standalone children. Drop -- those copies before ATTACH; PostgreSQL recreates inherited parent - -- triggers while attaching and rejects same-named child triggers - -- (both the push-match trigger and the replica-fence floor guard). + -- triggers while attaching and rejects same-named child triggers. DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_past; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_past; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_past; @@ -26,6 +24,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p_past; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p_past; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p_past; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p_past; ALTER TABLE events ATTACH PARTITION events_p_past FOR VALUES FROM (MINVALUE) TO ('2026-01-01'); END IF; @@ -43,6 +42,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p2026_01; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p2026_01; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p2026_01; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_01; ALTER TABLE events ATTACH PARTITION events_p2026_01 FOR VALUES FROM ('2026-01-01') TO ('2026-02-01'); END IF; @@ -60,6 +60,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p2026_02; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p2026_02; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p2026_02; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_02; ALTER TABLE events ATTACH PARTITION events_p2026_02 FOR VALUES FROM ('2026-02-01') TO ('2026-03-01'); END IF; @@ -77,6 +78,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p2026_03; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p2026_03; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p2026_03; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_03; ALTER TABLE events ATTACH PARTITION events_p2026_03 FOR VALUES FROM ('2026-03-01') TO ('2026-04-01'); END IF; @@ -94,6 +96,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p2026_04; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p2026_04; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p2026_04; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_04; ALTER TABLE events ATTACH PARTITION events_p2026_04 FOR VALUES FROM ('2026-04-01') TO ('2026-05-01'); END IF; @@ -111,6 +114,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p2026_05; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p2026_05; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p2026_05; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_05; ALTER TABLE events ATTACH PARTITION events_p2026_05 FOR VALUES FROM ('2026-05-01') TO ('2026-06-01'); END IF; @@ -128,6 +132,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p2026_06; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p2026_06; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p2026_06; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p2026_06; ALTER TABLE events ATTACH PARTITION events_p2026_06 FOR VALUES FROM ('2026-06-01') TO ('2026-07-01'); END IF; @@ -145,6 +150,7 @@ BEGIN DROP TRIGGER IF EXISTS trg_events_guard_nip_rs_hard_delete ON events_p_future; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_nip_rs ON events_p_future; DROP TRIGGER IF EXISTS trg_events_purge_soft_deleted_buzz_mesh_status ON events_p_future; + DROP TRIGGER IF EXISTS trg_events_guard_channel_roster_snapshot ON events_p_future; ALTER TABLE events ATTACH PARTITION events_p_future FOR VALUES FROM ('2026-07-01') TO (MAXVALUE); END IF; @@ -222,3 +228,32 @@ BEGIN FOR VALUES FROM ('2026-07-01') TO (MAXVALUE); END IF; END $$; + +-- pgschema reconciles DDL but does not apply seed DML or table storage +-- parameters from schema/schema.sql. Restore those parts of the desired-state +-- contract explicitly and fail the bootstrap if the live catalog disagrees. +ALTER TABLE replica_heartbeat SET (vacuum_truncate = false); + +INSERT INTO replica_heartbeat (id) VALUES (1) +ON CONFLICT (id) DO NOTHING; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_class AS relation + JOIN pg_namespace AS namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = current_schema() + AND relation.relname = 'replica_heartbeat' + AND COALESCE( + relation.reloptions @> ARRAY['vacuum_truncate=false']::text[], + false + ) + ) THEN + RAISE EXCEPTION 'replica_heartbeat must disable vacuum truncation after pgschema apply'; + END IF; + + IF (SELECT count(*) FROM replica_heartbeat WHERE id = 1) <> 1 THEN + RAISE EXCEPTION 'replica_heartbeat must contain its singleton row after pgschema apply'; + END IF; +END $$; diff --git a/scripts/run-real-shell-e2e.sh b/scripts/run-real-shell-e2e.sh index 73ae14243..a6f9699e4 100755 --- a/scripts/run-real-shell-e2e.sh +++ b/scripts/run-real-shell-e2e.sh @@ -182,7 +182,7 @@ start_relay_nohup() { ./scripts/ci-prefetch-hermit-pkg.sh pgschema ./bin/pgschema apply --file schema/schema.sql --auto-approve docker compose -p "${project}" -f "${compose_file}" exec -T postgres \ - psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql + psql -U buzz -d buzz -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql else warn "Backing services owned by ${owner}; reusing their database WITHOUT schema reset." fi diff --git a/scripts/start-relay-for-tests.sh b/scripts/start-relay-for-tests.sh index f98e84e8f..defc8ddd9 100755 --- a/scripts/start-relay-for-tests.sh +++ b/scripts/start-relay-for-tests.sh @@ -124,7 +124,7 @@ export PGSCHEMA_PLAN_PASSWORD=buzz_dev docker exec -i -e PGPASSWORD="${PGPASSWORD}" buzz-postgres \ psql -U "${PGUSER}" -d "${PGDATABASE}" -v ON_ERROR_STOP=1 < scripts/create-required-extensions.sql docker exec -i -e PGPASSWORD="${PGPASSWORD}" buzz-postgres \ - psql -U "${PGUSER}" -d "${PGDATABASE}" -v ON_ERROR_STOP=1 < scripts/attach-schema-partitions.sql + psql -U "${PGUSER}" -d "${PGDATABASE}" -v ON_ERROR_STOP=1 < scripts/reconcile-schema-after-pgschema.sql ok "Schema applied" # ── Seed the deployment community ────────────────────────────────────────────