feat(db): Database connection multiplexing for serverless environments (#600) (#651)

* feat(db): connection multiplexing for serverless environments (#600)

Route serverless DB access through a transaction-pooling proxy (RDS Proxy
or PgBouncer) so a small set of backend connections is multiplexed across
many concurrent function invocations, preventing connection exhaustion.

- backend/shared/db/serverlessPool.ts: transaction-pooling adapter with
  IAM/SCRAM-256 auth, credential refresh, withClient/withTransaction
  helpers, and abandoned-connection leak detection (>30s force-close).
- backend/serverless/dbConfig.ts + withDatabase.ts: env-driven pool config
  (RDS IAM token provider) and a Lambda wrapper that releases the client in
  a finally block after every invocation.
- backend/monitoring/connectionPoolMetrics.ts: Prometheus pool/leak metrics
  and structured leak alerting.
- infra/terraform/{rds_proxy,pgbouncer}.tf: proxy provisioning (max ~50
  pooled connections serving 500+ functions, transaction pooling).
- docker-compose.yml + .env.example: local PgBouncer + Postgres for parity;
  all credentials read from a gitignored .env (no hardcoded secrets).

Closes #600

* fix(deps): repair malformed package.json blocking install

The dependencies block had a duplicate "zustand" entry and missing
commas, producing invalid JSON that breaks `npm install` in CI. Keep a
single zustand ^5.0.0 and the redis ^4.6.7 entry.

Author: jadonamite

.env.example

@@ -24,3 +24,13 @@ COMPOSE_PORT_SOROBAN=8000
 COMPOSE_PORT_BACKEND=3000
 COMPOSE_PORT_ML=8001
 COMPOSE_PORT_EXPO=8081
+
+# Issue #600: serverless DB connection multiplexing via PgBouncer.
+# Point the app at PgBouncer (:6432), not Postgres directly.
+DB_PROXY_HOST=localhost
+DB_PROXY_PORT=6432
+DB_PROXY_AUTH_MODE=scram-256
+DB_PROXY_TXN_POOLING=true
+DB_PROXY_PREPARED_STATEMENTS=true
+DB_PROXY_MAX_CONN=50
+DB_LEAK_THRESHOLD_MS=30000

backend/monitoring/connectionPoolMetrics.ts

@@ -0,0 +1,77 @@
+/**
+ * Connection-pool Prometheus metrics and leak detection.
+ *
+ * Issue #600: surface multiplexed-pool health (active/idle/waiting, checked-out
+ * clients, and leaked-connection counts) so abandoned connections are visible
+ * and alertable. Mirrors the lightweight scrape style of viewFreshnessMetric.
+ */
+
+import type {
+  ServerlessConnectionPool,
+  CheckoutRecord,
+} from '../shared/db/serverlessPool';
+
+/** Running totals that persist across scrapes for counter-type metrics. */
+interface LeakCounters {
+  leakedTotal: number;
+}
+
+/**
+ * Render the pool stats as Prometheus exposition text. Counters
+ * (`*_total`) accumulate; gauges reflect the instantaneous pool state.
+ */
+export function renderPoolMetrics(pool: ServerlessConnectionPool): string {
+  const s = pool.stats();
+  const lines = [
+    '# HELP subtrackr_db_pool_connections Pooled connections to the DB proxy by state.',
+    '# TYPE subtrackr_db_pool_connections gauge',
+    `subtrackr_db_pool_connections{state="total"} ${s.total}`,
+    `subtrackr_db_pool_connections{state="idle"} ${s.idle}`,
+    `subtrackr_db_pool_connections{state="waiting"} ${s.waiting}`,
+    `subtrackr_db_pool_connections{state="checked_out"} ${s.checkedOut}`,
+    '# HELP subtrackr_db_pool_leaked_total Connections force-closed after exceeding the leak threshold.',
+    '# TYPE subtrackr_db_pool_leaked_total counter',
+    `subtrackr_db_pool_leaked_total ${s.leakedTotal}`,
+  ];
+  return lines.join('\n') + '\n';
+}
+
+/**
+ * Build an HTTP `/metrics` handler for the serverless pool. Generic request /
+ * response shape so it mounts in any Node.js HTTP server.
+ */
+export function createPoolMetricsHandler(pool: ServerlessConnectionPool) {
+  return function handleMetrics(
+    _req: unknown,
+    res: { setHeader(name: string, value: string): void; end(body: string): void },
+  ): void {
+    res.setHeader('Content-Type', 'text/plain; version=0.0.4; charset=utf-8');
+    res.end(renderPoolMetrics(pool));
+  };
+}
+
+/**
+ * Attach structured leak logging/alerting to a pool. Each force-closed
+ * abandoned connection is logged with its age and origin, and an optional
+ * `onLeak` sink (e.g. CloudWatch metric, PagerDuty) is invoked.
+ */
+export function installLeakDetection(
+  pool: ServerlessConnectionPool,
+  onLeak?: (info: { origin: string; ageMs: number }) => void,
+): LeakCounters {
+  const counters: LeakCounters = { leakedTotal: 0 };
+  pool.setLeakHandler((record: CheckoutRecord, ageMs: number) => {
+    counters.leakedTotal += 1;
+    console.error(
+      JSON.stringify({
+        level: 'error',
+        event: 'db_connection_leak',
+        origin: record.origin,
+        ageMs,
+        message: 'Abandoned database connection force-closed',
+      }),
+    );
+    onLeak?.({ origin: record.origin, ageMs });
+  });
+  return counters;
+}

backend/serverless/dbConfig.ts

@@ -0,0 +1,89 @@
+/**
+ * Serverless database configuration helpers.
+ *
+ * Issue #600: wires the serverless connection pool to the right proxy endpoint
+ * and authentication mode for each environment.
+ *
+ *   - Production (AWS): RDS Proxy with IAM authentication. The "password" is a
+ *     short-lived signed token regenerated on every connect.
+ *   - Self-hosted / staging: PgBouncer with SCRAM-SHA-256.
+ *   - Local dev: PgBouncer (docker-compose) with a static password.
+ */
+
+import {
+  getServerlessPool,
+  type ServerlessConnectionPool,
+  type ServerlessPoolConfig,
+  type ProxyAuthMode,
+} from '../shared/db/serverlessPool';
+
+/**
+ * Build an RDS IAM auth-token provider. The token is signed with the AWS SDK's
+ * RDS Signer and is valid for ~15 minutes, so we regenerate it on each connect.
+ *
+ * `@aws-sdk/rds-signer` is imported lazily so non-AWS deployments never need it.
+ */
+export function createRdsIamCredentialProvider(opts: {
+  hostname: string;
+  port: number;
+  username: string;
+  region?: string;
+}): () => Promise<string> {
+  return async () => {
+    const { Signer } = (await import('@aws-sdk/rds-signer')) as {
+      Signer: new (cfg: {
+        hostname: string;
+        port: number;
+        username: string;
+        region?: string;
+      }) => { getAuthToken(): Promise<string> };
+    };
+    const signer = new Signer({
+      hostname: opts.hostname,
+      port: opts.port,
+      username: opts.username,
+      region: opts.region ?? process.env['AWS_REGION'],
+    });
+    return signer.getAuthToken();
+  };
+}
+
+/**
+ * Resolve the serverless pool configuration from the environment. Centralised
+ * so every Lambda handler gets identical, correct pooling behaviour.
+ */
+export function resolveServerlessPoolConfig(): ServerlessPoolConfig {
+  const authMode = (process.env['DB_PROXY_AUTH_MODE'] as ProxyAuthMode) || 'scram-256';
+  const host = process.env['DB_PROXY_HOST'] ?? process.env['DB_HOST'] ?? 'localhost';
+  const port = Number(process.env['DB_PROXY_PORT'] ?? 6432);
+  const user = process.env['DB_USER'] ?? 'subtrackr_app';
+
+  const base: ServerlessPoolConfig = {
+    authMode,
+    host,
+    port,
+    user,
+    database: process.env['DB_NAME'] ?? 'subtrackr',
+    transactionPooling: process.env['DB_PROXY_TXN_POOLING'] !== 'false',
+    preparedStatements: process.env['DB_PROXY_PREPARED_STATEMENTS'] === 'true',
+    maxPooledConnections: Number(process.env['DB_PROXY_MAX_CONN'] ?? 50),
+    leakDetectionThresholdMs: Number(process.env['DB_LEAK_THRESHOLD_MS'] ?? 30_000),
+    ssl: process.env['DB_SSL'] === 'true' ? { rejectUnauthorized: true } : undefined,
+  };
+
+  if (authMode === 'iam') {
+    base.credentialProvider = createRdsIamCredentialProvider({
+      hostname: host,
+      port,
+      username: user,
+      region: process.env['AWS_REGION'],
+    });
+  }
+
+  return base;
+}
+
+/** Get the shared serverless pool configured from the environment. */
+export function getConfiguredServerlessPool(): ServerlessConnectionPool {
+  return getServerlessPool(resolveServerlessPoolConfig());
+}

backend/serverless/withDatabase.ts

@@ -0,0 +1,58 @@
+/**
+ * Lambda handler adaptation for pooled database access.
+ *
+ * Issue #600 acceptance criteria: "db.release() called after each invocation
+ * via finally block." This wrapper makes that guarantee structural — handlers
+ * receive a per-invocation client (or transaction) and the release happens in
+ * a finally regardless of success, throw, or timeout.
+ *
+ * Usage:
+ *
+ *   export const handler = withDatabase(async (event, ctx, db) => {
+ *     const { rows } = await db.query('SELECT 1');
+ *     return { statusCode: 200, body: JSON.stringify(rows) };
+ *   });
+ */
+
+import type { PoolClient } from '../shared/db/serverlessPool';
+import { getConfiguredServerlessPool } from './dbConfig';
+
+/** Minimal generic Lambda handler signature (provider-agnostic). */
+export type LambdaHandler<Event = unknown, Context = unknown, Result = unknown> = (
+  event: Event,
+  context: Context,
+) => Promise<Result>;
+
+export type DatabaseHandler<Event = unknown, Context = unknown, Result = unknown> = (
+  event: Event,
+  context: Context,
+  client: PoolClient,
+) => Promise<Result>;
+
+export interface WithDatabaseOptions {
+  /** Wrap the handler body in a single transaction. Default: false. */
+  transaction?: boolean;
+  /** Diagnostic label used in leak-detection logs. */
+  origin?: string;
+}
+
+/**
+ * Wrap a Lambda handler so it runs with a pooled client that is always
+ * released after the invocation. The underlying pool is a warm-reused
+ * singleton, so the proxy connection is multiplexed across invocations.
+ */
+export function withDatabase<Event = unknown, Context = unknown, Result = unknown>(
+  handler: DatabaseHandler<Event, Context, Result>,
+  options: WithDatabaseOptions = {},
+): LambdaHandler<Event, Context, Result> {
+  const origin = options.origin ?? handler.name ?? 'lambda';
+
+  return async (event, context) => {
+    const pool = getConfiguredServerlessPool();
+    const run = (client: PoolClient) => handler(event, context, client);
+    // withClient / withTransaction both release in a finally block.
+    return options.transaction
+      ? pool.withTransaction(run, origin)
+      : pool.withClient(run, origin);
+  };
+}