diff --git a/README.md b/README.md index 8349e875..1046cd1f 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ The migration is in `migrations/0019_disputes.sql` (rollback: `migrations/0019_d - JSON body parsing plus gateway API key authentication for upstream proxy routes - Per-user global REST rate limiting for authenticated `/api/billing`, `/api/usage`, `/api/developers`, `/api/vault`, and `/api/keys` traffic, with IP fallback for unauthenticated requests - Per-user token-bucket rate limiting for all `/api/quotas` traffic (capacity and refill rate independently configurable via `QUOTA_RATE_LIMIT_CAPACITY` / `QUOTA_RATE_LIMIT_REFILL_RATE`); exceeded requests return `HTTP 429` with a `Retry-After` header and the standardised error envelope +- Quota dependency probe: `GET /api/quotas/health` reports the status of `/api/quotas`'s external dependencies (currently the database) for ops dashboards/alerting, mirroring the `{ status, timestamp, dependencies }` shape of `GET /api/health/dependencies`; no auth required, subject to the same `/api/quotas` rate limit; see [docs/quotas-health-probe.md](./docs/quotas-health-probe.md) - In-memory `VaultRepository` with: - `create(userId, contractId, network)` - `findByUserId(userId, network)` diff --git a/docs/quotas-health-probe.md b/docs/quotas-health-probe.md new file mode 100644 index 00000000..75b12af3 --- /dev/null +++ b/docs/quotas-health-probe.md @@ -0,0 +1,96 @@ +# Quotas Dependency Probe + +**`GET /api/quotas/health`** reports the status of the external dependencies the `/api/quotas` route group relies on — for ops dashboards, alerting, and SRE runbooks. + +This endpoint requires no authentication (it exposes no tenant data, only aggregate dependency status), matching `GET /api/health/dependencies`. It is, however, subject to the same per-user/IP token-bucket rate limit as every other route under `/api/quotas` (see [README.md — What's included](../README.md#whats-included), `QUOTA_RATE_LIMIT_CAPACITY` / `QUOTA_RATE_LIMIT_REFILL_RATE`). + +--- + +## Why this exists + +`/api/quotas/counts` and the wider quota subsystem (`src/services/quotaService.ts`) ultimately depend on the shared PostgreSQL database for quota-request data and usage aggregation. Before this endpoint, there was no way to check that dependency's health without going through `/api/health/dependencies` (which reports on the *whole app's* dependencies, not specifically the ones `/api/quotas` needs) or the admin-only `/api/admin/health/probes`. `GET /api/quotas/health` fills that gap with a subsystem-scoped, publicly-reachable probe. + +--- + +## Response shape + +```json +{ + "status": "ok", + "timestamp": "2026-07-29T12:00:00.000Z", + "dependencies": { + "database": { "status": "ok", "responseTime": 4 } + }, + "correlationId": "5e4b3c9a-2f1d-4a6e-9c3b-1a2b3c4d5e6f" +} +``` + +On a database outage: + +```json +{ + "status": "down", + "timestamp": "2026-07-29T12:00:03.000Z", + "dependencies": { + "database": { "status": "down", "responseTime": 2001, "error": "unavailable" } + }, + "correlationId": "5e4b3c9a-2f1d-4a6e-9c3b-1a2b3c4d5e6f" +} +``` + +`error` is always a sanitized category (`unavailable`, `timeout`, or an `HTTP ` string) — never a raw driver error message, connection string, or hostname. See `sanitizeCheck()` in `src/routes/health/dependencies.ts` (reused here) for the exact rules. + +`dependencies` currently reports one entry, `database`. If the quota subsystem grows a second external dependency (e.g. a queue or third-party API), it will appear here alongside `database` without changing the shape of existing keys. + +--- + +## HTTP status codes + +| Overall `status` | HTTP code | Meaning | +|---|---|---| +| `ok` | 200 | Database reachable and responding within threshold | +| `degraded` | 200 | Database reachable but slow (> 1000 ms) | +| `down` | 503 | Database unreachable, timed out, or returned an unexpected result | + +--- + +## Correlation IDs + +Every request is assigned a correlation ID the same way as `GET /api/quotas/counts`: + +1. Echoes the inbound `x-correlation-id` header if present. +2. Falls back to the request ID set by the global request-id middleware. +3. Generates a fresh UUID v4 if neither is available. + +The resolved value is returned in both the `X-Correlation-Id` response header and the JSON body's `correlationId` field, so callers can correlate probe results with their own logs. + +--- + +## Structured logging + +Each request logs a `[quotas/health] probe requested` entry on entry and a `[quotas/health] probe completed` (or `probe failed`) entry on exit, both tagged with `requestId` and `correlationId` for tracing. + +--- + +## Configuration + +No dedicated environment variables — the probe reuses the app's shared PostgreSQL pool (`DATABASE_URL` / `DB_*`, see `src/db.ts`) and the shared health-check timeout logic in `src/services/healthCheck.ts` (default 2000 ms, `degraded` above 1000 ms). + +--- + +## Example request + +```bash +curl -s http://localhost:3000/api/quotas/health | jq +``` + +--- + +## Relationship to other health endpoints + +| Endpoint | Scope | Auth | +|---|---|---| +| `GET /api/health` | Whole app, summary only | No | +| `GET /api/health/dependencies` | Whole app, per-dependency detail | No | +| `GET /api/admin/health/probes` | Whole app, per-component detail, single-component drill-down | Admin | +| `GET /api/quotas/health` | `/api/quotas` subsystem only | No | diff --git a/src/routes/quotas.ts b/src/routes/quotas.ts index 18d34345..cfff4f33 100644 --- a/src/routes/quotas.ts +++ b/src/routes/quotas.ts @@ -29,6 +29,7 @@ import { config } from "../config/index.js"; // Sub-route handlers import quotaCountsRouter from "./quotas/counts.js"; +import { createQuotaHealthRouter } from "./quotas/health.js"; export interface QuotasRouterDeps { /** Inject a custom rate-limit middleware, primarily for testing. */ @@ -67,6 +68,11 @@ export function createQuotasRouter(deps: QuotasRouterDeps = {}): Router { // developer's quota requests. router.use("/counts", quotaCountsRouter); + // GET /api/quotas/health — dependency probe (database) for ops/monitoring. + // Mounted after the rate limiter above, so it shares the same per-user + // token bucket as every other /api/quotas route. + router.use("/health", createQuotaHealthRouter()); + return router; } diff --git a/src/routes/quotas/health.test.ts b/src/routes/quotas/health.test.ts new file mode 100644 index 00000000..6e3cf722 --- /dev/null +++ b/src/routes/quotas/health.test.ts @@ -0,0 +1,149 @@ +/** + * Tests for src/routes/quotas/health.ts — GET /api/quotas/health + * + * Coverage targets (≥90% on changed lines): + * + * ✓ 200 + status "ok" when the database check succeeds + * ✓ 503 + status "down" when the database check fails + * ✓ response shape: { status, timestamp, dependencies: { database }, correlationId } + * ✓ error messages are sanitized (no raw connection string / stack leakage) + * ✓ correlation ID is echoed back when x-correlation-id is provided + * ✓ a correlation ID is generated when none is provided + * ✓ X-Correlation-Id response header is set + * ✓ falls back to the shared app pool when no pool is injected + * ✓ drain tracker middleware is applied (does not break normal responses) + */ + +jest.mock('better-sqlite3', () => { + return class MockDatabase { + prepare() { return { get: () => null }; } + exec() {} + close() {} + }; +}); + +import express from 'express'; +import request from 'supertest'; +import type { Pool, QueryResult } from 'pg'; +import { createQuotaHealthRouter } from './health.js'; +import { errorHandler } from '../../middleware/errorHandler.js'; + +function buildApp(pool?: Pool) { + const app = express(); + app.use(express.json()); + app.use('/api/quotas/health', createQuotaHealthRouter(pool ? { pool } : {})); + app.use(errorHandler); + return app; +} + +function createMockPool(queryResult: QueryResult | Error): Pool { + return { + query: async () => { + if (queryResult instanceof Error) { + throw queryResult; + } + return queryResult; + }, + } as unknown as Pool; +} + +describe('GET /api/quotas/health', () => { + it('returns 200 with status "ok" when the database check succeeds', async () => { + const pool = createMockPool({ rows: [{ result: 1 }] } as QueryResult); + const app = buildApp(pool); + + const res = await request(app).get('/api/quotas/health'); + + expect(res.status).toBe(200); + expect(res.body.status).toBe('ok'); + expect(res.body.timestamp).toEqual(expect.any(String)); + expect(res.body.dependencies.database.status).toBe('ok'); + expect(typeof res.body.dependencies.database.responseTime).toBe('number'); + }); + + it('returns 503 with status "down" when the database is unreachable', async () => { + const pool = createMockPool(new Error('Connection refused')); + const app = buildApp(pool); + + const res = await request(app).get('/api/quotas/health'); + + expect(res.status).toBe(503); + expect(res.body.status).toBe('down'); + expect(res.body.dependencies.database.status).toBe('down'); + }); + + it('sanitizes error messages to prevent leaking connection details', async () => { + const pool = createMockPool( + new Error('FATAL: connection to postgres://admin:s3cret@db.internal:5432/prod failed'), + ); + const app = buildApp(pool); + + const res = await request(app).get('/api/quotas/health'); + + expect(res.status).toBe(503); + expect(res.body.dependencies.database.error).toBe('unavailable'); + const body = JSON.stringify(res.body); + expect(body).not.toContain('s3cret'); + expect(body).not.toContain('db.internal'); + expect(body).not.toContain('postgres://'); + }); + + it('only reports the database dependency (quotas has no other external dependency today)', async () => { + const pool = createMockPool({ rows: [{ result: 1 }] } as QueryResult); + const app = buildApp(pool); + + const res = await request(app).get('/api/quotas/health'); + + expect(Object.keys(res.body.dependencies)).toEqual(['database']); + }); + + it('echoes the correlation ID when x-correlation-id is provided', async () => { + const pool = createMockPool({ rows: [{ result: 1 }] } as QueryResult); + const app = buildApp(pool); + + const res = await request(app) + .get('/api/quotas/health') + .set('x-correlation-id', 'corr-quota-health-1'); + + expect(res.status).toBe(200); + expect(res.body.correlationId).toBe('corr-quota-health-1'); + expect(res.headers['x-correlation-id']).toBe('corr-quota-health-1'); + }); + + it('generates a correlation ID when none is provided', async () => { + const pool = createMockPool({ rows: [{ result: 1 }] } as QueryResult); + const app = buildApp(pool); + + const res = await request(app).get('/api/quotas/health'); + + expect(res.status).toBe(200); + expect(typeof res.body.correlationId).toBe('string'); + expect(res.body.correlationId.length).toBeGreaterThan(0); + expect(res.headers['x-correlation-id']).toBe(res.body.correlationId); + }); + + it('falls back to the shared app pool when no pool is injected', async () => { + // No pool override: the router falls back to src/db.ts's shared pool, + // which is not reachable in this unit test environment, so the probe + // should report the database as down rather than throwing. + const app = express(); + app.use(express.json()); + app.use('/api/quotas/health', createQuotaHealthRouter()); + app.use(errorHandler); + + const res = await request(app).get('/api/quotas/health'); + + expect([200, 503]).toContain(res.status); + expect(res.body.dependencies.database).toBeDefined(); + }, 10_000); + + it('applies the drain tracker middleware without breaking normal responses', async () => { + const pool = createMockPool({ rows: [{ result: 1 }] } as QueryResult); + const app = buildApp(pool); + + const res = await request(app).get('/api/quotas/health'); + + expect(res.status).toBe(200); + expect(res.body.dependencies).toBeDefined(); + }); +}); diff --git a/src/routes/quotas/health.ts b/src/routes/quotas/health.ts new file mode 100644 index 00000000..2f5da101 --- /dev/null +++ b/src/routes/quotas/health.ts @@ -0,0 +1,121 @@ +/** + * Quota subsystem dependency probe — GET /api/quotas/health + * + * Reports the status of the external dependencies the `/api/quotas` route + * group (this router plus src/routes/quotas/counts.ts and + * src/services/quotaService.ts) relies on to function. Today that is the + * shared PostgreSQL database: quota request data is ultimately persisted + * there and `listQuotaRequests()` / usage aggregation both depend on it + * being reachable. + * + * Response shape mirrors `/api/health/dependencies` + * (src/routes/health/dependencies.ts) — same `{ status, timestamp, + * dependencies }` envelope and the same {@link sanitizeCheck} rules — so ops + * tooling can treat every dependency probe in the app uniformly. Designed + * for monitoring dashboards / alerting, not for end users, so it does not + * require authentication (matching the global dependencies probe). + * + * ### Correlation IDs & graceful shutdown + * Mounts the same {@link correlationMiddleware} used by + * src/routes/quotas/counts.ts so every request carries an `X-Correlation-Id` + * for structured logging, and reuses the shared {@link quotasDrainTracker} + * so in-flight probe requests are drained on shutdown along with the rest + * of the `/api/quotas` surface. + * + * @module routes/quotas/health + */ + +import { Router } from 'express'; +import type { Request } from 'express'; +import type { Pool } from 'pg'; +import { pool as defaultPool } from '../../db.js'; +import { + checkDatabase, + determineOverallStatus, + type ComponentCheck, + type ComponentStatus, +} from '../../services/healthCheck.js'; +import { sanitizeCheck } from '../health/dependencies.js'; +import { correlationMiddleware } from '../../middleware/correlation.js'; +import { quotasDrainTracker } from './counts.js'; +import { InternalServerError } from '../../errors/index.js'; +import { logger } from '../../logger.js'; + +/** Response body for GET /api/quotas/health. */ +export interface QuotaHealthProbeResponse { + status: ComponentStatus; + timestamp: string; + dependencies: Record; + /** Correlation ID echoed back from the request context. */ + correlationId?: string; +} + +export interface QuotaHealthRouterDeps { + /** Postgres pool to probe. Defaults to the shared app pool (src/db.ts). */ + pool?: Pool; + /** Per-check timeout in ms, forwarded to {@link checkDatabase}. */ + timeoutMs?: number; +} + +/** + * Builds the `/api/quotas/health` router. + * + * @param deps Optional dependency overrides — primarily for unit tests that + * need to inject a mock pool to simulate a healthy or unreachable database. + */ +export function createQuotaHealthRouter(deps: QuotaHealthRouterDeps = {}): Router { + const router = Router(); + const pool = deps.pool ?? defaultPool; + + // Structured logging correlation ID, matching the rest of /api/quotas. + router.use(correlationMiddleware); + + // Count this request against the shared quotas in-flight drain tracker so + // graceful shutdown waits for it just like any other /api/quotas request. + router.use(quotasDrainTracker.middleware); + + router.get('/', async (req: Request, res, next) => { + const requestId = req.id || 'unknown'; + const correlationId = (req as Request & { correlationId?: string }).correlationId; + + logger.info('[quotas/health] probe requested', { requestId, correlationId }); + + try { + const dbCheck = await checkDatabase(pool, deps.timeoutMs); + const dependencies: Record = { + database: sanitizeCheck(dbCheck), + }; + + const overallStatus = determineOverallStatus({ + api: 'ok', + database: dbCheck.status, + }); + + logger.info('[quotas/health] probe completed', { + requestId, + correlationId, + overallStatus, + statuses: Object.fromEntries( + Object.entries(dependencies).map(([key, value]) => [key, value.status]), + ), + }); + + const response: QuotaHealthProbeResponse = { + status: overallStatus, + timestamp: new Date().toISOString(), + dependencies, + correlationId, + }; + + const statusCode = overallStatus === 'down' ? 503 : 200; + res.status(statusCode).json(response); + } catch (error) { + logger.error('[quotas/health] probe failed', { requestId, correlationId, error }); + next(new InternalServerError()); + } + }); + + return router; +} + +export default createQuotaHealthRouter;