From 545d0ca018b5d89b98950dfbe01750fc31474bc1 Mon Sep 17 00:00:00 2001 From: Yevanchen Date: Thu, 30 Jul 2026 00:01:26 +0800 Subject: [PATCH] feat(reliability): add production runtime canary policy --- README.md | 1 + apps/api/src/adapters/http/create-http-app.ts | 2 + .../http/routes/internal-route-auth.ts | 27 +++++ .../routes/lark-gateway-internal-route.ts | 32 +----- .../routes/status-canary-internal-route.ts | 98 +++++++++++++++++++ .../src/platform/cloudflare/worker-types.ts | 5 + .../status-canary-internal-route.test.ts | 82 ++++++++++++++++ docs/operations/incidents/README.md | 50 ++++++++++ docs/operations/reliability.md | 89 +++++++++++++++++ 9 files changed, 357 insertions(+), 29 deletions(-) create mode 100644 apps/api/src/adapters/http/routes/internal-route-auth.ts create mode 100644 apps/api/src/adapters/http/routes/status-canary-internal-route.ts create mode 100644 apps/api/tests/status-canary-internal-route.test.ts create mode 100644 docs/operations/incidents/README.md create mode 100644 docs/operations/reliability.md diff --git a/README.md b/README.md index 51958157..8633016c 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ https://github.com/user-attachments/assets/4a4bbaab-c192-4462-99e0-020eab966fff - API documentation: [mosoo.ai/docs](https://mosoo.ai/docs) - Canonical product contract: [docs/SPEC.md](./docs/SPEC.md) - Current implementation architecture: [docs/architecture.md](./docs/architecture.md) +- Production SLO and incident policy: [docs/operations/reliability.md](./docs/operations/reliability.md) - PRD index and historical implementation contracts: [docs/prd/README.md](./docs/prd/README.md) The public landing page and blog live in the private `langgenius/mosoo-website` repository and are deployed separately on `mosoo.ai`. diff --git a/apps/api/src/adapters/http/create-http-app.ts b/apps/api/src/adapters/http/create-http-app.ts index 6c0d5631..f1c5f5cf 100644 --- a/apps/api/src/adapters/http/create-http-app.ts +++ b/apps/api/src/adapters/http/create-http-app.ts @@ -24,6 +24,7 @@ import { registerPublicApiRoute } from "./routes/public-api-route"; import { registerRootRoute } from "./routes/root-route"; import { registerSkillRoute } from "./routes/skill-route"; import { registerSlackEventsRoute } from "./routes/slack-events-route"; +import { registerStatusCanaryInternalRoute } from "./routes/status-canary-internal-route"; import { registerTelegramEventsRoute } from "./routes/telegram-events-route"; export function createHttpApp() { @@ -52,6 +53,7 @@ export function createHttpApp() { registerLarkEventsRoute(publicApi); registerLarkGatewayInternalRoute(publicApi); registerSlackEventsRoute(publicApi); + registerStatusCanaryInternalRoute(publicApi); registerTelegramEventsRoute(publicApi); registerGraphQLRoute(publicApi); app.route(PUBLIC_API_PREFIX, publicApi); diff --git a/apps/api/src/adapters/http/routes/internal-route-auth.ts b/apps/api/src/adapters/http/routes/internal-route-auth.ts new file mode 100644 index 00000000..d41053e6 --- /dev/null +++ b/apps/api/src/adapters/http/routes/internal-route-auth.ts @@ -0,0 +1,27 @@ +const secretEncoder = new TextEncoder(); + +async function hashSecret(value: string): Promise { + return new Uint8Array(await crypto.subtle.digest("SHA-256", secretEncoder.encode(value))); +} + +export async function matchesInternalRouteSecret( + submitted: string | undefined, + configured: string, +): Promise { + if (!submitted) { + return false; + } + + const [submittedHash, configuredHash] = await Promise.all([ + hashSecret(submitted), + hashSecret(configured), + ]); + let diff = submittedHash.length ^ configuredHash.length; + const length = Math.max(submittedHash.length, configuredHash.length); + + for (let index = 0; index < length; index += 1) { + diff |= (submittedHash[index] ?? 0) ^ (configuredHash[index] ?? 0); + } + + return diff === 0; +} diff --git a/apps/api/src/adapters/http/routes/lark-gateway-internal-route.ts b/apps/api/src/adapters/http/routes/lark-gateway-internal-route.ts index 3ea1efc1..d26e32d5 100644 --- a/apps/api/src/adapters/http/routes/lark-gateway-internal-route.ts +++ b/apps/api/src/adapters/http/routes/lark-gateway-internal-route.ts @@ -17,10 +17,10 @@ import type { LarkSidecarBindingDescriptor } from "../../../modules/channels/lar import { logInfo } from "../../../platform/cloudflare/logger"; import type { ApiGatewayEnvironment } from "../../../platform/cloudflare/worker-types"; import { toPlatformId } from "../../../shared/platform-id"; +import { matchesInternalRouteSecret } from "./internal-route-auth"; import { platformIdRouteErrorResponse } from "./platform-id-route-error"; const SIDECAR_AUTH_HEADER = "x-sidecar-auth"; -const secretEncoder = new TextEncoder(); function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); @@ -45,32 +45,6 @@ function rejectDisabled(): Response { ); } -async function hashSidecarSecret(value: string): Promise { - return new Uint8Array(await crypto.subtle.digest("SHA-256", secretEncoder.encode(value))); -} - -async function matchesSidecarSecret( - submitted: string | undefined, - configured: string, -): Promise { - if (!submitted) { - return false; - } - - const [submittedHash, configuredHash] = await Promise.all([ - hashSidecarSecret(submitted), - hashSidecarSecret(configured), - ]); - let diff = submittedHash.length ^ configuredHash.length; - const length = Math.max(submittedHash.length, configuredHash.length); - - for (let index = 0; index < length; index += 1) { - diff |= (submittedHash[index] ?? 0) ^ (configuredHash[index] ?? 0); - } - - return diff === 0; -} - interface DescriptorPayload { appId: string; appSecret: string; @@ -95,7 +69,7 @@ export function registerLarkGatewayInternalRoute(app: Hono { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function statusCanarySecretFromEnv(env: { MOSOO_STATUS_CANARY_SECRET?: string }): string | null { + return env.MOSOO_STATUS_CANARY_SECRET?.trim() || null; +} + +function invalidRequest(error: string): Response { + return Response.json({ error, ok: false }, { status: 400 }); +} + +export function registerStatusCanaryInternalRoute(app: Hono) { + app.post("/v1/internal/status-canary/driver-reuse", async (c) => { + const configured = statusCanarySecretFromEnv(c.env); + + if (configured === null) { + return Response.json({ error: "Not Found", ok: false }, { status: 404 }); + } + + if (!(await matchesInternalRouteSecret(c.req.header(STATUS_CANARY_AUTH_HEADER), configured))) { + return Response.json({ error: "status canary auth required", ok: false }, { status: 401 }); + } + + const body: unknown = await c.req.json().catch(() => null); + + if ( + !isRecord(body) || + typeof body["threadId"] !== "string" || + !Array.isArray(body["runIds"]) || + body["runIds"].length !== 2 || + body["runIds"].some((runId) => typeof runId !== "string") + ) { + return invalidRequest("threadId and exactly two runIds are required."); + } + + let threadId: SessionId; + let runIds: [SessionRunId, SessionRunId]; + + try { + threadId = toPlatformId(body["threadId"], "Thread ID"); + runIds = [ + toPlatformId(body["runIds"][0] as string, "First run ID"), + toPlatformId(body["runIds"][1] as string, "Follow-up run ID"), + ]; + } catch (error) { + const response = platformIdRouteErrorResponse(error, (message) => ({ + error: message, + ok: false, + })); + + if (response !== null) { + return response; + } + throw error; + } + + if (runIds[0] === runIds[1]) { + return invalidRequest("runIds must be distinct."); + } + + const rows = await getAppDatabase(c.env.DB) + .select({ + driverInstanceId: sessionRunsTable.driverInstanceId, + id: sessionRunsTable.id, + }) + .from(sessionRunsTable) + .where(and(eq(sessionRunsTable.sessionId, threadId), inArray(sessionRunsTable.id, runIds))) + .all(); + const driverByRun = new Map(rows.map((row) => [row.id, row.driverInstanceId])); + const drivers = runIds.map((runId) => driverByRun.get(runId) ?? null); + + if (drivers.some((driver) => driver === null)) { + return Response.json( + { error: "Canary runs are not ready for comparison.", ok: false }, + { status: 409 }, + ); + } + + return c.json({ + ok: true, + sameDriver: drivers[0] === drivers[1], + }); + }); +} diff --git a/apps/api/src/platform/cloudflare/worker-types.ts b/apps/api/src/platform/cloudflare/worker-types.ts index f7737d19..faa75b3f 100644 --- a/apps/api/src/platform/cloudflare/worker-types.ts +++ b/apps/api/src/platform/cloudflare/worker-types.ts @@ -87,6 +87,10 @@ interface OptionalLarkSidecarBindings { MOSOO_LARK_SIDECAR_SECRET?: string; } +interface OptionalStatusCanaryBindings { + MOSOO_STATUS_CANARY_SECRET?: string; +} + interface OptionalRuntimeSubjectPlatformBindings { runtimeSubjectHandleFactory?: (runtimeSubjectId: string) => unknown; } @@ -106,6 +110,7 @@ export type ApiBindings = Env & OptionalSlackAdapterBindings & OptionalWeChatIlinkBindings & OptionalLarkSidecarBindings & + OptionalStatusCanaryBindings & OptionalRuntimeSubjectPlatformBindings; export interface ApiGatewayEnvironment { diff --git a/apps/api/tests/status-canary-internal-route.test.ts b/apps/api/tests/status-canary-internal-route.test.ts new file mode 100644 index 00000000..ac7dbeb8 --- /dev/null +++ b/apps/api/tests/status-canary-internal-route.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from "bun:test"; + +import { PUBLIC_API_PREFIX } from "@mosoo/contracts/public-api"; + +import { createHttpApp } from "../src/adapters/http/create-http-app"; +import type { ApiBindings } from "../src/platform/cloudflare/worker-types"; +import { SqliteD1Database } from "./helpers/sqlite-d1"; + +const THREAD_ID = "01J00000000000000000000001"; +const FIRST_RUN_ID = "01J00000000000000000000002"; +const FOLLOW_UP_RUN_ID = "01J00000000000000000000003"; +const DRIVER_ID = "01J00000000000000000000004"; +const WRONG_THREAD_ID = "01J00000000000000000000005"; +const SECRET = "test-status-canary-secret"; + +function createBindings(): ApiBindings { + const database = new SqliteD1Database(); + + database.execute(` + CREATE TABLE session_run ( + id text PRIMARY KEY NOT NULL, + session_id text NOT NULL, + driver_instance_id text + ); + INSERT INTO session_run (id, session_id, driver_instance_id) VALUES + ('${FIRST_RUN_ID}', '${THREAD_ID}', '${DRIVER_ID}'), + ('${FOLLOW_UP_RUN_ID}', '${THREAD_ID}', '${DRIVER_ID}'); + `); + + return { + DB: database, + MOSOO_STATUS_CANARY_SECRET: SECRET, + } as ApiBindings; +} + +describe("status canary driver reuse route", () => { + test("requires its shared secret and compares only runs from the requested thread", async () => { + const bindings = createBindings(); + const url = `${PUBLIC_API_PREFIX}/v1/internal/status-canary/driver-reuse`; + const body = JSON.stringify({ + runIds: [FIRST_RUN_ID, FOLLOW_UP_RUN_ID], + threadId: THREAD_ID, + }); + const unauthorized = await createHttpApp().request( + url, + { body, headers: { "content-type": "application/json" }, method: "POST" }, + bindings, + ); + const authorized = await createHttpApp().request( + url, + { + body, + headers: { + "content-type": "application/json", + "x-status-canary-auth": SECRET, + }, + method: "POST", + }, + bindings, + ); + const wrongThread = await createHttpApp().request( + url, + { + body: JSON.stringify({ + runIds: [FIRST_RUN_ID, FOLLOW_UP_RUN_ID], + threadId: WRONG_THREAD_ID, + }), + headers: { + "content-type": "application/json", + "x-status-canary-auth": SECRET, + }, + method: "POST", + }, + bindings, + ); + + expect(unauthorized.status).toBe(401); + expect(authorized.status).toBe(200); + expect(await authorized.json()).toEqual({ ok: true, sameDriver: true }); + expect(wrongThread.status).toBe(409); + }); +}); diff --git a/docs/operations/incidents/README.md b/docs/operations/incidents/README.md new file mode 100644 index 00000000..036fb1bc --- /dev/null +++ b/docs/operations/incidents/README.md @@ -0,0 +1,50 @@ +# Incident Postmortems + +Archive every production incident here as +`YYYY-MM-DD-short-description.md`. Use UTC throughout, link the originating +issue or PR, and write for an affected user rather than for the implementation +team. + +Copy this one-page template: + +```markdown +# YYYY-MM-DD — Incident title + +- Status: Resolved +- Severity: SEV-1 / SEV-2 / SEV-3 +- Window: YYYY-MM-DD HH:MM–HH:MM UTC +- Affected surface: runtime / API / console / website +- Public status update: URL +- Tracking issue or PR: URL + +## Summary And Impact + +What users tried to do, what they saw, how many were affected, and for how long. +State unknowns explicitly. + +## Timeline + +- HH:MM — Detection +- HH:MM — User impact confirmed +- HH:MM — Mitigation started +- HH:MM — Recovery confirmed by production canary + +## Root Cause + +The technical and organizational conditions that made the incident possible. +Do not stop at the last failing line of code. + +## Detection And Response + +What detected the incident, what did not, and which response steps helped or +slowed recovery. + +## Corrective Actions + +- [ ] Owner — action — due date +- [ ] Owner — regression check or canary change — due date + +## Lessons + +What should change in the product, release process, or operating assumptions. +``` diff --git a/docs/operations/reliability.md b/docs/operations/reliability.md new file mode 100644 index 00000000..a226fb3c --- /dev/null +++ b/docs/operations/reliability.md @@ -0,0 +1,89 @@ +# Production Reliability + +mosoo publishes production runtime health at +[mosoo.ai/status](https://mosoo.ai/status). This is a public SLO and measured +history, not a contractual SLA. + +## Synthetic Canary + +Every five minutes, the website Worker calls the production Public API for each +public runtime: + +1. Start a dedicated Thread with a cheap deterministic prompt. +2. Require assistant output and a completed Run within that target's TTFT + budget. +3. Wait six seconds, then send a follow-up through the same Thread. +4. Require the follow-up within budget and verify both Runs used the same + driver. +5. Delete the canary Thread and publish only timings, pass/fail state, and daily + counts to `/status.json`. + +The monitored runtimes are OpenAI Codex (`openai-runtime`), Claude Agent SDK +(`claude-agent-sdk`), and OpenCode (`acp-fallback`). Missing two scheduled +intervals makes the component `unknown`; a shallow health ping never counts as +a passing check. + +## SLO And Error Budget + +The target is **99.5% passing canary checks over a rolling 30-day window**. A +check passes only when both turns meet the configured TTFT budget and the +follow-up reuses the first turn's driver. + +If any runtime breaches the check three consecutive times, feature releases +freeze. Until a passing canary and incident review clear the freeze, the team +ships reliability fixes only. A customer-visible production failure triggers +the same rule even before the third canary breach. + +## Production Activation + +Create one dedicated, published canary Agent per runtime under a dedicated +service account. Store the shared diagnostic secret in both Workers, and store +the PAT plus Agent IDs only in the website Worker: + +```bash +# From the mosoo repository: +cd apps/api +../../node_modules/.bin/vp exec wrangler secret put MOSOO_STATUS_CANARY_SECRET --env prod + +# From the mosoo-website repository: +npx --yes --package wrangler@4.115.0 wrangler secret put STATUS_CANARY_SECRET --env prod +npx --yes --package wrangler@4.115.0 wrangler secret put STATUS_CANARY_TARGETS --env prod +``` + +`STATUS_CANARY_TARGETS` is one secret JSON value: + +```json +{ + "token": "", + "targets": [ + { + "id": "openai-runtime", + "agentId": "", + "ttftBudgetMs": 20000 + }, + { + "id": "claude-agent-sdk", + "agentId": "", + "ttftBudgetMs": 20000 + }, + { + "id": "acp-fallback", + "agentId": "", + "ttftBudgetMs": 20000 + } + ] +} +``` + +Never commit either secret. After both Workers are deployed, verify +`https://mosoo.ai/status.json` records all three components and that a second +turn reports `driverReused: true`. + +## Incident Discipline + +Open an incident when a canary reaches the freeze threshold, production +failures affect users, or the status feed is stale for more than two intervals. +Publish the timeline and customer impact while investigating. After recovery, +archive a postmortem using +[the incident template](./incidents/README.md) before normal feature releases +resume.