feat(reliability): add production runtime canary policy

README.md

@@ -103,6 +103,7 @@ https://github.com/user-attachments/assets/4a4bbaab-c192-4462-99e0-020eab966fff
 - API documentation: [mosoo.ai/docs](https://mosoo.ai/docs)
 - Canonical product contract: [docs/SPEC.md](./docs/SPEC.md)
 - Current implementation architecture: [docs/architecture.md](./docs/architecture.md)
+- Production SLO and incident policy: [docs/operations/reliability.md](./docs/operations/reliability.md)
 - PRD index and historical implementation contracts: [docs/prd/README.md](./docs/prd/README.md)
 
 The public landing page and blog live in the private `langgenius/mosoo-website` repository and are deployed separately on `mosoo.ai`.

apps/api/src/adapters/http/create-http-app.ts

@@ -24,6 +24,7 @@ import { registerPublicApiRoute } from "./routes/public-api-route";
 import { registerRootRoute } from "./routes/root-route";
 import { registerSkillRoute } from "./routes/skill-route";
 import { registerSlackEventsRoute } from "./routes/slack-events-route";
+import { registerStatusCanaryInternalRoute } from "./routes/status-canary-internal-route";
 import { registerTelegramEventsRoute } from "./routes/telegram-events-route";
 
 export function createHttpApp() {
@@ -52,6 +53,7 @@ export function createHttpApp() {
   registerLarkEventsRoute(publicApi);
   registerLarkGatewayInternalRoute(publicApi);
   registerSlackEventsRoute(publicApi);
+  registerStatusCanaryInternalRoute(publicApi);
   registerTelegramEventsRoute(publicApi);
   registerGraphQLRoute(publicApi);
   app.route(PUBLIC_API_PREFIX, publicApi);

apps/api/src/adapters/http/routes/internal-route-auth.ts

@@ -0,0 +1,27 @@
+const secretEncoder = new TextEncoder();
+
+async function hashSecret(value: string): Promise<Uint8Array> {
+  return new Uint8Array(await crypto.subtle.digest("SHA-256", secretEncoder.encode(value)));
+}
+
+export async function matchesInternalRouteSecret(
+  submitted: string | undefined,
+  configured: string,
+): Promise<boolean> {
+  if (!submitted) {
+    return false;
+  }
+
+  const [submittedHash, configuredHash] = await Promise.all([
+    hashSecret(submitted),
+    hashSecret(configured),
+  ]);
+  let diff = submittedHash.length ^ configuredHash.length;
+  const length = Math.max(submittedHash.length, configuredHash.length);
+
+  for (let index = 0; index < length; index += 1) {
+    diff |= (submittedHash[index] ?? 0) ^ (configuredHash[index] ?? 0);
+  }
+
+  return diff === 0;
+}

apps/api/src/adapters/http/routes/lark-gateway-internal-route.ts

@@ -17,10 +17,10 @@ import type { LarkSidecarBindingDescriptor } from "../../../modules/channels/lar
 import { logInfo } from "../../../platform/cloudflare/logger";
 import type { ApiGatewayEnvironment } from "../../../platform/cloudflare/worker-types";
 import { toPlatformId } from "../../../shared/platform-id";
+import { matchesInternalRouteSecret } from "./internal-route-auth";
 import { platformIdRouteErrorResponse } from "./platform-id-route-error";
 
 const SIDECAR_AUTH_HEADER = "x-sidecar-auth";
-const secretEncoder = new TextEncoder();
 
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -45,32 +45,6 @@ function rejectDisabled(): Response {
   );
 }
 
-async function hashSidecarSecret(value: string): Promise<Uint8Array> {
-  return new Uint8Array(await crypto.subtle.digest("SHA-256", secretEncoder.encode(value)));
-}
-
-async function matchesSidecarSecret(
-  submitted: string | undefined,
-  configured: string,
-): Promise<boolean> {
-  if (!submitted) {
-    return false;
-  }
-
-  const [submittedHash, configuredHash] = await Promise.all([
-    hashSidecarSecret(submitted),
-    hashSidecarSecret(configured),
-  ]);
-  let diff = submittedHash.length ^ configuredHash.length;
-  const length = Math.max(submittedHash.length, configuredHash.length);
-
-  for (let index = 0; index < length; index += 1) {
-    diff |= (submittedHash[index] ?? 0) ^ (configuredHash[index] ?? 0);
-  }
-
-  return diff === 0;
-}
-
 interface DescriptorPayload {
   appId: string;
   appSecret: string;
@@ -95,7 +69,7 @@ export function registerLarkGatewayInternalRoute(app: Hono<ApiGatewayEnvironment
     }
 
     const submitted = c.req.header(SIDECAR_AUTH_HEADER);
-    if (!(await matchesSidecarSecret(submitted, configured))) {
+    if (!(await matchesInternalRouteSecret(submitted, configured))) {
       return rejectUnauthenticated();
     }
 
@@ -113,7 +87,7 @@ export function registerLarkGatewayInternalRoute(app: Hono<ApiGatewayEnvironment
     }
 
     const submitted = c.req.header(SIDECAR_AUTH_HEADER);
-    if (!(await matchesSidecarSecret(submitted, configured))) {
+    if (!(await matchesInternalRouteSecret(submitted, configured))) {
       return rejectUnauthenticated();
     }
 

apps/api/src/adapters/http/routes/status-canary-internal-route.ts

@@ -0,0 +1,98 @@
+import { sessionRunsTable } from "@mosoo/db";
+import type { SessionId, SessionRunId } from "@mosoo/id";
+import { and, eq, inArray } from "drizzle-orm";
+import type { Hono } from "hono";
+
+import type { ApiGatewayEnvironment } from "../../../platform/cloudflare/worker-types";
+import { getAppDatabase } from "../../../platform/db/drizzle";
+import { toPlatformId } from "../../../shared/platform-id";
+import { matchesInternalRouteSecret } from "./internal-route-auth";
+import { platformIdRouteErrorResponse } from "./platform-id-route-error";
+
+const STATUS_CANARY_AUTH_HEADER = "x-status-canary-auth";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function statusCanarySecretFromEnv(env: { MOSOO_STATUS_CANARY_SECRET?: string }): string | null {
+  return env.MOSOO_STATUS_CANARY_SECRET?.trim() || null;
+}
+
+function invalidRequest(error: string): Response {
+  return Response.json({ error, ok: false }, { status: 400 });
+}
+
+export function registerStatusCanaryInternalRoute(app: Hono<ApiGatewayEnvironment>) {
+  app.post("/v1/internal/status-canary/driver-reuse", async (c) => {
+    const configured = statusCanarySecretFromEnv(c.env);
+
+    if (configured === null) {
+      return Response.json({ error: "Not Found", ok: false }, { status: 404 });
+    }
+
+    if (!(await matchesInternalRouteSecret(c.req.header(STATUS_CANARY_AUTH_HEADER), configured))) {
+      return Response.json({ error: "status canary auth required", ok: false }, { status: 401 });
+    }
+
+    const body: unknown = await c.req.json().catch(() => null);
+
+    if (
+      !isRecord(body) ||
+      typeof body["threadId"] !== "string" ||
+      !Array.isArray(body["runIds"]) ||
+      body["runIds"].length !== 2 ||
+      body["runIds"].some((runId) => typeof runId !== "string")
+    ) {
+      return invalidRequest("threadId and exactly two runIds are required.");
+    }
+
+    let threadId: SessionId;
+    let runIds: [SessionRunId, SessionRunId];
+
+    try {
+      threadId = toPlatformId<SessionId>(body["threadId"], "Thread ID");
+      runIds = [
+        toPlatformId<SessionRunId>(body["runIds"][0] as string, "First run ID"),
+        toPlatformId<SessionRunId>(body["runIds"][1] as string, "Follow-up run ID"),
+      ];
+    } catch (error) {
+      const response = platformIdRouteErrorResponse(error, (message) => ({
+        error: message,
+        ok: false,
+      }));
+
+      if (response !== null) {
+        return response;
+      }
+      throw error;
+    }
+
+    if (runIds[0] === runIds[1]) {
+      return invalidRequest("runIds must be distinct.");
+    }
+
+    const rows = await getAppDatabase(c.env.DB)
+      .select({
+        driverInstanceId: sessionRunsTable.driverInstanceId,
+        id: sessionRunsTable.id,
+      })
+      .from(sessionRunsTable)
+      .where(and(eq(sessionRunsTable.sessionId, threadId), inArray(sessionRunsTable.id, runIds)))
+      .all();
+    const driverByRun = new Map(rows.map((row) => [row.id, row.driverInstanceId]));
+    const drivers = runIds.map((runId) => driverByRun.get(runId) ?? null);
+
+    if (drivers.some((driver) => driver === null)) {
+      return Response.json(
+        { error: "Canary runs are not ready for comparison.", ok: false },
+        { status: 409 },
+      );
+    }
+
+    return c.json({
+      ok: true,
+      sameDriver: drivers[0] === drivers[1],
+    });
+  });
+}

apps/api/src/platform/cloudflare/worker-types.ts

@@ -87,6 +87,10 @@ interface OptionalLarkSidecarBindings {
   MOSOO_LARK_SIDECAR_SECRET?: string;
 }
 
+interface OptionalStatusCanaryBindings {
+  MOSOO_STATUS_CANARY_SECRET?: string;
+}
+
 interface OptionalRuntimeSubjectPlatformBindings {
   runtimeSubjectHandleFactory?: (runtimeSubjectId: string) => unknown;
 }
@@ -106,6 +110,7 @@ export type ApiBindings = Env &
   OptionalSlackAdapterBindings &
   OptionalWeChatIlinkBindings &
   OptionalLarkSidecarBindings &
+  OptionalStatusCanaryBindings &
   OptionalRuntimeSubjectPlatformBindings;
 
 export interface ApiGatewayEnvironment {

apps/api/tests/status-canary-internal-route.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, test } from "bun:test";
+
+import { PUBLIC_API_PREFIX } from "@mosoo/contracts/public-api";
+
+import { createHttpApp } from "../src/adapters/http/create-http-app";
+import type { ApiBindings } from "../src/platform/cloudflare/worker-types";
+import { SqliteD1Database } from "./helpers/sqlite-d1";
+
+const THREAD_ID = "01J00000000000000000000001";
+const FIRST_RUN_ID = "01J00000000000000000000002";
+const FOLLOW_UP_RUN_ID = "01J00000000000000000000003";
+const DRIVER_ID = "01J00000000000000000000004";
+const WRONG_THREAD_ID = "01J00000000000000000000005";
+const SECRET = "test-status-canary-secret";
+
+function createBindings(): ApiBindings {
+  const database = new SqliteD1Database();
+
+  database.execute(`
+    CREATE TABLE session_run (
+      id text PRIMARY KEY NOT NULL,
+      session_id text NOT NULL,
+      driver_instance_id text
+    );
+    INSERT INTO session_run (id, session_id, driver_instance_id) VALUES
+      ('${FIRST_RUN_ID}', '${THREAD_ID}', '${DRIVER_ID}'),
+      ('${FOLLOW_UP_RUN_ID}', '${THREAD_ID}', '${DRIVER_ID}');
+  `);
+
+  return {
+    DB: database,
+    MOSOO_STATUS_CANARY_SECRET: SECRET,
+  } as ApiBindings;
+}
+
+describe("status canary driver reuse route", () => {
+  test("requires its shared secret and compares only runs from the requested thread", async () => {
+    const bindings = createBindings();
+    const url = `${PUBLIC_API_PREFIX}/v1/internal/status-canary/driver-reuse`;
+    const body = JSON.stringify({
+      runIds: [FIRST_RUN_ID, FOLLOW_UP_RUN_ID],
+      threadId: THREAD_ID,
+    });
+    const unauthorized = await createHttpApp().request(
+      url,
+      { body, headers: { "content-type": "application/json" }, method: "POST" },
+      bindings,
+    );
+    const authorized = await createHttpApp().request(
+      url,
+      {
+        body,
+        headers: {
+          "content-type": "application/json",
+          "x-status-canary-auth": SECRET,
+        },
+        method: "POST",
+      },
+      bindings,
+    );
+    const wrongThread = await createHttpApp().request(
+      url,
+      {
+        body: JSON.stringify({
+          runIds: [FIRST_RUN_ID, FOLLOW_UP_RUN_ID],
+          threadId: WRONG_THREAD_ID,
+        }),
+        headers: {
+          "content-type": "application/json",
+          "x-status-canary-auth": SECRET,
+        },
+        method: "POST",
+      },
+      bindings,
+    );
+
+    expect(unauthorized.status).toBe(401);
+    expect(authorized.status).toBe(200);
+    expect(await authorized.json()).toEqual({ ok: true, sameDriver: true });
+    expect(wrongThread.status).toBe(409);
+  });
+});

docs/operations/incidents/README.md

@@ -0,0 +1,50 @@
+# Incident Postmortems
+
+Archive every production incident here as
+`YYYY-MM-DD-short-description.md`. Use UTC throughout, link the originating
+issue or PR, and write for an affected user rather than for the implementation
+team.
+
+Copy this one-page template:
+
+```markdown
+# YYYY-MM-DD — Incident title
+
+- Status: Resolved
+- Severity: SEV-1 / SEV-2 / SEV-3
+- Window: YYYY-MM-DD HH:MM–HH:MM UTC
+- Affected surface: runtime / API / console / website
+- Public status update: URL
+- Tracking issue or PR: URL
+
+## Summary And Impact
+
+What users tried to do, what they saw, how many were affected, and for how long.
+State unknowns explicitly.
+
+## Timeline
+
+- HH:MM — Detection
+- HH:MM — User impact confirmed
+- HH:MM — Mitigation started
+- HH:MM — Recovery confirmed by production canary
+
+## Root Cause
+
+The technical and organizational conditions that made the incident possible.
+Do not stop at the last failing line of code.
+
+## Detection And Response
+
+What detected the incident, what did not, and which response steps helped or
+slowed recovery.
+
+## Corrective Actions
+
+- [ ] Owner — action — due date
+- [ ] Owner — regression check or canary change — due date
+
+## Lessons
+
+What should change in the product, release process, or operating assumptions.
+```