fix(producer): retry transient deterministic font fetches (#2865)

* fix(producer): retry transient deterministic font fetches

* fix(producer): secure Lambda font cache directory

Author: jrusso1020

packages/aws-lambda/src/cdk/HyperframesRenderStack.snapshot.test.ts

@@ -182,6 +182,7 @@ describe("HyperframesRenderStack — snapshot", () => {
         present: true,
       });
     }
+    expect(collected.has("FONT_FETCH_UNAVAILABLE")).toBe(false);
   });
 
   it("classifies plan v2 integrity failures as terminal in every v2 Lambda task", () => {

packages/aws-lambda/src/handler.test.ts

@@ -249,6 +249,8 @@ describe("handler dispatch", () => {
       "PLAN_TOO_LARGE",
       "PLAN_PROTOCOL_UNSUPPORTED",
       "PLAN_V2_INTEGRITY_UNRECOVERABLE",
+      "FONT_FETCH_FAILED",
+      "FONT_FETCH_UNAVAILABLE",
       "VIDEO_SOURCE_UNRENDERABLE",
       "VIDEO_EXTRACTION_FAILED",
       "INVALID_VIDEO_METADATA",

packages/aws-lambda/src/handler.ts

@@ -150,6 +150,8 @@ function normalizeTerminalErrorName(error: unknown): void {
     candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" ||
     candidate.code === "PLAN_TOO_LARGE" ||
     candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" ||
+    candidate.code === "FONT_FETCH_FAILED" ||
+    candidate.code === "FONT_FETCH_UNAVAILABLE" ||
     candidate.code === "VIDEO_SOURCE_UNRENDERABLE" ||
     candidate.code === "VIDEO_EXTRACTION_FAILED" ||
     candidate.code === "INVALID_VIDEO_METADATA"

packages/gcp-cloud-run/src/server.test.ts

@@ -578,7 +578,9 @@ describe("createApp HTTP mapping", () => {
   it.each([
     ["VIDEO_SOURCE_UNRENDERABLE", 400],
     ["VIDEO_EXTRACTION_FAILED", 500],
-  ] as const)("routes producer video code %s with HTTP %s", async (code, expectedStatus) => {
+    ["FONT_FETCH_FAILED", 400],
+    ["FONT_FETCH_UNAVAILABLE", 500],
+  ] as const)("routes producer code %s with HTTP %s", async (code, expectedStatus) => {
     const gcs = new FakeGcs();
     await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
     const app = createApp(

packages/gcp-cloud-run/src/server.ts

@@ -183,6 +183,8 @@ function normalizeTerminalErrorName(error: unknown): void {
     candidate.code === "PLAN_PROTOCOL_UNSUPPORTED" ||
     candidate.code === "PLAN_TOO_LARGE" ||
     candidate.code === "PLAN_V2_INTEGRITY_UNRECOVERABLE" ||
+    candidate.code === "FONT_FETCH_FAILED" ||
+    candidate.code === "FONT_FETCH_UNAVAILABLE" ||
     candidate.code === "VIDEO_SOURCE_UNRENDERABLE" ||
     candidate.code === "VIDEO_EXTRACTION_FAILED" ||
     candidate.code === "INVALID_VIDEO_METADATA"

packages/producer/src/index.ts

@@ -114,7 +114,13 @@ export { normalizeErrorMessage } from "./utils/errorMessage.js";
 // timing. The render pipeline runs this in its compile stage; the CLI audit
 // paths (snapshot/check) reuse it so their captures match the render.
 export {
+  FONT_FETCH_FAILED,
+  FONT_FETCH_UNAVAILABLE,
+  FontFetchError,
+  FontFetchUnavailableError,
   injectDeterministicFontFaces,
+  type FontFetchErrorCode,
+  type FontFetchRetryPolicy,
   type InjectDeterministicFontFacesOptions,
 } from "./services/deterministicFonts.js";
 export { quantizeTimeToFrame } from "./utils/parityContract.js";

packages/producer/src/services/deterministicFonts-failClosed.test.ts

@@ -17,7 +17,9 @@
 import { describe, expect, it } from "bun:test";
 import {
   FONT_FETCH_FAILED,
+  FONT_FETCH_UNAVAILABLE,
   FontFetchError,
+  FontFetchUnavailableError,
   injectDeterministicFontFaces,
 } from "./deterministicFonts.js";
 
@@ -75,6 +77,15 @@ function makeGoogleFontFetch(cssRequests: string[]): typeof fetch {
   }) as unknown as typeof fetch;
 }
 
+async function rejectedError(promise: Promise<unknown>): Promise<unknown> {
+  try {
+    await promise;
+  } catch (error) {
+    return error;
+  }
+  throw new Error("Expected promise to reject");
+}
+
 describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)", () => {
   it("swallows a network failure and returns the original HTML (no throw)", async () => {
     const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
@@ -141,19 +152,18 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: true", () => {
     });
   }
 
-  it("throws FontFetchError on a network failure", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+  it("throws FontFetchUnavailableError after retrying a network failure", async () => {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         fetchImpl: makeFailingFetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+        fontFetchRetryPolicy: { baseDelayMs: 0 },
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
-    expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
-    expect((caught as FontFetchError).code).toBe("FONT_FETCH_FAILED");
+    expect(caught).toBeInstanceOf(FontFetchUnavailableError);
+    expect((caught as FontFetchError).code).toBe(FONT_FETCH_UNAVAILABLE);
+    expect((caught as FontFetchError).code).toBe("FONT_FETCH_UNAVAILABLE");
     expect((caught as FontFetchError).familyName).toBe("NotARealFontFamilyForTest");
     expect((caught as Error).message).toContain("simulated network failure");
   });
@@ -164,62 +174,53 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: true", () => {
     // unresolvable (no alias, no Google Fonts, no system capture) which IS
     // a fail-closed error — the render would use a fallback font, producing
     // non-deterministic output across machines.
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         allowSystemFontCapture: false,
         fetchImpl: makeHttp400Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
     expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
     expect((caught as FontFetchError).familyName).toBe("NotARealFontFamilyForTest");
   });
 
   it("throws on a 404 response when font is completely unresolvable", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         allowSystemFontCapture: false,
         fetchImpl: makeHttp404Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
     expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
   });
 
-  it("throws FontFetchError on a 5xx response — non-deterministic, could differ on retry", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+  it("throws FontFetchUnavailableError on an exhausted 5xx response", async () => {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         fetchImpl: makeHttp503Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+        fontFetchRetryPolicy: { baseDelayMs: 0 },
+      }),
+    );
     expect(caught).toBeInstanceOf(FontFetchError);
-    expect((caught as FontFetchError).code).toBe(FONT_FETCH_FAILED);
+    expect(caught).toBeInstanceOf(FontFetchUnavailableError);
+    expect((caught as FontFetchError).code).toBe(FONT_FETCH_UNAVAILABLE);
     expect((caught as Error).message).toContain("HTTP 503");
     expect((caught as Error).message).toContain("NotARealFontFamilyForTest");
   });
 
   it("includes the requested URL in 5xx errors", async () => {
-    let caught: unknown;
-    try {
-      await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
+    const caught = await rejectedError(
+      injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, {
         failClosedFontFetch: true,
         fetchImpl: makeHttp503Fetch(),
-      });
-    } catch (err) {
-      caught = err;
-    }
+        fontFetchRetryPolicy: { baseDelayMs: 0 },
+      }),
+    );
     expect((caught as FontFetchError).url).toContain("fonts.googleapis.com");
     expect((caught as FontFetchError).url).toContain("NotARealFontFamilyForTest");
   });