feat(capture): vertex service-account auth for vision captioning

packages/cli/src/capture/contentExtractor.test.ts

@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
   captionImagesWithGemini,
+  resolveVertexCaptionConfig,
   resolveVisionPhaseCompletion,
   type VisionCaptionOutcome,
 } from "./contentExtractor.js";
@@ -414,3 +415,72 @@ describe("captionImagesWithGemini — Gemini provider", () => {
     });
   });
 });
+
+describe("resolveVertexCaptionConfig", () => {
+  afterEach(() => {
+    vi.unstubAllEnvs();
+  });
+
+  it("is null unless GOOGLE_GENAI_USE_VERTEXAI is true", () => {
+    vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "");
+    vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "p1" }));
+    expect(resolveVertexCaptionConfig()).toBeNull();
+  });
+
+  it("takes credentials and project from inline service-account JSON", () => {
+    vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true");
+    vi.stubEnv("GOOGLE_CLOUD_PROJECT", "");
+    vi.stubEnv("GOOGLE_PROJECT_ID", "");
+    vi.stubEnv("GOOGLE_CLOUD_LOCATION", "");
+    vi.stubEnv("GOOGLE_LOCATION", "");
+    vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "sa-project" }));
+    const config = resolveVertexCaptionConfig();
+    expect(config?.project).toBe("sa-project");
+    expect(config?.location).toBe("global");
+    expect(config?.googleAuthOptions?.credentials).toEqual({ project_id: "sa-project" });
+  });
+
+  it("explicit project/location env beat the service-account fallback", () => {
+    vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true");
+    vi.stubEnv("GOOGLE_CLOUD_PROJECT", "explicit-project");
+    vi.stubEnv("GOOGLE_CLOUD_LOCATION", "us-central1");
+    vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "sa-project" }));
+    const config = resolveVertexCaptionConfig();
+    expect(config?.project).toBe("explicit-project");
+    expect(config?.location).toBe("us-central1");
+  });
+
+  it("treats unparseable service-account JSON as not configured", () => {
+    vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true");
+    vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", "{not json");
+    expect(resolveVertexCaptionConfig()).toBeNull();
+  });
+});
+
+describe("captionImagesWithGemini — Vertex provider", () => {
+  const dirs: string[] = [];
+
+  afterEach(() => {
+    generateContentMock.mockReset();
+    vi.unstubAllEnvs();
+    for (const d of dirs) rmSync(d, { recursive: true, force: true });
+    dirs.length = 0;
+  });
+
+  it("captions through the SDK when only Vertex is configured (no API keys)", async () => {
+    const dir = makeProjectWithImages();
+    dirs.push(dir);
+    vi.stubEnv("OPENROUTER_API_KEY", "");
+    vi.stubEnv("GEMINI_API_KEY", "");
+    vi.stubEnv("GOOGLE_API_KEY", "");
+    vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true");
+    vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "sa-project" }));
+    generateContentMock.mockResolvedValue({ text: "a red product shot" });
+
+    const captions = await captionImagesWithGemini(dir, () => {}, []);
+
+    expect(Object.values(captions)).toEqual(["a red product shot"]);
+    const request = generateContentMock.mock.calls[0]?.[0];
+    expect(request?.model).toBe("gemini-3.1-flash-lite");
+  });
+});

packages/cli/src/capture/contentExtractor.ts

@@ -229,12 +229,53 @@ export async function extractVisibleText(page: Page): Promise<string> {
   return visibleTextContent;
 }
 
+interface VertexCaptionConfig {
+  project?: string;
+  location: string;
+  googleAuthOptions?: { credentials: Record<string, unknown>; scopes: string[] };
+}
+
+/**
+ * Vertex AI is the opt-in for Google Cloud tenants whose Gemini access is
+ * service-account based — the Vertex endpoint rejects plain API keys, so the
+ * GEMINI_API_KEY path can never reach it. Enable with
+ * GOOGLE_GENAI_USE_VERTEXAI=true (the @google/genai SDK's own convention).
+ * Credentials come from GOOGLE_SERVICE_ACCOUNT_INFO (inline SA JSON) when set,
+ * else the SDK falls through to Application Default Credentials. Project falls
+ * back to the SA's own project_id; location defaults to "global".
+ */
+export function resolveVertexCaptionConfig(): VertexCaptionConfig | null {
+  if ((process.env.GOOGLE_GENAI_USE_VERTEXAI || "").toLowerCase() !== "true") return null;
+  let project = process.env.GOOGLE_CLOUD_PROJECT || process.env.GOOGLE_PROJECT_ID;
+  let googleAuthOptions: VertexCaptionConfig["googleAuthOptions"];
+  const saRaw = process.env.GOOGLE_SERVICE_ACCOUNT_INFO;
+  if (saRaw) {
+    let sa: Record<string, unknown>;
+    try {
+      sa = JSON.parse(saRaw) as Record<string, unknown>;
+    } catch {
+      return null;
+    }
+    googleAuthOptions = {
+      credentials: sa,
+      scopes: ["https://www.googleapis.com/auth/cloud-platform"],
+    };
+    project = project || (typeof sa.project_id === "string" ? sa.project_id : undefined);
+  }
+  const location = process.env.GOOGLE_CLOUD_LOCATION || process.env.GOOGLE_LOCATION || "global";
+  return { project, location, googleAuthOptions };
+}
+
 /**
  * Caption downloaded images using a vision model.
  *
- * Provider is chosen by which API key is present: OPENROUTER_API_KEY → OpenRouter
- * (any vision model via its OpenAI-style API), else GEMINI_API_KEY/GOOGLE_API_KEY
- * → Google Gemini, else no captioning. OpenRouter wins if both are set.
+ * Provider is chosen by which credentials are present: OPENROUTER_API_KEY →
+ * OpenRouter (any vision model via its OpenAI-style API), else
+ * GOOGLE_GENAI_USE_VERTEXAI=true → Gemini on Vertex AI (service-account/ADC
+ * auth — see resolveVertexCaptionConfig), else GEMINI_API_KEY/GOOGLE_API_KEY →
+ * Google Gemini API, else no captioning. OpenRouter wins if several are set;
+ * the Vertex flag wins over a bare Gemini key because a tenant that sets it is
+ * saying its key material is Vertex-side.
  *
  * Batches requests to stay under free-tier rate limits.
  * Returns a map of filename -> caption string.
@@ -266,22 +307,28 @@ export async function captionImagesWithGemini(
   }
   const openRouterKey = process.env.OPENROUTER_API_KEY;
   const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
-  if (!openRouterKey && !geminiKey) {
+  const vertexConfig = resolveVertexCaptionConfig();
+  if (!openRouterKey && !geminiKey && !vertexConfig) {
     reportOutcome();
     return geminiCaptions;
   }
 
-  // OpenRouter takes priority when both keys are set — it's the explicit opt-in
-  // for users without Google access. Both providers satisfy the same
-  // single-image → one-line-caption contract (`captionOne`), so the batching and
-  // SVG-rasterization loops below stay provider-agnostic.
+  // OpenRouter takes priority when several credentials are set — it's the
+  // explicit opt-in for users without Google access. All providers satisfy the
+  // same single-image → one-line-caption contract (`captionOne`), so the
+  // batching and SVG-rasterization loops below stay provider-agnostic.
   const useOpenRouter = Boolean(openRouterKey);
-  const providerName = useOpenRouter ? "OpenRouter" : "Gemini";
-  // Default mirrors the Gemini path's tier (3.x flash-lite). Override per
-  // provider via HYPERFRAMES_OPENROUTER_MODEL / HYPERFRAMES_GEMINI_MODEL.
+  const useVertex = !useOpenRouter && Boolean(vertexConfig);
+  const providerName = useOpenRouter ? "OpenRouter" : useVertex ? "Vertex Gemini" : "Gemini";
+  // Defaults are the same 3.1 flash-lite tier on every surface; the two Google
+  // surfaces publish it under different names (Vertex serves the GA
+  // "gemini-3.1-flash-lite", the Gemini API only its "-preview" alias).
+  // Override per provider via HYPERFRAMES_OPENROUTER_MODEL /
+  // HYPERFRAMES_GEMINI_MODEL.
   const model = useOpenRouter
     ? process.env.HYPERFRAMES_OPENROUTER_MODEL || "google/gemini-3.1-flash-lite"
-    : process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview";
+    : process.env.HYPERFRAMES_GEMINI_MODEL ||
+      (useVertex ? "gemini-3.1-flash-lite" : "gemini-3.1-flash-lite-preview");
   const requestTimeoutMs = resolveVisionRequestTimeoutMs();
 
   progress("design", `Captioning images with ${providerName} vision...`);
@@ -335,10 +382,18 @@ export async function captionImagesWithGemini(
         }, timeoutMs);
       };
     } else {
-      // Unreachable when geminiKey is unset (guarded above); re-narrow for TS.
-      if (!geminiKey) return geminiCaptions;
+      // Unreachable when neither Vertex nor a key is configured (guarded above); re-narrow for TS.
+      if (!geminiKey && !vertexConfig) return geminiCaptions;
       const { GoogleGenAI } = await import("@google/genai");
-      const ai = new GoogleGenAI({ apiKey: geminiKey });
+      const ai =
+        useVertex && vertexConfig
+          ? new GoogleGenAI({
+              vertexai: true,
+              project: vertexConfig.project,
+              location: vertexConfig.location,
+              googleAuthOptions: vertexConfig.googleAuthOptions,
+            })
+          : new GoogleGenAI({ apiKey: geminiKey });
       captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => {
         const response = await runBoundedVisionRequest(
           (signal) =>

packages/cli/src/capture/index.ts

@@ -36,6 +36,7 @@ import {
   detectLibraries,
   extractVisibleText,
   captionImagesWithGemini,
+  resolveVertexCaptionConfig,
   generateAssetDescriptions,
   resolveVisionPhaseCompletion,
 } from "./contentExtractor.js";
@@ -761,11 +762,12 @@ export async function captureWebsite(
           !skipVision &&
           (process.env.OPENROUTER_API_KEY ||
             process.env.GEMINI_API_KEY ||
-            process.env.GOOGLE_API_KEY)
+            process.env.GOOGLE_API_KEY ||
+            resolveVertexCaptionConfig())
         );
         const header = hasVisionKey
           ? "# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually.\n\nTo find a specific brand or icon, **grep this file for the brand name in the description text** (e.g. `grep -i 'autodesk' asset-descriptions.md`). The Gemini Vision captions identify what's actually in each file — that's the agent's selector.\n\nThe `logo-<hash>.svg` filename prefix is a cheap structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). It is NOT a content claim — many `logo-*` files are nav icons or decorative shapes. Trust the captions, not the filename prefix.\n\n"
-          : "# Asset Descriptions\n\n⚠️  GEMINI_API_KEY not set — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY) and re-run.\n\nThe `logo-<hash>.svg` filename prefix is a structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). To pick the actual brand logo without Vision, open the `logo-*` candidates in a previewer or rasterize them with `sharp` before referencing — composing a fake logo ships off-brand in the final video.\n\n";
+          : "# Asset Descriptions\n\n⚠️  No vision credentials — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY), or GOOGLE_GENAI_USE_VERTEXAI=true for Vertex service-account auth, and re-run.\n\nThe `logo-<hash>.svg` filename prefix is a structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). To pick the actual brand logo without Vision, open the `logo-*` candidates in a previewer or rasterize them with `sharp` before referencing — composing a fake logo ships off-brand in the final video.\n\n";
         writeFileSync(
           join(outputDir, "extracted", "asset-descriptions.md"),
           header + lines.map((l) => "- " + l).join("\n") + "\n",