diff --git a/packages/cli/src/capture/contentExtractor.test.ts b/packages/cli/src/capture/contentExtractor.test.ts index 5117f73d2e..94e381ff7b 100644 --- a/packages/cli/src/capture/contentExtractor.test.ts +++ b/packages/cli/src/capture/contentExtractor.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { captionImagesWithGemini, + resolveVertexCaptionConfig, resolveVisionPhaseCompletion, type VisionCaptionOutcome, } from "./contentExtractor.js"; @@ -414,3 +415,72 @@ describe("captionImagesWithGemini — Gemini provider", () => { }); }); }); + +describe("resolveVertexCaptionConfig", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("is null unless GOOGLE_GENAI_USE_VERTEXAI is true", () => { + vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", ""); + vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "p1" })); + expect(resolveVertexCaptionConfig()).toBeNull(); + }); + + it("takes credentials and project from inline service-account JSON", () => { + vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true"); + vi.stubEnv("GOOGLE_CLOUD_PROJECT", ""); + vi.stubEnv("GOOGLE_PROJECT_ID", ""); + vi.stubEnv("GOOGLE_CLOUD_LOCATION", ""); + vi.stubEnv("GOOGLE_LOCATION", ""); + vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "sa-project" })); + const config = resolveVertexCaptionConfig(); + expect(config?.project).toBe("sa-project"); + expect(config?.location).toBe("global"); + expect(config?.googleAuthOptions?.credentials).toEqual({ project_id: "sa-project" }); + }); + + it("explicit project/location env beat the service-account fallback", () => { + vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true"); + vi.stubEnv("GOOGLE_CLOUD_PROJECT", "explicit-project"); + vi.stubEnv("GOOGLE_CLOUD_LOCATION", "us-central1"); + vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "sa-project" })); + const config = resolveVertexCaptionConfig(); + expect(config?.project).toBe("explicit-project"); + expect(config?.location).toBe("us-central1"); + }); + + it("treats unparseable service-account JSON as not configured", () => { + vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true"); + vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", "{not json"); + expect(resolveVertexCaptionConfig()).toBeNull(); + }); +}); + +describe("captionImagesWithGemini — Vertex provider", () => { + const dirs: string[] = []; + + afterEach(() => { + generateContentMock.mockReset(); + vi.unstubAllEnvs(); + for (const d of dirs) rmSync(d, { recursive: true, force: true }); + dirs.length = 0; + }); + + it("captions through the SDK when only Vertex is configured (no API keys)", async () => { + const dir = makeProjectWithImages(); + dirs.push(dir); + vi.stubEnv("OPENROUTER_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", ""); + vi.stubEnv("GOOGLE_API_KEY", ""); + vi.stubEnv("GOOGLE_GENAI_USE_VERTEXAI", "true"); + vi.stubEnv("GOOGLE_SERVICE_ACCOUNT_INFO", JSON.stringify({ project_id: "sa-project" })); + generateContentMock.mockResolvedValue({ text: "a red product shot" }); + + const captions = await captionImagesWithGemini(dir, () => {}, []); + + expect(Object.values(captions)).toEqual(["a red product shot"]); + const request = generateContentMock.mock.calls[0]?.[0]; + expect(request?.model).toBe("gemini-3.1-flash-lite"); + }); +}); diff --git a/packages/cli/src/capture/contentExtractor.ts b/packages/cli/src/capture/contentExtractor.ts index acea6b11ea..44d051aeb0 100644 --- a/packages/cli/src/capture/contentExtractor.ts +++ b/packages/cli/src/capture/contentExtractor.ts @@ -229,12 +229,53 @@ export async function extractVisibleText(page: Page): Promise { return visibleTextContent; } +interface VertexCaptionConfig { + project?: string; + location: string; + googleAuthOptions?: { credentials: Record; scopes: string[] }; +} + +/** + * Vertex AI is the opt-in for Google Cloud tenants whose Gemini access is + * service-account based — the Vertex endpoint rejects plain API keys, so the + * GEMINI_API_KEY path can never reach it. Enable with + * GOOGLE_GENAI_USE_VERTEXAI=true (the @google/genai SDK's own convention). + * Credentials come from GOOGLE_SERVICE_ACCOUNT_INFO (inline SA JSON) when set, + * else the SDK falls through to Application Default Credentials. Project falls + * back to the SA's own project_id; location defaults to "global". + */ +export function resolveVertexCaptionConfig(): VertexCaptionConfig | null { + if ((process.env.GOOGLE_GENAI_USE_VERTEXAI || "").toLowerCase() !== "true") return null; + let project = process.env.GOOGLE_CLOUD_PROJECT || process.env.GOOGLE_PROJECT_ID; + let googleAuthOptions: VertexCaptionConfig["googleAuthOptions"]; + const saRaw = process.env.GOOGLE_SERVICE_ACCOUNT_INFO; + if (saRaw) { + let sa: Record; + try { + sa = JSON.parse(saRaw) as Record; + } catch { + return null; + } + googleAuthOptions = { + credentials: sa, + scopes: ["https://www.googleapis.com/auth/cloud-platform"], + }; + project = project || (typeof sa.project_id === "string" ? sa.project_id : undefined); + } + const location = process.env.GOOGLE_CLOUD_LOCATION || process.env.GOOGLE_LOCATION || "global"; + return { project, location, googleAuthOptions }; +} + /** * Caption downloaded images using a vision model. * - * Provider is chosen by which API key is present: OPENROUTER_API_KEY → OpenRouter - * (any vision model via its OpenAI-style API), else GEMINI_API_KEY/GOOGLE_API_KEY - * → Google Gemini, else no captioning. OpenRouter wins if both are set. + * Provider is chosen by which credentials are present: OPENROUTER_API_KEY → + * OpenRouter (any vision model via its OpenAI-style API), else + * GOOGLE_GENAI_USE_VERTEXAI=true → Gemini on Vertex AI (service-account/ADC + * auth — see resolveVertexCaptionConfig), else GEMINI_API_KEY/GOOGLE_API_KEY → + * Google Gemini API, else no captioning. OpenRouter wins if several are set; + * the Vertex flag wins over a bare Gemini key because a tenant that sets it is + * saying its key material is Vertex-side. * * Batches requests to stay under free-tier rate limits. * Returns a map of filename -> caption string. @@ -266,22 +307,28 @@ export async function captionImagesWithGemini( } const openRouterKey = process.env.OPENROUTER_API_KEY; const geminiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY; - if (!openRouterKey && !geminiKey) { + const vertexConfig = resolveVertexCaptionConfig(); + if (!openRouterKey && !geminiKey && !vertexConfig) { reportOutcome(); return geminiCaptions; } - // OpenRouter takes priority when both keys are set — it's the explicit opt-in - // for users without Google access. Both providers satisfy the same - // single-image → one-line-caption contract (`captionOne`), so the batching and - // SVG-rasterization loops below stay provider-agnostic. + // OpenRouter takes priority when several credentials are set — it's the + // explicit opt-in for users without Google access. All providers satisfy the + // same single-image → one-line-caption contract (`captionOne`), so the + // batching and SVG-rasterization loops below stay provider-agnostic. const useOpenRouter = Boolean(openRouterKey); - const providerName = useOpenRouter ? "OpenRouter" : "Gemini"; - // Default mirrors the Gemini path's tier (3.x flash-lite). Override per - // provider via HYPERFRAMES_OPENROUTER_MODEL / HYPERFRAMES_GEMINI_MODEL. + const useVertex = !useOpenRouter && Boolean(vertexConfig); + const providerName = useOpenRouter ? "OpenRouter" : useVertex ? "Vertex Gemini" : "Gemini"; + // Defaults are the same 3.1 flash-lite tier on every surface; the two Google + // surfaces publish it under different names (Vertex serves the GA + // "gemini-3.1-flash-lite", the Gemini API only its "-preview" alias). + // Override per provider via HYPERFRAMES_OPENROUTER_MODEL / + // HYPERFRAMES_GEMINI_MODEL. const model = useOpenRouter ? process.env.HYPERFRAMES_OPENROUTER_MODEL || "google/gemini-3.1-flash-lite" - : process.env.HYPERFRAMES_GEMINI_MODEL || "gemini-3.1-flash-lite-preview"; + : process.env.HYPERFRAMES_GEMINI_MODEL || + (useVertex ? "gemini-3.1-flash-lite" : "gemini-3.1-flash-lite-preview"); const requestTimeoutMs = resolveVisionRequestTimeoutMs(); progress("design", `Captioning images with ${providerName} vision...`); @@ -335,10 +382,18 @@ export async function captionImagesWithGemini( }, timeoutMs); }; } else { - // Unreachable when geminiKey is unset (guarded above); re-narrow for TS. - if (!geminiKey) return geminiCaptions; + // Unreachable when neither Vertex nor a key is configured (guarded above); re-narrow for TS. + if (!geminiKey && !vertexConfig) return geminiCaptions; const { GoogleGenAI } = await import("@google/genai"); - const ai = new GoogleGenAI({ apiKey: geminiKey }); + const ai = + useVertex && vertexConfig + ? new GoogleGenAI({ + vertexai: true, + project: vertexConfig.project, + location: vertexConfig.location, + googleAuthOptions: vertexConfig.googleAuthOptions, + }) + : new GoogleGenAI({ apiKey: geminiKey }); captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => { const response = await runBoundedVisionRequest( (signal) => diff --git a/packages/cli/src/capture/index.ts b/packages/cli/src/capture/index.ts index cdcbfd1e9c..902e0f8ca0 100644 --- a/packages/cli/src/capture/index.ts +++ b/packages/cli/src/capture/index.ts @@ -36,6 +36,7 @@ import { detectLibraries, extractVisibleText, captionImagesWithGemini, + resolveVertexCaptionConfig, generateAssetDescriptions, resolveVisionPhaseCompletion, } from "./contentExtractor.js"; @@ -761,11 +762,12 @@ export async function captureWebsite( !skipVision && (process.env.OPENROUTER_API_KEY || process.env.GEMINI_API_KEY || - process.env.GOOGLE_API_KEY) + process.env.GOOGLE_API_KEY || + resolveVertexCaptionConfig()) ); const header = hasVisionKey ? "# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually.\n\nTo find a specific brand or icon, **grep this file for the brand name in the description text** (e.g. `grep -i 'autodesk' asset-descriptions.md`). The Gemini Vision captions identify what's actually in each file — that's the agent's selector.\n\nThe `logo-.svg` filename prefix is a cheap structural hint (DOM said this SVG was inside a `
`, home-link ``, or had an aria-label matching the page brand). It is NOT a content claim — many `logo-*` files are nav icons or decorative shapes. Trust the captions, not the filename prefix.\n\n" - : "# Asset Descriptions\n\n⚠️ GEMINI_API_KEY not set — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY) and re-run.\n\nThe `logo-.svg` filename prefix is a structural hint (DOM said this SVG was inside a `
`, home-link ``, or had an aria-label matching the page brand). To pick the actual brand logo without Vision, open the `logo-*` candidates in a previewer or rasterize them with `sharp` before referencing — composing a fake logo ships off-brand in the final video.\n\n"; + : "# Asset Descriptions\n\n⚠️ No vision credentials — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY), or GOOGLE_GENAI_USE_VERTEXAI=true for Vertex service-account auth, and re-run.\n\nThe `logo-.svg` filename prefix is a structural hint (DOM said this SVG was inside a `
`, home-link ``, or had an aria-label matching the page brand). To pick the actual brand logo without Vision, open the `logo-*` candidates in a previewer or rasterize them with `sharp` before referencing — composing a fake logo ships off-brand in the final video.\n\n"; writeFileSync( join(outputDir, "extracted", "asset-descriptions.md"), header + lines.map((l) => "- " + l).join("\n") + "\n",