From 9038afac6847a42b7733bd3f87ed3a2210e71345 Mon Sep 17 00:00:00 2001 From: ayush-that Date: Wed, 12 Aug 2026 01:30:19 +0530 Subject: [PATCH 1/8] add Grok Build agent backend for deepsec process and setup --- docs/models.md | 25 + .../src/__tests__/agent-defaults.test.ts | 2 + packages/deepsec/src/agent-defaults.ts | 3 + packages/deepsec/src/auth/model-picker.ts | 61 +- packages/deepsec/src/auth/model-route.ts | 10 + packages/deepsec/src/cli.ts | 10 +- packages/deepsec/src/preflight.ts | 35 +- packages/deepsec/src/resolve-agent-type.ts | 5 +- packages/deepsec/src/setup/options.ts | 4 +- .../src/__tests__/grok-build.test.ts | 72 ++ .../processor/src/__tests__/registry.test.ts | 3 +- packages/processor/src/agents/grok-build.ts | 910 ++++++++++++++++++ packages/processor/src/index.ts | 3 + packages/processor/src/setup-agent.ts | 6 +- 14 files changed, 1129 insertions(+), 20 deletions(-) create mode 100644 packages/processor/src/__tests__/grok-build.test.ts create mode 100644 packages/processor/src/agents/grok-build.ts diff --git a/docs/models.md b/docs/models.md index 609d89cb..a9c7c653 100644 --- a/docs/models.md +++ b/docs/models.md @@ -10,8 +10,33 @@ deepsec talks to LLMs through interchangeable agent backends: | `codex` (default) | `gpt-5.5` | `process`, `revalidate` | | `claude` | `claude-opus-4-8` | `process`, `revalidate` | | `pi` | `zai/glm-5.2` | `process`, `revalidate` | +| `grok` | `grok-4.5` | `process`, `revalidate`, setup | | `claude` (triage) | `claude-sonnet-4-6` | `triage` (Claude-only) | +### Grok Build (`--agent grok`) + +Uses the local [Grok Build](https://grok.com) CLI (`grok`) in headless mode +(`grok -p … --output-format json`). Auth is independent of Vercel AI Gateway: + +```bash +# Option A: API key from https://console.x.ai +export XAI_API_KEY=xai-... + +# Option B: browser / device login once +grok login + +# Then: +npx deepsec init --agent grok --model grok-4.5 +# or, later: +pnpm deepsec process --project-id my-app --agent grok --model grok-4.5 +``` + +Requires the `grok` binary on `PATH` (or `GROK_EXECUTABLE`). Each batch runs +with an isolated `GROK_HOME` (auth mirrored, skills/plugins not loaded), +`--tools read_file,grep,list_dir,run_terminal_cmd`, and `--sandbox read-only` +by default (`DEEPSEC_GROK_SANDBOX` overrides; nested sandbox is off when +`DEEPSEC_INSIDE_SANDBOX=1`). + Interactive one-shot setup recommends five benchmark-backed combinations: GPT-5.6 Sol, Claude Opus 5, Kimi K3, Grok 4.5, and the current DeepSeek entry. Deepsec fetches the latest score, reasoning level, harness, and total run cost diff --git a/packages/deepsec/src/__tests__/agent-defaults.test.ts b/packages/deepsec/src/__tests__/agent-defaults.test.ts index fdb62658..96624de0 100644 --- a/packages/deepsec/src/__tests__/agent-defaults.test.ts +++ b/packages/deepsec/src/__tests__/agent-defaults.test.ts @@ -9,6 +9,8 @@ describe("defaultModelForAgent", () => { expect(defaultModelForAgent("codex")).toBe("gpt-5.5"); expect(defaultModelForAgent("pi")).toBe("zai/glm-5.2"); expect(defaultModelForAgent("claude-agent-sdk")).toBe("claude-opus-4-8"); + expect(defaultModelForAgent("grok")).toBe("grok-4.5"); + expect(defaultModelForAgent("grok-build")).toBe("grok-4.5"); }); it("uses the model persisted for the configured harness", () => { diff --git a/packages/deepsec/src/agent-defaults.ts b/packages/deepsec/src/agent-defaults.ts index 92bce879..aca22b3e 100644 --- a/packages/deepsec/src/agent-defaults.ts +++ b/packages/deepsec/src/agent-defaults.ts @@ -14,6 +14,9 @@ export function defaultModelForAgent(agentType: string): string { return "gpt-5.5"; case "pi": return "zai/glm-5.2"; + case "grok": + case "grok-build": + return "grok-4.5"; default: return "claude-opus-4-8"; } diff --git a/packages/deepsec/src/auth/model-picker.ts b/packages/deepsec/src/auth/model-picker.ts index fc69e406..e7eed5f7 100644 --- a/packages/deepsec/src/auth/model-picker.ts +++ b/packages/deepsec/src/auth/model-picker.ts @@ -6,7 +6,7 @@ import type { ModelRoute } from "./model-route.js"; export const DEEPSEC_BENCHMARK_URL = "https://vercel.com/ai-gateway/leaderboards/deepsecbench/results.json"; -export type ModelHarness = "codex" | "claude" | "pi"; +export type ModelHarness = "codex" | "claude" | "pi" | "grok"; export interface BenchmarkResult { rank: number; @@ -108,7 +108,10 @@ function isBenchmarkResult(value: unknown): value is BenchmarkResult { typeof result.modelId === "string" && typeof result.score === "number" && typeof result.cost === "number" && - (result.harness === "codex" || result.harness === "claude" || result.harness === "pi") + (result.harness === "codex" || + result.harness === "claude" || + result.harness === "pi" || + result.harness === "grok") ); } @@ -136,7 +139,14 @@ function strongest(results: BenchmarkResult[], modelId: string): BenchmarkResult } function configuredModel(result: BenchmarkResult): string { - return result.harness === "pi" ? result.modelId : result.model; + // Pi needs the provider/model gateway id. Grok Build takes the bare model id. + if (result.harness === "pi") return result.modelId; + if (result.harness === "grok") { + return result.modelId.startsWith("xai/") + ? result.modelId.slice("xai/".length) + : result.model; + } + return result.model; } function thinkingLevel(reasoning: string): string | undefined { @@ -161,14 +171,23 @@ export function buildRecommendedModelChoices(results: BenchmarkResult[]): Recomm function canonicalHarness(value: string | undefined): ModelHarness | undefined { if (value === "claude-agent-sdk" || value === "claude") return "claude"; + if (value === "grok-build" || value === "grok") return "grok"; if (value === "codex" || value === "pi") return value; return undefined; } function compatibleHarness(route: ModelRoute, requested?: string): ModelHarness | undefined { - if (route.mode === "direct") return route.provider === "anthropic" ? "claude" : "codex"; + // Grok Build authenticates via XAI_API_KEY / grok login, independent of + // the Vercel AI Gateway model route. Prefer an explicit grok request. + const requestedHarness = canonicalHarness(requested); + if (requestedHarness === "grok") return "grok"; + if (route.mode === "direct") { + if (route.provider === "anthropic") return "claude"; + if (route.provider === "xai" || route.provider === "grok") return "grok"; + return "codex"; + } if (route.mode === "custom") return "pi"; - return canonicalHarness(requested); + return requestedHarness; } export function parseModelProfile(value: string | undefined): ModelProfile | undefined { @@ -229,12 +248,14 @@ export async function resolveModelProfile(options: { export function inferModelHarness(slug: string): ModelHarness { if (/^(?:openai\/)?gpt-/i.test(slug)) return "codex"; if (/^(?:anthropic\/)?claude-/i.test(slug)) return "claude"; + if (/^(?:xai\/)?grok-/i.test(slug)) return "grok"; if (slug.includes("/")) return "pi"; return "pi"; } function modelForHarness(slug: string, harness: ModelHarness): string { if (harness === "codex" && slug.startsWith("openai/")) return slug.slice("openai/".length); + if (harness === "grok" && slug.startsWith("xai/")) return slug.slice("xai/".length); if (harness === "claude" && slug.startsWith("anthropic/")) { return slug.slice("anthropic/".length); } @@ -244,6 +265,7 @@ function modelForHarness(slug: string, harness: ModelHarness): string { function displayHarness(harness: ModelHarness): string { if (harness === "claude") return "Claude"; if (harness === "codex") return "Codex"; + if (harness === "grok") return "Grok Build"; return "Pi"; } @@ -261,9 +283,26 @@ export async function promptForModelSelection(options: { const benchmark = await fetchBenchmarkResults(options.fetchImpl); const requiredHarness = compatibleHarness(options.route, options.agent); const recommendations = buildRecommendedModelChoices(benchmark.results); - const choices = recommendations.filter( - (choice) => !requiredHarness || choice.agent === requiredHarness, - ); + // DeepSecBench currently scores Grok models under the Pi harness (AI + // Gateway). When the operator asked for Grok Build, remap those rows so + // they still appear with the native harness + bare model id. + const choices = recommendations + .map((choice) => { + if ( + requiredHarness === "grok" && + choice.agent !== "grok" && + /^(?:xai\/)?grok-/i.test(choice.modelId) + ) { + return { + ...choice, + agent: "grok" as const, + harness: "grok" as const, + configuredModel: modelForHarness(choice.modelId, "grok"), + }; + } + return choice; + }) + .filter((choice) => !requiredHarness || choice.agent === requiredHarness); const prompt = createInterface({ input: process.stdin, output: process.stdout }); try { if (choices.length < recommendations.length) { @@ -300,10 +339,12 @@ export async function promptForModelSelection(options: { let agent = requiredHarness ?? inferModelHarness(slug); if (!requiredHarness) { const harnessAnswer = ( - await prompt.question(`Agent harness (codex, claude, pi) [${agent}]: `) + await prompt.question(`Agent harness (codex, claude, pi, grok) [${agent}]: `) ).trim(); const selectedHarness = canonicalHarness(harnessAnswer || agent); - if (!selectedHarness) throw new Error("Agent harness must be codex, claude, or pi"); + if (!selectedHarness) { + throw new Error("Agent harness must be codex, claude, pi, or grok"); + } agent = selectedHarness; } const level = (await prompt.question("Thinking level [medium]: ")).trim() || "medium"; diff --git a/packages/deepsec/src/auth/model-route.ts b/packages/deepsec/src/auth/model-route.ts index bf19422d..cfcdb0b2 100644 --- a/packages/deepsec/src/auth/model-route.ts +++ b/packages/deepsec/src/auth/model-route.ts @@ -48,6 +48,10 @@ const DEFAULTS: Record = { env: "OPENAI_API_KEY", baseUrl: "https://api.openai.com/v1", }, + xai: { + env: "XAI_API_KEY", + baseUrl: "https://api.x.ai/v1", + }, }; function checkedUrl(value: string, label: string): URL { @@ -66,6 +70,12 @@ export function modelRouteCompatibilityError( route: ModelRoute, agentType: string, ): string | undefined { + // Grok Build authenticates via XAI_API_KEY / `grok login`, not AI Gateway + // or the OpenAI/Anthropic direct routes. Leave env alone and let preflight + // check the Grok CLI credentials instead. + if (agentType === "grok" || agentType === "grok-build") { + return `Grok Build does not use the configured model route (uses XAI_API_KEY / grok login)`; + } if (route.mode === "custom" && agentType !== "pi") { return `Custom model routes require --agent pi (received ${agentType})`; } diff --git a/packages/deepsec/src/cli.ts b/packages/deepsec/src/cli.ts index 44071e7d..d905529a 100755 --- a/packages/deepsec/src/cli.ts +++ b/packages/deepsec/src/cli.ts @@ -92,7 +92,7 @@ program .option("--scaffold-only", "Only create files; do not install, connect, scan, or process") .option("--skip-install", "Require dependencies to exist instead of running pnpm/npm install") .option("--package-manager ", "Installer to use: pnpm or npm", parsePackageManager) - .option("--agent ", "AI agent: codex, claude, or pi") + .option("--agent ", "AI agent: codex, claude, pi, or grok") .option("--model ", "Model for repository analysis and processing") .option("--model-profile ", "Benchmark profile: best, value, or budget") .option("--thinking-level ", "Reasoning effort: minimal, low, medium, high, or xhigh") @@ -186,7 +186,7 @@ program .option("--status", "Show resumable phase status without running setup") .option("--skip-install", "Require dependencies to exist instead of running pnpm/npm install") .option("--package-manager ", "Installer to use: pnpm or npm", parsePackageManager) - .option("--agent ", "AI agent: codex, claude, or pi") + .option("--agent ", "AI agent: codex, claude, pi, or grok") .option("--model ", "Model for repository analysis and processing") .option("--model-profile ", "Benchmark profile: best, value, or budget") .option("--thinking-level ", "Reasoning effort: minimal, low, medium, high, or xhigh") @@ -290,7 +290,7 @@ program .option("--run-id ", "Resume a specific processing run") .option( "--agent ", - "Agent plugin type: codex, claude, or pi (default: defaultAgent in deepsec.config.ts, else codex)", + "Agent plugin type: codex, claude, pi, or grok (default: defaultAgent in deepsec.config.ts, else codex)", ) .option( "--model ", @@ -369,7 +369,7 @@ program .option("--run-id ", "Resume a specific revalidation run") .option( "--agent ", - "Agent plugin type: codex, claude, or pi (default: defaultAgent in deepsec.config.ts, else codex)", + "Agent plugin type: codex, claude, pi, or grok (default: defaultAgent in deepsec.config.ts, else codex)", ) .option( "--model ", @@ -481,7 +481,7 @@ program .option("--require-owner", "Drop findings that have no ownership data (no assignee, no teams)") .option( "--only-agent ", - "Only export findings produced by this agent backend (e.g. codex, claude, pi)", + "Only export findings produced by this agent backend (e.g. codex, claude, pi, grok)", ) .option( "--only-marker ", diff --git a/packages/deepsec/src/preflight.ts b/packages/deepsec/src/preflight.ts index 8519da9f..a2de4517 100644 --- a/packages/deepsec/src/preflight.ts +++ b/packages/deepsec/src/preflight.ts @@ -187,11 +187,31 @@ function hasLocalPiAgent(): boolean { return existsSync(join(piHome, "auth.json")); } +/** + * Grok Build: either XAI_API_KEY is set, or the user has run `grok login` + * and we can see auth.json under GROK_HOME / ~/.grok. + */ +function hasLocalGrokAgent(): boolean { + if (process.env.XAI_API_KEY) return true; + const homes = [process.env.GROK_HOME, join(homedir(), ".grok")].filter( + (p): p is string => typeof p === "string" && p.length > 0, + ); + for (const home of homes) { + if (existsSync(join(home, "auth.json"))) return true; + } + // Binary on PATH is a soft signal; the CLI errors clearly if not logged in. + return whichSync("grok"); +} + +function isGrok(agentType: string | undefined): boolean { + return agentType === "grok" || agentType === "grok-build"; +} + // Built-in backends we know how to credential-check. Agents registered // via plugins (deepsec.config.ts → plugins: [{ agents: [...] }]) handle // their own credential resolution, so we skip the check for anything // other than these. -const KNOWN_BACKENDS = new Set(["claude-agent-sdk", "codex", "pi"]); +const KNOWN_BACKENDS = new Set(["claude-agent-sdk", "codex", "pi", "grok"]); /** * Verify the orchestrator has an AI credential the chosen agent can use. @@ -262,6 +282,19 @@ export function assertAgentCredential( ); } + if (isGrok(agentType)) { + if (process.env.XAI_API_KEY) return; + if (!options.inSandbox && hasLocalGrokAgent()) return; + throw new Error( + `Missing AI credentials for --agent grok.\n` + + `\n` + + ` Option A: export XAI_API_KEY=xai-… (from https://console.x.ai)\n` + + ` Option B: run \`grok login\` once on this machine\n` + + ` Ensure the \`grok\` CLI is on PATH (or set GROK_EXECUTABLE).\n` + + ` Setup: ${SETUP_DOC_URL}`, + ); + } + if (anthropic || anthropicApi) return; if (!options.inSandbox && hasLocalClaudeAgent()) return; const displayAgent = diff --git a/packages/deepsec/src/resolve-agent-type.ts b/packages/deepsec/src/resolve-agent-type.ts index 3932a5b2..bbfdf19e 100644 --- a/packages/deepsec/src/resolve-agent-type.ts +++ b/packages/deepsec/src/resolve-agent-type.ts @@ -10,5 +10,8 @@ import { getConfig } from "@deepsec/core"; */ export function resolveAgentType(provided: string | undefined): string { const resolved = provided ?? getConfig()?.defaultAgent ?? "codex"; - return resolved === "claude" ? "claude-agent-sdk" : resolved; + if (resolved === "claude") return "claude-agent-sdk"; + // Accept both short and long forms for the Grok Build harness. + if (resolved === "grok-build") return "grok"; + return resolved; } diff --git a/packages/deepsec/src/setup/options.ts b/packages/deepsec/src/setup/options.ts index 32102093..ccac63a4 100644 --- a/packages/deepsec/src/setup/options.ts +++ b/packages/deepsec/src/setup/options.ts @@ -21,7 +21,9 @@ export function modelRouteFromCli(options: ModelRouteCliOptions): ModelRoute { ? "anthropic" : options.agent === "codex" ? "openai" - : undefined); + : options.agent === "grok" || options.agent === "grok-build" + ? "xai" + : undefined); if (!provider) throw new Error(`--model-auth ${mode} requires --ai-provider`); if (mode === "direct") { return { diff --git a/packages/processor/src/__tests__/grok-build.test.ts b/packages/processor/src/__tests__/grok-build.test.ts new file mode 100644 index 00000000..6ed89d08 --- /dev/null +++ b/packages/processor/src/__tests__/grok-build.test.ts @@ -0,0 +1,72 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildGrokEnv, makeIsolatedGrokHome } from "../agents/grok-build.js"; +import { createDefaultAgentRegistry } from "../index.js"; + +const homes: string[] = []; + +afterEach(() => { + for (const home of homes.splice(0)) { + try { + fs.rmSync(home, { recursive: true, force: true }); + } catch { + // ignore + } + } +}); + +describe("Grok Build agent", () => { + it("registers as type 'grok' in the default agent registry", () => { + const registry = createDefaultAgentRegistry(); + expect(registry.types()).toContain("grok"); + expect(registry.get("grok")?.type).toBe("grok"); + }); + + it("buildGrokEnv allowlists only safe vars and injects GROK_HOME", () => { + const prev = { ...process.env }; + try { + process.env.PATH = "/usr/bin"; + process.env.HOME = "/Users/test"; + process.env.XAI_API_KEY = "xai-test-key"; + process.env.GITHUB_TOKEN = "should-not-leak"; + process.env.AWS_SECRET_ACCESS_KEY = "should-not-leak"; + process.env.LC_ALL = "en_US.UTF-8"; + + const env = buildGrokEnv("/tmp/fake-grok-home"); + expect(env.GROK_HOME).toBe("/tmp/fake-grok-home"); + expect(env.GROK_DISABLE_AUTOUPDATER).toBe("1"); + expect(env.XAI_API_KEY).toBe("xai-test-key"); + expect(env.PATH).toBe("/usr/bin"); + expect(env.LC_ALL).toBe("en_US.UTF-8"); + expect(env.GITHUB_TOKEN).toBeUndefined(); + expect(env.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + } finally { + process.env = prev; + } + }); + + it("makeIsolatedGrokHome creates a config and mirrors auth when available", () => { + // Seed a fake user auth so we exercise the mirror path even if the + // developer machine has no real grok login in this environment. + const seed = fs.mkdtempSync(path.join(os.tmpdir(), "deepsec-grok-seed-")); + homes.push(seed); + fs.writeFileSync(path.join(seed, "auth.json"), JSON.stringify({ token: "test" }), { + mode: 0o600, + }); + const prev = process.env.GROK_HOME; + process.env.GROK_HOME = seed; + try { + const home = makeIsolatedGrokHome(); + homes.push(home); + expect(fs.existsSync(path.join(home, "config.toml"))).toBe(true); + expect(fs.existsSync(path.join(home, "auth.json"))).toBe(true); + // No skills / plugins directory copied in. + expect(fs.existsSync(path.join(home, "skills"))).toBe(false); + } finally { + if (prev === undefined) delete process.env.GROK_HOME; + else process.env.GROK_HOME = prev; + } + }); +}); diff --git a/packages/processor/src/__tests__/registry.test.ts b/packages/processor/src/__tests__/registry.test.ts index 2f9e06c1..6f245217 100644 --- a/packages/processor/src/__tests__/registry.test.ts +++ b/packages/processor/src/__tests__/registry.test.ts @@ -57,6 +57,7 @@ describe("createDefaultAgentRegistry", () => { expect(registry.get("claude-agent-sdk")).toBeDefined(); expect(registry.get("codex")).toBeDefined(); expect(registry.get("pi")).toBeDefined(); - expect(registry.types().sort()).toEqual(["claude-agent-sdk", "codex", "pi"]); + expect(registry.get("grok")).toBeDefined(); + expect(registry.types().sort()).toEqual(["claude-agent-sdk", "codex", "grok", "pi"]); }); }); diff --git a/packages/processor/src/agents/grok-build.ts b/packages/processor/src/agents/grok-build.ts new file mode 100644 index 00000000..673fe34b --- /dev/null +++ b/packages/processor/src/agents/grok-build.ts @@ -0,0 +1,910 @@ +import { spawn } from "node:child_process"; +import * as crypto from "node:crypto"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { RefusalReport } from "@deepsec/core"; +import { + backoff, + buildInvestigateJsonRepairPrompt, + buildInvestigatePrompt, + buildRevalidateJsonRepairPrompt, + buildRevalidatePrompt, + classifyQuotaError, + formatJsonRepairFailureDebugText, + isTransientError, + jsonRepairFailureError, + MAX_ATTEMPTS, + type ParsedInvestigateResults, + parseInvestigateResults, + parseRefusalReport, + parseRevalidateVerdicts, + QuotaExhaustedError, + REFUSAL_FOLLOWUP_PROMPT, + runInvestigateFieldRepairLoop, + runRevalidateIdRepairLoop, + writeParseFailureDebug, +} from "./shared.js"; +import type { + AgentPlugin, + AgentProgress, + BatchMeta, + InvestigateOutput, + InvestigateParams, + InvestigateResult, + RevalidateOutput, + RevalidateParams, + RevalidateRawResponse, + RevalidateVerdict, + SetupTaskParams, +} from "./types.js"; + +/** + * Grok Build (xAI) coding-agent backend. + * + * Spawns the local `grok` CLI in headless mode (`-p`) with a restricted + * tool allowlist and optional OS sandbox. Same prompt + JSON schema as the + * other backends; investigation output is parsed by the shared helpers. + * + * Auth: + * - `XAI_API_KEY` in the environment, or + * - an existing `~/.grok/auth.json` from `grok login` (mirrored into an + * isolated per-run GROK_HOME so user skills/plugins are not loaded). + */ + +const DEFAULT_MODEL = "grok-4.5"; +const DEFAULT_THINKING_LEVEL = "xhigh"; + +/** Read-only tools for investigation / revalidation (internal Grok tool ids). */ +const INVESTIGATE_TOOLS = "read_file,grep,list_dir,run_terminal_cmd"; +/** Even tighter set for setup analysis. */ +const SETUP_TOOLS = "read_file,grep,list_dir"; + +const GROK_ENV_ALLOWLIST = new Set([ + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "TZ", + "LANG", + "LANGUAGE", + "LC_ALL", + "LC_CTYPE", + "LC_MESSAGES", + "LC_COLLATE", + "LC_NUMERIC", + "LC_TIME", + "TMPDIR", + "TMP", + "TEMP", + "PWD", + "GROK_HOME", + "GROK_DISABLE_AUTOUPDATER", + "GROK_SANDBOX", + "RUST_LOG", + "RUST_BACKTRACE", +]); + +export interface GrokJsonResult { + text?: string; + stopReason?: string; + sessionId?: string; + requestId?: string; + num_turns?: number; + total_cost_usd?: number; + usage?: { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; + reasoning_tokens?: number; + total_tokens?: number; + }; + modelUsage?: Record; + type?: string; + message?: string; +} + +interface GrokRunResult { + resultText: string; + meta: Partial; + raw: GrokJsonResult; +} + +interface GrokRunOptions { + prompt: string; + projectRoot: string; + model: string; + maxTurns: number; + tools: string; + thinkingLevel: string; + /** Resume an existing session (JSON repair / refusal follow-up). */ + resumeSessionId?: string; + signal?: AbortSignal; + onProgress?: (progress: AgentProgress) => void; + /** Isolated GROK_HOME for this batch; created if omitted. */ + grokHome?: string; + /** Keep the home dir after the run (needed when resuming). */ + keepHome?: boolean; +} + +function resolveThinkingLevel(config: Record): string { + const level = config.thinkingLevel ?? config.reasoningEffort; + if (typeof level === "string" && level.length > 0) return level; + return DEFAULT_THINKING_LEVEL; +} + +function resolveGrokBinary(): string { + if (process.env.GROK_EXECUTABLE) return process.env.GROK_EXECUTABLE; + // Common install locations before falling back to PATH. + const candidates = [ + path.join(os.homedir(), ".local", "bin", "grok"), + path.join(os.homedir(), ".grok", "bin", "grok"), + "/opt/homebrew/bin/grok", + "/usr/local/bin/grok", + ]; + for (const c of candidates) { + try { + fs.accessSync(c, fs.constants.X_OK); + return c; + } catch { + // try next + } + } + return "grok"; +} + +/** + * Build a minimal GROK_HOME so deepsec does not inherit the operator's + * 400+ skills, MCP servers, and plugins (those bloat the system prompt + * and burn tokens on every batch). + * + * Mirrors `auth.json` when present so OAuth login works without + * XAI_API_KEY. Prefer symlink so token refresh writes back to the + * real home; copy as fallback. + */ +export function makeIsolatedGrokHome(): string { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "deepsec-grok-home-")); + fs.writeFileSync( + path.join(home, "config.toml"), + [ + "[ui]", + 'permission_mode = "dontAsk"', + "", + "[cli]", + "auto_update = false", + "", + ].join("\n"), + { mode: 0o600 }, + ); + + const userHomes = [ + process.env.GROK_HOME, + path.join(os.homedir(), ".grok"), + ].filter((p): p is string => typeof p === "string" && p.length > 0); + + for (const userHome of userHomes) { + const auth = path.join(userHome, "auth.json"); + if (!fs.existsSync(auth)) continue; + const dst = path.join(home, "auth.json"); + try { + fs.symlinkSync(auth, dst); + } catch { + fs.copyFileSync(auth, dst); + fs.chmodSync(dst, 0o600); + } + break; + } + + return home; +} + +/** Exported for tests. */ +export function buildGrokEnv(grokHome: string): Record { + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (typeof v !== "string") continue; + if (GROK_ENV_ALLOWLIST.has(k) || k.startsWith("LC_")) { + env[k] = v; + } + } + env.GROK_HOME = grokHome; + env.GROK_DISABLE_AUTOUPDATER = "1"; + // Forward only the credential the CLI actually needs. Never ship + // GITHUB_TOKEN / AWS_* / etc. into a prompt-injectionable shell. + for (const k of ["XAI_API_KEY", "XAI_API_BASE_URL"]) { + const v = process.env[k]; + if (typeof v === "string") env[k] = v; + } + return env; +} + +function sandboxProfile(): string { + // Nested OS sandbox inside a Vercel Sandbox microVM is unnecessary. + if (process.env.DEEPSEC_INSIDE_SANDBOX === "1") return "off"; + // Read-only project FS; agent can still write session state under GROK_HOME. + return process.env.DEEPSEC_GROK_SANDBOX ?? "read-only"; +} + +function parseGrokStdout(stdout: string): GrokJsonResult { + const trimmed = stdout.trim(); + if (!trimmed) throw new Error("Grok produced empty stdout"); + // Prefer the last complete JSON object (in case any banner leaked). + try { + return JSON.parse(trimmed) as GrokJsonResult; + } catch { + const start = trimmed.lastIndexOf("{"); + if (start < 0) throw new Error(`Grok stdout was not JSON: ${trimmed.slice(0, 200)}`); + return JSON.parse(trimmed.slice(start)) as GrokJsonResult; + } +} + +function metaFromGrokJson(raw: GrokJsonResult): Partial { + const meta: Partial = { + agentSessionId: raw.sessionId, + numTurns: raw.num_turns, + }; + if (typeof raw.total_cost_usd === "number") { + meta.costUsd = raw.total_cost_usd; + } + if (raw.usage) { + meta.usage = { + inputTokens: raw.usage.input_tokens ?? 0, + outputTokens: raw.usage.output_tokens ?? 0, + cacheReadInputTokens: raw.usage.cache_read_input_tokens ?? 0, + cacheCreationInputTokens: raw.usage.cache_creation_input_tokens ?? 0, + }; + } + return meta; +} + +/** + * Run one headless Grok prompt. Uses `--output-format json` so the final + * result is a single parseable object with text + spend metadata. + * Progress is coarse (started / complete) because the json format only + * emits at the end; streaming-messages-json is available later if we + * need tool-level progress. + */ +export async function runGrokHeadless(opts: GrokRunOptions): Promise { + const bin = resolveGrokBinary(); + const grokHome = opts.grokHome ?? makeIsolatedGrokHome(); + const ownHome = opts.grokHome === undefined; + const sandbox = sandboxProfile(); + + const args = [ + "-p", + opts.prompt, + "--cwd", + opts.projectRoot, + "--output-format", + "json", + "--permission-mode", + "dontAsk", + "--tools", + opts.tools, + "--max-turns", + String(opts.maxTurns), + "-m", + opts.model, + "--reasoning-effort", + opts.thinkingLevel, + "--sandbox", + sandbox, + "--no-subagents", + "--disable-web-search", + "--no-memory", + "--verbatim", + // Block write tools even if a future CLI change expands the allowlist. + "--disallowed-tools", + "search_replace,write,image_gen,image_edit,image_to_video,reference_to_video,Agent", + ]; + + if (opts.resumeSessionId) { + args.push("--resume", opts.resumeSessionId); + } + + const env = buildGrokEnv(grokHome); + // Write large prompts to a temp file to stay under ARG_MAX. + let promptFile: string | undefined; + if (Buffer.byteLength(opts.prompt, "utf8") > 80_000) { + promptFile = path.join( + os.tmpdir(), + `deepsec-grok-prompt-${crypto.randomBytes(8).toString("hex")}.txt`, + ); + fs.writeFileSync(promptFile, opts.prompt, { mode: 0o600 }); + // Replace -p PROMPT with --prompt-file. + const pIdx = args.indexOf("-p"); + if (pIdx >= 0) { + args.splice(pIdx, 2, "--prompt-file", promptFile); + } + } + + opts.onProgress?.({ + type: "started", + message: `Running Grok Build (${opts.model}, effort=${opts.thinkingLevel})`, + }); + + try { + const { stdout, stderr, code } = await spawnCollect({ + bin, + args, + env, + cwd: opts.projectRoot, + signal: opts.signal, + }); + + if (code !== 0) { + const errText = (stderr || stdout || `exit ${code}`).trim(); + const quota = classifyQuotaError(errText); + if (quota) throw new QuotaExhaustedError(quota, errText); + // JSON error objects on stdout with non-zero exit. + try { + const errObj = parseGrokStdout(stdout); + if (errObj.type === "error" || errObj.message) { + const msg = errObj.message ?? errText; + const q = classifyQuotaError(msg); + if (q) throw new QuotaExhaustedError(q, msg); + throw new Error(`Grok failed: ${msg}`); + } + } catch (e) { + if (e instanceof QuotaExhaustedError) throw e; + // fall through + } + throw new Error(`Grok exited ${code}: ${errText.slice(0, 500)}`); + } + + const raw = parseGrokStdout(stdout); + if (raw.type === "error") { + const msg = raw.message ?? "unknown Grok error"; + const q = classifyQuotaError(msg); + if (q) throw new QuotaExhaustedError(q, msg); + throw new Error(`Grok error: ${msg}`); + } + + const resultText = String(raw.text ?? "").trim(); + if (!resultText) { + throw new Error( + `Grok produced no result text (stopReason=${raw.stopReason ?? "?"}, turns=${raw.num_turns ?? "?"}).`, + ); + } + + return { + resultText, + meta: metaFromGrokJson(raw), + raw, + }; + } finally { + if (promptFile) { + try { + fs.unlinkSync(promptFile); + } catch { + // ignore + } + } + if (ownHome && !opts.keepHome) { + try { + fs.rmSync(grokHome, { recursive: true, force: true }); + } catch { + // ignore + } + } + } +} + +function spawnCollect(params: { + bin: string; + args: string[]; + env: Record; + cwd: string; + signal?: AbortSignal; +}): Promise<{ stdout: string; stderr: string; code: number | null }> { + return new Promise((resolve, reject) => { + if (params.signal?.aborted) { + reject(new Error("Aborted before Grok spawn")); + return; + } + + const child = spawn(params.bin, params.args, { + cwd: params.cwd, + env: params.env, + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + child.stdout?.setEncoding("utf8"); + child.stderr?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr?.on("data", (chunk: string) => { + stderr += chunk; + }); + + const onAbort = () => { + child.kill("SIGTERM"); + // Escalate if the CLI ignores SIGTERM. + setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { + // ignore + } + }, 2_000).unref?.(); + }; + params.signal?.addEventListener("abort", onAbort, { once: true }); + + child.on("error", (err) => { + params.signal?.removeEventListener("abort", onAbort); + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + reject( + new Error( + `Grok Build CLI not found (${params.bin}). Install Grok Build and ensure \`grok\` is on PATH, or set GROK_EXECUTABLE.`, + ), + ); + return; + } + reject(err); + }); + + child.on("close", (code) => { + params.signal?.removeEventListener("abort", onAbort); + resolve({ stdout, stderr, code }); + }); + }); +} + +async function runToollessFollowUp(params: { + sessionId: string | undefined; + grokHome: string; + projectRoot: string; + model: string; + thinkingLevel: string; + prompt: string; + signal?: AbortSignal; +}): Promise { + if (!params.sessionId) return undefined; + try { + const run = await runGrokHeadless({ + prompt: params.prompt, + projectRoot: params.projectRoot, + model: params.model, + maxTurns: 1, + tools: "", + thinkingLevel: "low", + resumeSessionId: params.sessionId, + signal: params.signal, + grokHome: params.grokHome, + keepHome: true, + }); + return run.resultText; + } catch { + return undefined; + } +} + +export async function runGrokSetupTask(params: SetupTaskParams): Promise { + const model = (params.config.model as string) ?? DEFAULT_MODEL; + const thinkingLevel = resolveThinkingLevel(params.config); + params.onProgress?.({ + type: "started", + message: `Understanding repository with Grok Build (${model})`, + }); + const run = await runGrokHeadless({ + prompt: params.prompt, + projectRoot: params.projectRoot, + model, + maxTurns: (params.config.maxTurns as number) ?? 40, + tools: SETUP_TOOLS, + thinkingLevel, + signal: params.signal, + onProgress: params.onProgress, + }); + if (!run.resultText.trim()) throw new Error("Grok produced no setup result"); + params.onProgress?.({ type: "complete", message: "Repository setup analysis complete" }); + return run.resultText.trim(); +} + +export class GrokBuildAgentPlugin implements AgentPlugin { + type = "grok"; + + async *investigate(params: InvestigateParams): AsyncGenerator { + const { batch, projectRoot, promptTemplate, projectInfo, config, signal, projectId } = params; + const model = (config.model as string) ?? DEFAULT_MODEL; + const maxTurns = (config.maxTurns as number) ?? 150; + const thinkingLevel = resolveThinkingLevel(config); + const prompt = buildInvestigatePrompt({ promptTemplate, projectInfo, batch }); + const startTime = Date.now(); + + let resultText = ""; + let lastError = ""; + let sdkMeta: Partial = {}; + let sessionId: string | undefined; + let grokHome: string | undefined; + let turnCount = 0; + + yield { + type: "started", + message: `Investigating ${batch.length} file(s) with Grok Build (${model})`, + }; + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + if (attempt > 1) { + yield { + type: "thinking", + message: `Retrying Grok batch after transient error (attempt ${attempt}/${MAX_ATTEMPTS}): ${lastError.slice(0, 200)}`, + }; + resultText = ""; + lastError = ""; + sdkMeta = {}; + sessionId = undefined; + if (grokHome) { + try { + fs.rmSync(grokHome, { recursive: true, force: true }); + } catch { + // ignore + } + grokHome = undefined; + } + } + + try { + grokHome = makeIsolatedGrokHome(); + const run = await runGrokHeadless({ + prompt, + projectRoot, + model, + maxTurns, + tools: INVESTIGATE_TOOLS, + thinkingLevel, + signal, + grokHome, + keepHome: true, + onProgress: (p) => { + // Coarse progress only from the headless json path. + if (p.type === "tool_use" || p.type === "thinking") { + // no-op bridge; reserved for streaming mode + } + }, + }); + resultText = run.resultText; + sdkMeta = run.meta; + sessionId = run.raw.sessionId; + turnCount = run.raw.num_turns ?? 0; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + if (err instanceof QuotaExhaustedError) { + cleanupHome(grokHome); + throw err; + } + yield { type: "error", message: `Grok error: ${lastError.slice(0, 300)}` }; + } + + if (resultText) break; + const quotaSource = classifyQuotaError(lastError); + if (quotaSource) { + cleanupHome(grokHome); + throw new QuotaExhaustedError(quotaSource, lastError); + } + if (attempt >= MAX_ATTEMPTS || !isTransientError(lastError)) break; + await backoff(attempt); + } + + if (!resultText) { + cleanupHome(grokHome); + throw new Error( + `Grok Build produced no investigation result after ${MAX_ATTEMPTS} attempt(s). ` + + `Last error: ${lastError || "(none captured)"}.`, + ); + } + + const durationMs = Date.now() - startTime; + let parsed: ParsedInvestigateResults; + try { + parsed = parseInvestigateResults(resultText, batch); + } catch (err) { + yield { + type: "thinking", + message: "Grok returned non-JSON investigation output; requesting JSON-only repair", + }; + const repairText = await runToollessFollowUp({ + sessionId, + grokHome: grokHome!, + projectRoot, + model, + thinkingLevel, + prompt: buildInvestigateJsonRepairPrompt(batch), + signal, + }); + if (repairText === undefined) { + writeParseFailureDebug({ + projectId, + phase: "investigate", + agentType: this.type, + resultText, + error: err, + batch, + }); + cleanupHome(grokHome); + throw err; + } + try { + parsed = parseInvestigateResults(repairText, batch); + resultText = repairText; + yield { type: "thinking", message: "Grok JSON repair succeeded" }; + } catch (repairErr) { + const combinedError = jsonRepairFailureError(err, repairErr); + writeParseFailureDebug({ + projectId, + phase: "investigate", + agentType: this.type, + resultText: formatJsonRepairFailureDebugText(resultText, repairText), + error: combinedError, + batch, + }); + cleanupHome(grokHome); + throw combinedError; + } + } + + let results: InvestigateResult[] = parsed.results; + if (parsed.invalid.length > 0) { + const fieldRepair = yield* runInvestigateFieldRepairLoop({ + results, + invalid: parsed.invalid, + batch, + followUp: (p) => + runToollessFollowUp({ + sessionId, + grokHome: grokHome!, + projectRoot, + model, + thinkingLevel, + prompt: p, + signal, + }), + agentLabel: "Grok", + agentType: this.type, + projectId, + }); + results = fieldRepair.results; + } + + let refusal: RefusalReport | undefined; + const refusalRaw = await runToollessFollowUp({ + sessionId, + grokHome: grokHome!, + projectRoot, + model, + thinkingLevel, + prompt: REFUSAL_FOLLOWUP_PROMPT, + signal, + }); + if (refusalRaw) refusal = parseRefusalReport(refusalRaw); + if (refusal?.refused) { + yield { + type: "thinking", + message: `Refusal detected: ${refusal.reason ?? "see raw"}`, + }; + } + + const costStr = sdkMeta.costUsd != null ? ` $${sdkMeta.costUsd.toFixed(3)}` : ""; + const tokensStr = sdkMeta.usage + ? ` ${sdkMeta.usage.inputTokens + sdkMeta.usage.outputTokens} tokens` + : ""; + yield { + type: "complete", + message: `Investigation complete (${(durationMs / 1000).toFixed(1)}s, ${turnCount} turns${costStr}${tokensStr}${refusal?.refused ? " refusal" : ""})`, + }; + + cleanupHome(grokHome); + return { + results, + meta: { + durationMs, + ...sdkMeta, + refusal, + }, + }; + } + + async *revalidate(params: RevalidateParams): AsyncGenerator { + const { + batch, + projectRoot, + projectInfo, + config, + force = false, + onlyFindingIds, + signal, + projectId, + } = params; + const model = (config.model as string) ?? DEFAULT_MODEL; + const maxTurns = (config.maxTurns as number) ?? 150; + const thinkingLevel = resolveThinkingLevel(config); + + const { prompt, totalFindings, expected } = buildRevalidatePrompt({ + batch, + projectRoot, + projectInfo, + force, + onlyFindingIds: onlyFindingIds ? new Set(onlyFindingIds) : undefined, + }); + + yield { + type: "started", + message: `Revalidating ${totalFindings} finding(s) across ${batch.length} file(s) with Grok Build (${model})`, + }; + + const startTime = Date.now(); + let resultText = ""; + let lastError = ""; + let sdkMeta: Partial = {}; + let sessionId: string | undefined; + let grokHome: string | undefined; + const rawResponses: RevalidateRawResponse[] = []; + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + if (attempt > 1) { + yield { + type: "thinking", + message: `Retrying Grok revalidation after transient error (attempt ${attempt}/${MAX_ATTEMPTS}): ${lastError.slice(0, 200)}`, + }; + resultText = ""; + lastError = ""; + sdkMeta = {}; + sessionId = undefined; + cleanupHome(grokHome); + grokHome = undefined; + } + + try { + grokHome = makeIsolatedGrokHome(); + const run = await runGrokHeadless({ + prompt, + projectRoot, + model, + maxTurns, + tools: INVESTIGATE_TOOLS, + thinkingLevel, + signal, + grokHome, + keepHome: true, + }); + resultText = run.resultText; + sdkMeta = run.meta; + sessionId = run.raw.sessionId; + rawResponses.push({ kind: "initial", rawText: resultText }); + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + if (err instanceof QuotaExhaustedError) { + cleanupHome(grokHome); + throw err; + } + yield { type: "error", message: `Grok error: ${lastError.slice(0, 300)}` }; + } + + if (resultText) break; + const quotaSource = classifyQuotaError(lastError); + if (quotaSource) { + cleanupHome(grokHome); + throw new QuotaExhaustedError(quotaSource, lastError); + } + if (attempt >= MAX_ATTEMPTS || !isTransientError(lastError)) break; + await backoff(attempt); + } + + if (!resultText) { + cleanupHome(grokHome); + throw new Error( + `Grok Build produced no revalidation result after ${MAX_ATTEMPTS} attempt(s). ` + + `Last error: ${lastError || "(none captured)"}.`, + ); + } + + let verdicts: RevalidateVerdict[]; + try { + verdicts = parseRevalidateVerdicts(resultText); + } catch (err) { + yield { + type: "thinking", + message: "Grok returned non-JSON revalidation output; requesting JSON-only repair", + }; + const repairPrompt = buildRevalidateJsonRepairPrompt(expected); + const repairText = await runToollessFollowUp({ + sessionId, + grokHome: grokHome!, + projectRoot, + model, + thinkingLevel, + prompt: repairPrompt, + signal, + }); + if (repairText === undefined) { + writeParseFailureDebug({ + projectId, + phase: "revalidate", + agentType: this.type, + resultText, + error: err, + batch, + }); + cleanupHome(grokHome); + throw err; + } + rawResponses.push({ kind: "json-repair", prompt: repairPrompt, rawText: repairText }); + try { + verdicts = parseRevalidateVerdicts(repairText); + resultText = repairText; + yield { type: "thinking", message: "Grok revalidation JSON repair succeeded" }; + } catch (repairErr) { + const combinedError = jsonRepairFailureError(err, repairErr); + writeParseFailureDebug({ + projectId, + phase: "revalidate", + agentType: this.type, + resultText: formatJsonRepairFailureDebugText(resultText, repairText), + error: combinedError, + batch, + }); + cleanupHome(grokHome); + throw combinedError; + } + } + + const idRepair = yield* runRevalidateIdRepairLoop({ + expected, + verdicts, + initialRawText: resultText, + followUp: async (p) => + runToollessFollowUp({ + sessionId, + grokHome: grokHome!, + projectRoot, + model, + thinkingLevel, + prompt: p, + signal, + }), + agentLabel: "Grok", + }); + verdicts = idRepair.verdicts; + // Prefer the repair loop's complete transcript (includes initial). + const finalRawResponses = + idRepair.rawResponses.length > 0 + ? [ + ...rawResponses.filter((r) => r.kind !== "initial"), + ...idRepair.rawResponses, + ] + : rawResponses; + + const durationMs = Date.now() - startTime; + const costStr = sdkMeta.costUsd != null ? ` $${sdkMeta.costUsd.toFixed(3)}` : ""; + yield { + type: "complete", + message: `Revalidation complete (${(durationMs / 1000).toFixed(1)}s, ${verdicts.length} verdicts${costStr})`, + }; + + cleanupHome(grokHome); + return { + verdicts, + meta: { + durationMs, + ...sdkMeta, + }, + rawResponses: finalRawResponses, + repairAttempts: idRepair.repairAttempts, + }; + } +} + +function cleanupHome(home: string | undefined): void { + if (!home) return; + try { + fs.rmSync(home, { recursive: true, force: true }); + } catch { + // ignore + } +} diff --git a/packages/processor/src/index.ts b/packages/processor/src/index.ts index 09bd3923..966be786 100644 --- a/packages/processor/src/index.ts +++ b/packages/processor/src/index.ts @@ -22,6 +22,7 @@ import { import { noiseScore, readTechJson } from "@deepsec/scanner"; import { ClaudeAgentSdkPlugin } from "./agents/claude-agent-sdk.js"; import { CodexAgentSdkPlugin } from "./agents/codex-sdk.js"; +import { GrokBuildAgentPlugin } from "./agents/grok-build.js"; import { PiAgentPlugin } from "./agents/pi-sdk.js"; import { AgentRegistry } from "./agents/registry.js"; import { QuotaExhaustedError, type QuotaSource } from "./agents/shared.js"; @@ -45,6 +46,7 @@ import { export { ClaudeAgentSdkPlugin } from "./agents/claude-agent-sdk.js"; export { CodexAgentSdkPlugin } from "./agents/codex-sdk.js"; +export { GrokBuildAgentPlugin } from "./agents/grok-build.js"; export { PiAgentPlugin } from "./agents/pi-sdk.js"; export { AgentRegistry } from "./agents/registry.js"; export { @@ -90,6 +92,7 @@ export function createDefaultAgentRegistry(): AgentRegistry { registry.register(new ClaudeAgentSdkPlugin()); registry.register(new CodexAgentSdkPlugin()); registry.register(new PiAgentPlugin()); + registry.register(new GrokBuildAgentPlugin()); // Plugins can contribute additional backends via `agents: []` in their // DeepsecPlugin export. The shape is validated by AgentRegistry at use. for (const a of getRegistry().agents as AgentPlugin[]) { diff --git a/packages/processor/src/setup-agent.ts b/packages/processor/src/setup-agent.ts index 30f8dd5e..0ef809e8 100644 --- a/packages/processor/src/setup-agent.ts +++ b/packages/processor/src/setup-agent.ts @@ -1,5 +1,6 @@ import { runClaudeSetupTask } from "./agents/claude-agent-sdk.js"; import { runCodexSetupTask } from "./agents/codex-sdk.js"; +import { runGrokSetupTask } from "./agents/grok-build.js"; import { runPiSetupTask } from "./agents/pi-sdk.js"; import type { SetupTaskParams } from "./agents/types.js"; @@ -17,9 +18,12 @@ export async function runSetupTask(params: RunSetupTaskParams): Promise return runClaudeSetupTask(params); case "pi": return runPiSetupTask(params); + case "grok": + case "grok-build": + return runGrokSetupTask(params); default: throw new Error( - `Agent "${params.agentType}" does not support automated setup. Use codex, claude, or pi.`, + `Agent "${params.agentType}" does not support automated setup. Use codex, claude, pi, or grok.`, ); } } From 0f823055e087fe3913b9a1b404f6afc2ab405081 Mon Sep 17 00:00:00 2001 From: ayush-that Date: Wed, 12 Aug 2026 01:45:49 +0530 Subject: [PATCH 2/8] deslop and fix Grok agent setup sandbox and cleanup issues --- .review-grok-agent.md | 33 + docs/models.md | 4 + .../src/__tests__/model-picker.test.ts | 2 +- .../deepsec/src/__tests__/model-route.test.ts | 20 + .../src/auth/ensure-connected-workspace.ts | 9 +- packages/deepsec/src/auth/model-route.ts | 46 +- .../deepsec/src/commands/sandbox-process.ts | 14 + packages/deepsec/src/preflight.ts | 7 +- .../src/__tests__/grok-build.test.ts | 11 +- packages/processor/src/agents/grok-build.ts | 703 +++++++++--------- 10 files changed, 467 insertions(+), 382 deletions(-) create mode 100644 .review-grok-agent.md diff --git a/.review-grok-agent.md b/.review-grok-agent.md new file mode 100644 index 00000000..68ad56d2 --- /dev/null +++ b/.review-grok-agent.md @@ -0,0 +1,33 @@ +# Thermo-nuclear review: Grok Build agent (feat/grok-build-agent) + +Reviewer subagent + deslop pass. Status after fixes. + +## Summary + +Solid local harness peer of Claude/Codex/Pi for process/revalidate. First review found 4 bugs / 3 suggestions / 1 nit. Deslop + fixes landed before merge to fork main. + +## Issues (original → status) + +| # | Severity | Topic | Status | +|---|----------|--------|--------| +| 1 | bug | `init`/`setup` hard-fail: resolveModelRoute always threw for grok | **fixed** — grok resolves synthetic XAI route; OAuth skips HTTP verify | +| 2 | bug | Sandbox path unbrokered for XAI/grok binary | **fixed** — early hard-fail with clear local-only message | +| 3 | bug | Temp GROK_HOME leak without try/finally | **fixed** — investigate/revalidate wrap body in try/finally | +| 4 | bug | Banner JSON used lastIndexOf `{` (breaks nested objects) | **fixed** — brace-match extractJsonObject + unit test | +| 5 | suggestion | Toolless follow-ups still had tools + maxTurns 2 | **fixed** — TOOLLESS=read_file, maxTurns 1 | +| 6 | suggestion | Preflight accepted `which grok` without auth | **fixed** — requires XAI_API_KEY or auth.json | +| 7 | suggestion | Weak tests; model-picker expected pi for xai/grok | **fixed** — harness test + parse/route tests | +| 8 | nit | Shell + XAI key + network under read-only sandbox | **open** (acceptable residual; revalidation needs git shell) | + +## Deslop + +- Removed dead no-op onProgress bridge +- Trimmed AI-style file comments +- Fixed error path that swallowed structured Grok failures +- Cleared SIGKILL timer on process close +- Simplified revalidate rawResponses assembly + +## Tests (green) + +- processor: grok-build (4), registry (5) +- cli: model-route (10), model-picker (6), agent-defaults (2) diff --git a/docs/models.md b/docs/models.md index a9c7c653..8d93be83 100644 --- a/docs/models.md +++ b/docs/models.md @@ -37,6 +37,10 @@ with an isolated `GROK_HOME` (auth mirrored, skills/plugins not loaded), by default (`DEEPSEC_GROK_SANDBOX` overrides; nested sandbox is off when `DEEPSEC_INSIDE_SANDBOX=1`). +**Local only for now.** `deepsec sandbox … --agent grok` exits early: Vercel +Sandbox does not yet install the Grok CLI or broker `XAI_API_KEY` into the +microVM. Use local `process` / `revalidate` / `init` instead. + Interactive one-shot setup recommends five benchmark-backed combinations: GPT-5.6 Sol, Claude Opus 5, Kimi K3, Grok 4.5, and the current DeepSeek entry. Deepsec fetches the latest score, reasoning level, harness, and total run cost diff --git a/packages/deepsec/src/__tests__/model-picker.test.ts b/packages/deepsec/src/__tests__/model-picker.test.ts index 9bcbdaee..c5e3abcf 100644 --- a/packages/deepsec/src/__tests__/model-picker.test.ts +++ b/packages/deepsec/src/__tests__/model-picker.test.ts @@ -69,7 +69,7 @@ describe("DeepSecBench model picker", () => { it("infers provider-prefixed OpenAI and Anthropic slugs before generic Pi slugs", () => { expect(inferModelHarness("openai/gpt-5.6-sol")).toBe("codex"); expect(inferModelHarness("anthropic/claude-opus-5")).toBe("claude"); - expect(inferModelHarness("xai/grok-4.5")).toBe("pi"); + expect(inferModelHarness("xai/grok-4.5")).toBe("grok"); }); it("uses the highest-scoring combo for each recommendation and normalizes price", () => { const choices = buildRecommendedModelChoices(results); diff --git a/packages/deepsec/src/__tests__/model-route.test.ts b/packages/deepsec/src/__tests__/model-route.test.ts index e9f64e35..21985b67 100644 --- a/packages/deepsec/src/__tests__/model-route.test.ts +++ b/packages/deepsec/src/__tests__/model-route.test.ts @@ -2,6 +2,26 @@ import { describe, expect, it, vi } from "vitest"; import { applyResolvedModelRoute, resolveModelRoute } from "../auth/model-route.js"; describe("resolveModelRoute", () => { + it("resolves Grok Build to an XAI route independent of the stored gateway config", async () => { + const resolved = await resolveModelRoute( + { mode: "gateway", provider: "vercel" }, + { agentType: "grok", env: { XAI_API_KEY: "xai-secret" } }, + ); + expect(resolved.route.provider).toBe("xai"); + expect(resolved.environment.XAI_API_KEY).toBe("xai-secret"); + expect(resolved.broker.host).toBe("api.x.ai"); + }); + + it("allows Grok Build without XAI_API_KEY (OAuth / grok login)", async () => { + const resolved = await resolveModelRoute( + { mode: "gateway", provider: "vercel" }, + { agentType: "grok", env: {} }, + ); + expect(resolved.route.provider).toBe("xai"); + expect(resolved.credential).toBe(""); + expect(resolved.environment.XAI_API_KEY).toBeUndefined(); + }); + it("still rejects an explicitly selected route that is incompatible with the harness", async () => { await expect( resolveModelRoute( diff --git a/packages/deepsec/src/auth/ensure-connected-workspace.ts b/packages/deepsec/src/auth/ensure-connected-workspace.ts index c6f0ce22..9845de77 100644 --- a/packages/deepsec/src/auth/ensure-connected-workspace.ts +++ b/packages/deepsec/src/auth/ensure-connected-workspace.ts @@ -1,9 +1,10 @@ import { join } from "node:path"; import { getVercelOidcToken } from "@vercel/oidc"; import { updateEnvFile } from "../env-file.js"; -import { assertSandboxCredential } from "../preflight.js"; +import { assertAgentCredential, assertSandboxCredential } from "../preflight.js"; import { applyResolvedModelRoute, + isGrokAgent, type ModelRoute, type ModelRouteVerifier, type ResolvedModelRoute, @@ -132,6 +133,10 @@ export async function ensureConnectedWorkspace( assertSandboxCredential({ env }); const resolvedRoutes: ResolvedModelRoute[] = []; for (const agentType of options.agentTypes) { + // Grok Build uses XAI_API_KEY / grok login; verify that path before route resolve. + if (isGrokAgent(agentType)) { + assertAgentCredential(agentType); + } const resolved = await (deps.resolveRoute ?? resolveModelRoute)(options.modelRoute, { agentType, env, @@ -152,6 +157,8 @@ export async function ensureConnectedWorkspace( ); if (!reuseModel) { for (const resolved of resolvedRoutes) { + // Skip HTTP probe when Grok auth is OAuth-only (no API key). + if (resolved.route.provider === "xai" && !resolved.credential) continue; await (deps.verifyModelRoute ?? verifyModelRouteWithFetch)(resolved); } } diff --git a/packages/deepsec/src/auth/model-route.ts b/packages/deepsec/src/auth/model-route.ts index cfcdb0b2..defb2fb5 100644 --- a/packages/deepsec/src/auth/model-route.ts +++ b/packages/deepsec/src/auth/model-route.ts @@ -66,16 +66,17 @@ function checkedUrl(value: string, label: string): URL { return url; } +export function isGrokAgent(agentType: string): boolean { + return agentType === "grok" || agentType === "grok-build"; +} + export function modelRouteCompatibilityError( route: ModelRoute, agentType: string, ): string | undefined { - // Grok Build authenticates via XAI_API_KEY / `grok login`, not AI Gateway - // or the OpenAI/Anthropic direct routes. Leave env alone and let preflight - // check the Grok CLI credentials instead. - if (agentType === "grok" || agentType === "grok-build") { - return `Grok Build does not use the configured model route (uses XAI_API_KEY / grok login)`; - } + // Grok resolves its own XAI route inside resolveModelRoute; any stored + // gateway/custom config is ignored for this harness (not an error). + if (isGrokAgent(agentType)) return undefined; if (route.mode === "custom" && agentType !== "pi") { return `Custom model routes require --agent pi (received ${agentType})`; } @@ -88,6 +89,31 @@ export function modelRouteCompatibilityError( return undefined; } +/** Synthetic route for Grok Build (XAI_API_KEY or local grok login). */ +function resolveGrokModelRoute(env: NodeJS.ProcessEnv): ResolvedModelRoute { + const credentialEnv = "XAI_API_KEY"; + const credential = env[credentialEnv] ?? ""; + return { + route: { + mode: "direct", + provider: "xai", + apiKeyEnv: credentialEnv, + baseUrl: "https://api.x.ai/v1", + }, + credentialEnv, + credential, + environment: credential ? { XAI_API_KEY: credential } : {}, + broker: { + host: "api.x.ai", + placeholderEnv: credentialEnv, + header: { + name: "authorization", + value: credential ? `Bearer ${credential}` : "", + }, + }, + }; +} + function assertCompatible(route: ModelRoute, agentType: string): void { const error = modelRouteCompatibilityError(route, agentType); if (error) throw new Error(error); @@ -106,6 +132,11 @@ export async function resolveModelRoute( options: ResolveModelRouteOptions, ): Promise { const env = options.env ?? process.env; + // Grok Build authenticates via XAI_API_KEY or `grok login`, independent of + // the selected gateway/direct/custom route stored for other harnesses. + if (isGrokAgent(options.agentType)) { + return resolveGrokModelRoute(env); + } assertCompatible(route, options.agentType); if (route.mode === "gateway") { @@ -252,6 +283,9 @@ export async function verifyModelRouteWithFetch( route: ResolvedModelRoute, fetchImpl: typeof fetch = fetch, ): Promise { + // Grok OAuth (`grok login`) has no API key to probe; skip HTTP verification. + if (route.route.provider === "xai" && !route.credential) return; + const endpoint = modelsEndpoint(route); const headers: Record = { [route.broker.header.name]: route.broker.header.value, diff --git a/packages/deepsec/src/commands/sandbox-process.ts b/packages/deepsec/src/commands/sandbox-process.ts index 44ae962d..bcb0f3ab 100644 --- a/packages/deepsec/src/commands/sandbox-process.ts +++ b/packages/deepsec/src/commands/sandbox-process.ts @@ -148,6 +148,20 @@ export async function sandboxCommand(subcommand: string, opts: SandboxOpts) { const projectId = resolveProjectId(opts.projectId); const config = buildConfig(subcommand as SandboxSubcommand, projectId, opts); + // Grok Build is local-only until sandbox brokering installs the CLI and + // injects XAI credentials into the microVM network policy. + if (config.agentType === "grok" || config.agentType === "grok-build") { + console.error( + `Sandbox mode does not support --agent grok yet.\n` + + `\n` + + ` Use a local process instead:\n` + + ` pnpm deepsec process --project-id ${projectId} --agent grok\n` + + `\n` + + ` Or pick a sandbox-capable harness: --agent codex | claude | pi`, + ); + process.exit(1); + } + if (!config.aiApiKeyEnv && !config.aiBaseUrl) { const resolved = await applyConfiguredModelRoute(config.agentType ?? "codex"); config.brokeredModelCredential = resolved?.broker; diff --git a/packages/deepsec/src/preflight.ts b/packages/deepsec/src/preflight.ts index a2de4517..5b653c24 100644 --- a/packages/deepsec/src/preflight.ts +++ b/packages/deepsec/src/preflight.ts @@ -188,8 +188,8 @@ function hasLocalPiAgent(): boolean { } /** - * Grok Build: either XAI_API_KEY is set, or the user has run `grok login` - * and we can see auth.json under GROK_HOME / ~/.grok. + * Grok Build: XAI_API_KEY, or a real auth.json from `grok login`. + * `which grok` alone is not enough (logged-out CLI would pass preflight). */ function hasLocalGrokAgent(): boolean { if (process.env.XAI_API_KEY) return true; @@ -199,8 +199,7 @@ function hasLocalGrokAgent(): boolean { for (const home of homes) { if (existsSync(join(home, "auth.json"))) return true; } - // Binary on PATH is a soft signal; the CLI errors clearly if not logged in. - return whichSync("grok"); + return false; } function isGrok(agentType: string | undefined): boolean { diff --git a/packages/processor/src/__tests__/grok-build.test.ts b/packages/processor/src/__tests__/grok-build.test.ts index 6ed89d08..76595173 100644 --- a/packages/processor/src/__tests__/grok-build.test.ts +++ b/packages/processor/src/__tests__/grok-build.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { buildGrokEnv, makeIsolatedGrokHome } from "../agents/grok-build.js"; +import { buildGrokEnv, makeIsolatedGrokHome, parseGrokStdout } from "../agents/grok-build.js"; import { createDefaultAgentRegistry } from "../index.js"; const homes: string[] = []; @@ -47,6 +47,15 @@ describe("Grok Build agent", () => { } }); + it("parseGrokStdout recovers a banner + nested JSON object", () => { + const raw = parseGrokStdout( + 'note: starting\n{"text":"{\\"ok\\":true}","usage":{"input_tokens":12},"sessionId":"abc","num_turns":1}\n', + ); + expect(raw.text).toBe('{"ok":true}'); + expect(raw.sessionId).toBe("abc"); + expect(raw.usage?.input_tokens).toBe(12); + }); + it("makeIsolatedGrokHome creates a config and mirrors auth when available", () => { // Seed a fake user auth so we exercise the mirror path even if the // developer machine has no real grok login in this environment. diff --git a/packages/processor/src/agents/grok-build.ts b/packages/processor/src/agents/grok-build.ts index 673fe34b..4c5187d3 100644 --- a/packages/processor/src/agents/grok-build.ts +++ b/packages/processor/src/agents/grok-build.ts @@ -40,25 +40,18 @@ import type { } from "./types.js"; /** - * Grok Build (xAI) coding-agent backend. - * - * Spawns the local `grok` CLI in headless mode (`-p`) with a restricted - * tool allowlist and optional OS sandbox. Same prompt + JSON schema as the - * other backends; investigation output is parsed by the shared helpers. - * - * Auth: - * - `XAI_API_KEY` in the environment, or - * - an existing `~/.grok/auth.json` from `grok login` (mirrored into an - * isolated per-run GROK_HOME so user skills/plugins are not loaded). + * Grok Build coding-agent backend: headless `grok -p` with a restricted + * tool allowlist. Auth is XAI_API_KEY or mirrored `~/.grok/auth.json`. */ const DEFAULT_MODEL = "grok-4.5"; const DEFAULT_THINKING_LEVEL = "xhigh"; -/** Read-only tools for investigation / revalidation (internal Grok tool ids). */ +// Internal Grok tool ids (not Claude's Read/Grep names). const INVESTIGATE_TOOLS = "read_file,grep,list_dir,run_terminal_cmd"; -/** Even tighter set for setup analysis. */ const SETUP_TOOLS = "read_file,grep,list_dir"; +// Follow-ups (JSON repair / refusal) should not re-open the tool loop. +const TOOLLESS = "read_file"; const GROK_ENV_ALLOWLIST = new Set([ "PATH", @@ -120,13 +113,11 @@ interface GrokRunOptions { maxTurns: number; tools: string; thinkingLevel: string; - /** Resume an existing session (JSON repair / refusal follow-up). */ resumeSessionId?: string; signal?: AbortSignal; onProgress?: (progress: AgentProgress) => void; - /** Isolated GROK_HOME for this batch; created if omitted. */ grokHome?: string; - /** Keep the home dir after the run (needed when resuming). */ + /** Keep GROK_HOME after the run so --resume can find the session. */ keepHome?: boolean; } @@ -138,7 +129,6 @@ function resolveThinkingLevel(config: Record): string { function resolveGrokBinary(): string { if (process.env.GROK_EXECUTABLE) return process.env.GROK_EXECUTABLE; - // Common install locations before falling back to PATH. const candidates = [ path.join(os.homedir(), ".local", "bin", "grok"), path.join(os.homedir(), ".grok", "bin", "grok"), @@ -157,33 +147,20 @@ function resolveGrokBinary(): string { } /** - * Build a minimal GROK_HOME so deepsec does not inherit the operator's - * 400+ skills, MCP servers, and plugins (those bloat the system prompt - * and burn tokens on every batch). - * - * Mirrors `auth.json` when present so OAuth login works without - * XAI_API_KEY. Prefer symlink so token refresh writes back to the - * real home; copy as fallback. + * Minimal GROK_HOME without the operator's skills/MCP plugins (those + * inflate the system prompt). Mirrors auth.json when present. */ export function makeIsolatedGrokHome(): string { const home = fs.mkdtempSync(path.join(os.tmpdir(), "deepsec-grok-home-")); fs.writeFileSync( path.join(home, "config.toml"), - [ - "[ui]", - 'permission_mode = "dontAsk"', - "", - "[cli]", - "auto_update = false", - "", - ].join("\n"), + ['[ui]', 'permission_mode = "dontAsk"', "", "[cli]", "auto_update = false", ""].join("\n"), { mode: 0o600 }, ); - const userHomes = [ - process.env.GROK_HOME, - path.join(os.homedir(), ".grok"), - ].filter((p): p is string => typeof p === "string" && p.length > 0); + const userHomes = [process.env.GROK_HOME, path.join(os.homedir(), ".grok")].filter( + (p): p is string => typeof p === "string" && p.length > 0, + ); for (const userHome of userHomes) { const auth = path.join(userHome, "auth.json"); @@ -212,8 +189,6 @@ export function buildGrokEnv(grokHome: string): Record { } env.GROK_HOME = grokHome; env.GROK_DISABLE_AUTOUPDATER = "1"; - // Forward only the credential the CLI actually needs. Never ship - // GITHUB_TOKEN / AWS_* / etc. into a prompt-injectionable shell. for (const k of ["XAI_API_KEY", "XAI_API_BASE_URL"]) { const v = process.env[k]; if (typeof v === "string") env[k] = v; @@ -222,22 +197,51 @@ export function buildGrokEnv(grokHome: string): Record { } function sandboxProfile(): string { - // Nested OS sandbox inside a Vercel Sandbox microVM is unnecessary. if (process.env.DEEPSEC_INSIDE_SANDBOX === "1") return "off"; - // Read-only project FS; agent can still write session state under GROK_HOME. return process.env.DEEPSEC_GROK_SANDBOX ?? "read-only"; } -function parseGrokStdout(stdout: string): GrokJsonResult { +function extractJsonObject(text: string): string | undefined { + const start = text.indexOf("{"); + if (start < 0) return undefined; + let depth = 0; + let inString = false; + let escape = false; + for (let i = start; i < text.length; i++) { + const ch = text[i]; + if (inString) { + if (escape) { + escape = false; + } else if (ch === "\\") { + escape = true; + } else if (ch === '"') { + inString = false; + } + continue; + } + if (ch === '"') { + inString = true; + continue; + } + if (ch === "{") depth++; + else if (ch === "}") { + depth--; + if (depth === 0) return text.slice(start, i + 1); + } + } + return undefined; +} + +/** Exported for tests. */ +export function parseGrokStdout(stdout: string): GrokJsonResult { const trimmed = stdout.trim(); if (!trimmed) throw new Error("Grok produced empty stdout"); - // Prefer the last complete JSON object (in case any banner leaked). try { return JSON.parse(trimmed) as GrokJsonResult; } catch { - const start = trimmed.lastIndexOf("{"); - if (start < 0) throw new Error(`Grok stdout was not JSON: ${trimmed.slice(0, 200)}`); - return JSON.parse(trimmed.slice(start)) as GrokJsonResult; + const slice = extractJsonObject(trimmed); + if (!slice) throw new Error(`Grok stdout was not JSON: ${trimmed.slice(0, 200)}`); + return JSON.parse(slice) as GrokJsonResult; } } @@ -260,13 +264,7 @@ function metaFromGrokJson(raw: GrokJsonResult): Partial { return meta; } -/** - * Run one headless Grok prompt. Uses `--output-format json` so the final - * result is a single parseable object with text + spend metadata. - * Progress is coarse (started / complete) because the json format only - * emits at the end; streaming-messages-json is available later if we - * need tool-level progress. - */ +/** One headless `grok -p` turn. Returns final JSON text + spend meta. */ export async function runGrokHeadless(opts: GrokRunOptions): Promise { const bin = resolveGrokBinary(); const grokHome = opts.grokHome ?? makeIsolatedGrokHome(); @@ -296,7 +294,6 @@ export async function runGrokHeadless(opts: GrokRunOptions): Promise 80_000) { promptFile = path.join( @@ -314,7 +310,6 @@ export async function runGrokHeadless(opts: GrokRunOptions): Promise= 0) { args.splice(pIdx, 2, "--prompt-file", promptFile); @@ -339,20 +334,21 @@ export async function runGrokHeadless(opts: GrokRunOptions): Promise | undefined; child.stdout?.setEncoding("utf8"); child.stderr?.setEncoding("utf8"); child.stdout?.on("data", (chunk: string) => { @@ -425,33 +418,39 @@ function spawnCollect(params: { const onAbort = () => { child.kill("SIGTERM"); - // Escalate if the CLI ignores SIGTERM. - setTimeout(() => { + killTimer = setTimeout(() => { try { child.kill("SIGKILL"); } catch { // ignore } - }, 2_000).unref?.(); + }, 2_000); + killTimer.unref?.(); }; params.signal?.addEventListener("abort", onAbort, { once: true }); - child.on("error", (err) => { + const finish = (fn: () => void) => { + if (killTimer) clearTimeout(killTimer); params.signal?.removeEventListener("abort", onAbort); - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - reject( - new Error( - `Grok Build CLI not found (${params.bin}). Install Grok Build and ensure \`grok\` is on PATH, or set GROK_EXECUTABLE.`, - ), - ); - return; - } - reject(err); + fn(); + }; + + child.on("error", (err) => { + finish(() => { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + reject( + new Error( + `Grok Build CLI not found (${params.bin}). Install Grok Build and ensure \`grok\` is on PATH, or set GROK_EXECUTABLE.`, + ), + ); + return; + } + reject(err); + }); }); child.on("close", (code) => { - params.signal?.removeEventListener("abort", onAbort); - resolve({ stdout, stderr, code }); + finish(() => resolve({ stdout, stderr, code })); }); }); } @@ -472,7 +471,7 @@ async function runToollessFollowUp(params: { projectRoot: params.projectRoot, model: params.model, maxTurns: 1, - tools: "", + tools: TOOLLESS, thinkingLevel: "low", resumeSessionId: params.sessionId, signal: params.signal, @@ -530,184 +529,166 @@ export class GrokBuildAgentPlugin implements AgentPlugin { message: `Investigating ${batch.length} file(s) with Grok Build (${model})`, }; - for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { - if (attempt > 1) { - yield { - type: "thinking", - message: `Retrying Grok batch after transient error (attempt ${attempt}/${MAX_ATTEMPTS}): ${lastError.slice(0, 200)}`, - }; - resultText = ""; - lastError = ""; - sdkMeta = {}; - sessionId = undefined; - if (grokHome) { - try { - fs.rmSync(grokHome, { recursive: true, force: true }); - } catch { - // ignore - } + try { + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + if (attempt > 1) { + yield { + type: "thinking", + message: `Retrying Grok batch after transient error (attempt ${attempt}/${MAX_ATTEMPTS}): ${lastError.slice(0, 200)}`, + }; + resultText = ""; + lastError = ""; + sdkMeta = {}; + sessionId = undefined; + cleanupHome(grokHome); grokHome = undefined; } + + try { + grokHome = makeIsolatedGrokHome(); + const run = await runGrokHeadless({ + prompt, + projectRoot, + model, + maxTurns, + tools: INVESTIGATE_TOOLS, + thinkingLevel, + signal, + grokHome, + keepHome: true, + }); + resultText = run.resultText; + sdkMeta = run.meta; + sessionId = run.raw.sessionId; + turnCount = run.raw.num_turns ?? 0; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + if (err instanceof QuotaExhaustedError) throw err; + yield { type: "error", message: `Grok error: ${lastError.slice(0, 300)}` }; + } + + if (resultText) break; + const quotaSource = classifyQuotaError(lastError); + if (quotaSource) throw new QuotaExhaustedError(quotaSource, lastError); + if (attempt >= MAX_ATTEMPTS || !isTransientError(lastError)) break; + await backoff(attempt); } + if (!resultText) { + throw new Error( + `Grok Build produced no investigation result after ${MAX_ATTEMPTS} attempt(s). ` + + `Last error: ${lastError || "(none captured)"}.`, + ); + } + + const durationMs = Date.now() - startTime; + let parsed: ParsedInvestigateResults; try { - grokHome = makeIsolatedGrokHome(); - const run = await runGrokHeadless({ - prompt, + parsed = parseInvestigateResults(resultText, batch); + } catch (err) { + yield { + type: "thinking", + message: "Grok returned non-JSON investigation output; requesting JSON-only repair", + }; + const repairText = await runToollessFollowUp({ + sessionId, + grokHome: grokHome!, projectRoot, model, - maxTurns, - tools: INVESTIGATE_TOOLS, thinkingLevel, + prompt: buildInvestigateJsonRepairPrompt(batch), signal, - grokHome, - keepHome: true, - onProgress: (p) => { - // Coarse progress only from the headless json path. - if (p.type === "tool_use" || p.type === "thinking") { - // no-op bridge; reserved for streaming mode - } - }, }); - resultText = run.resultText; - sdkMeta = run.meta; - sessionId = run.raw.sessionId; - turnCount = run.raw.num_turns ?? 0; - } catch (err) { - lastError = err instanceof Error ? err.message : String(err); - if (err instanceof QuotaExhaustedError) { - cleanupHome(grokHome); + if (repairText === undefined) { + writeParseFailureDebug({ + projectId, + phase: "investigate", + agentType: this.type, + resultText, + error: err, + batch, + }); throw err; } - yield { type: "error", message: `Grok error: ${lastError.slice(0, 300)}` }; + try { + parsed = parseInvestigateResults(repairText, batch); + resultText = repairText; + yield { type: "thinking", message: "Grok JSON repair succeeded" }; + } catch (repairErr) { + const combinedError = jsonRepairFailureError(err, repairErr); + writeParseFailureDebug({ + projectId, + phase: "investigate", + agentType: this.type, + resultText: formatJsonRepairFailureDebugText(resultText, repairText), + error: combinedError, + batch, + }); + throw combinedError; + } } - if (resultText) break; - const quotaSource = classifyQuotaError(lastError); - if (quotaSource) { - cleanupHome(grokHome); - throw new QuotaExhaustedError(quotaSource, lastError); + let results: InvestigateResult[] = parsed.results; + if (parsed.invalid.length > 0) { + const fieldRepair = yield* runInvestigateFieldRepairLoop({ + results, + invalid: parsed.invalid, + batch, + followUp: (p) => + runToollessFollowUp({ + sessionId, + grokHome: grokHome!, + projectRoot, + model, + thinkingLevel, + prompt: p, + signal, + }), + agentLabel: "Grok", + agentType: this.type, + projectId, + }); + results = fieldRepair.results; } - if (attempt >= MAX_ATTEMPTS || !isTransientError(lastError)) break; - await backoff(attempt); - } - if (!resultText) { - cleanupHome(grokHome); - throw new Error( - `Grok Build produced no investigation result after ${MAX_ATTEMPTS} attempt(s). ` + - `Last error: ${lastError || "(none captured)"}.`, - ); - } - - const durationMs = Date.now() - startTime; - let parsed: ParsedInvestigateResults; - try { - parsed = parseInvestigateResults(resultText, batch); - } catch (err) { - yield { - type: "thinking", - message: "Grok returned non-JSON investigation output; requesting JSON-only repair", - }; - const repairText = await runToollessFollowUp({ + let refusal: RefusalReport | undefined; + const refusalRaw = await runToollessFollowUp({ sessionId, grokHome: grokHome!, projectRoot, model, thinkingLevel, - prompt: buildInvestigateJsonRepairPrompt(batch), + prompt: REFUSAL_FOLLOWUP_PROMPT, signal, }); - if (repairText === undefined) { - writeParseFailureDebug({ - projectId, - phase: "investigate", - agentType: this.type, - resultText, - error: err, - batch, - }); - cleanupHome(grokHome); - throw err; - } - try { - parsed = parseInvestigateResults(repairText, batch); - resultText = repairText; - yield { type: "thinking", message: "Grok JSON repair succeeded" }; - } catch (repairErr) { - const combinedError = jsonRepairFailureError(err, repairErr); - writeParseFailureDebug({ - projectId, - phase: "investigate", - agentType: this.type, - resultText: formatJsonRepairFailureDebugText(resultText, repairText), - error: combinedError, - batch, - }); - cleanupHome(grokHome); - throw combinedError; + if (refusalRaw) refusal = parseRefusalReport(refusalRaw); + if (refusal?.refused) { + yield { + type: "thinking", + message: `Refusal detected: ${refusal.reason ?? "see raw"}`, + }; } - } - - let results: InvestigateResult[] = parsed.results; - if (parsed.invalid.length > 0) { - const fieldRepair = yield* runInvestigateFieldRepairLoop({ - results, - invalid: parsed.invalid, - batch, - followUp: (p) => - runToollessFollowUp({ - sessionId, - grokHome: grokHome!, - projectRoot, - model, - thinkingLevel, - prompt: p, - signal, - }), - agentLabel: "Grok", - agentType: this.type, - projectId, - }); - results = fieldRepair.results; - } - let refusal: RefusalReport | undefined; - const refusalRaw = await runToollessFollowUp({ - sessionId, - grokHome: grokHome!, - projectRoot, - model, - thinkingLevel, - prompt: REFUSAL_FOLLOWUP_PROMPT, - signal, - }); - if (refusalRaw) refusal = parseRefusalReport(refusalRaw); - if (refusal?.refused) { + const costStr = sdkMeta.costUsd != null ? ` $${sdkMeta.costUsd.toFixed(3)}` : ""; + const tokensStr = sdkMeta.usage + ? ` ${sdkMeta.usage.inputTokens + sdkMeta.usage.outputTokens} tokens` + : ""; yield { - type: "thinking", - message: `Refusal detected: ${refusal.reason ?? "see raw"}`, + type: "complete", + message: `Investigation complete (${(durationMs / 1000).toFixed(1)}s, ${turnCount} turns${costStr}${tokensStr}${refusal?.refused ? " refusal" : ""})`, }; - } - - const costStr = sdkMeta.costUsd != null ? ` $${sdkMeta.costUsd.toFixed(3)}` : ""; - const tokensStr = sdkMeta.usage - ? ` ${sdkMeta.usage.inputTokens + sdkMeta.usage.outputTokens} tokens` - : ""; - yield { - type: "complete", - message: `Investigation complete (${(durationMs / 1000).toFixed(1)}s, ${turnCount} turns${costStr}${tokensStr}${refusal?.refused ? " refusal" : ""})`, - }; - cleanupHome(grokHome); - return { - results, - meta: { - durationMs, - ...sdkMeta, - refusal, - }, - }; + return { + results, + meta: { + durationMs, + ...sdkMeta, + refusal, + }, + }; + } finally { + cleanupHome(grokHome); + } } async *revalidate(params: RevalidateParams): AsyncGenerator { @@ -746,157 +727,141 @@ export class GrokBuildAgentPlugin implements AgentPlugin { let grokHome: string | undefined; const rawResponses: RevalidateRawResponse[] = []; - for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { - if (attempt > 1) { - yield { - type: "thinking", - message: `Retrying Grok revalidation after transient error (attempt ${attempt}/${MAX_ATTEMPTS}): ${lastError.slice(0, 200)}`, - }; - resultText = ""; - lastError = ""; - sdkMeta = {}; - sessionId = undefined; - cleanupHome(grokHome); - grokHome = undefined; + try { + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + if (attempt > 1) { + yield { + type: "thinking", + message: `Retrying Grok revalidation after transient error (attempt ${attempt}/${MAX_ATTEMPTS}): ${lastError.slice(0, 200)}`, + }; + resultText = ""; + lastError = ""; + sdkMeta = {}; + sessionId = undefined; + cleanupHome(grokHome); + grokHome = undefined; + } + + try { + grokHome = makeIsolatedGrokHome(); + const run = await runGrokHeadless({ + prompt, + projectRoot, + model, + maxTurns, + tools: INVESTIGATE_TOOLS, + thinkingLevel, + signal, + grokHome, + keepHome: true, + }); + resultText = run.resultText; + sdkMeta = run.meta; + sessionId = run.raw.sessionId; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + if (err instanceof QuotaExhaustedError) throw err; + yield { type: "error", message: `Grok error: ${lastError.slice(0, 300)}` }; + } + + if (resultText) break; + const quotaSource = classifyQuotaError(lastError); + if (quotaSource) throw new QuotaExhaustedError(quotaSource, lastError); + if (attempt >= MAX_ATTEMPTS || !isTransientError(lastError)) break; + await backoff(attempt); + } + + if (!resultText) { + throw new Error( + `Grok Build produced no revalidation result after ${MAX_ATTEMPTS} attempt(s). ` + + `Last error: ${lastError || "(none captured)"}.`, + ); } + let verdicts: RevalidateVerdict[]; try { - grokHome = makeIsolatedGrokHome(); - const run = await runGrokHeadless({ - prompt, + verdicts = parseRevalidateVerdicts(resultText); + } catch (err) { + yield { + type: "thinking", + message: "Grok returned non-JSON revalidation output; requesting JSON-only repair", + }; + const repairPrompt = buildRevalidateJsonRepairPrompt(expected); + const repairText = await runToollessFollowUp({ + sessionId, + grokHome: grokHome!, projectRoot, model, - maxTurns, - tools: INVESTIGATE_TOOLS, thinkingLevel, + prompt: repairPrompt, signal, - grokHome, - keepHome: true, }); - resultText = run.resultText; - sdkMeta = run.meta; - sessionId = run.raw.sessionId; - rawResponses.push({ kind: "initial", rawText: resultText }); - } catch (err) { - lastError = err instanceof Error ? err.message : String(err); - if (err instanceof QuotaExhaustedError) { - cleanupHome(grokHome); + if (repairText === undefined) { + writeParseFailureDebug({ + projectId, + phase: "revalidate", + agentType: this.type, + resultText, + error: err, + batch, + }); throw err; } - yield { type: "error", message: `Grok error: ${lastError.slice(0, 300)}` }; - } - - if (resultText) break; - const quotaSource = classifyQuotaError(lastError); - if (quotaSource) { - cleanupHome(grokHome); - throw new QuotaExhaustedError(quotaSource, lastError); + rawResponses.push({ kind: "json-repair", prompt: repairPrompt, rawText: repairText }); + try { + verdicts = parseRevalidateVerdicts(repairText); + resultText = repairText; + yield { type: "thinking", message: "Grok revalidation JSON repair succeeded" }; + } catch (repairErr) { + const combinedError = jsonRepairFailureError(err, repairErr); + writeParseFailureDebug({ + projectId, + phase: "revalidate", + agentType: this.type, + resultText: formatJsonRepairFailureDebugText(resultText, repairText), + error: combinedError, + batch, + }); + throw combinedError; + } } - if (attempt >= MAX_ATTEMPTS || !isTransientError(lastError)) break; - await backoff(attempt); - } - if (!resultText) { - cleanupHome(grokHome); - throw new Error( - `Grok Build produced no revalidation result after ${MAX_ATTEMPTS} attempt(s). ` + - `Last error: ${lastError || "(none captured)"}.`, - ); - } + const idRepair = yield* runRevalidateIdRepairLoop({ + expected, + verdicts, + initialRawText: resultText, + followUp: async (p) => + runToollessFollowUp({ + sessionId, + grokHome: grokHome!, + projectRoot, + model, + thinkingLevel, + prompt: p, + signal, + }), + agentLabel: "Grok", + }); - let verdicts: RevalidateVerdict[]; - try { - verdicts = parseRevalidateVerdicts(resultText); - } catch (err) { + const durationMs = Date.now() - startTime; + const costStr = sdkMeta.costUsd != null ? ` $${sdkMeta.costUsd.toFixed(3)}` : ""; yield { - type: "thinking", - message: "Grok returned non-JSON revalidation output; requesting JSON-only repair", + type: "complete", + message: `Revalidation complete (${(durationMs / 1000).toFixed(1)}s, ${idRepair.verdicts.length} verdicts${costStr})`, }; - const repairPrompt = buildRevalidateJsonRepairPrompt(expected); - const repairText = await runToollessFollowUp({ - sessionId, - grokHome: grokHome!, - projectRoot, - model, - thinkingLevel, - prompt: repairPrompt, - signal, - }); - if (repairText === undefined) { - writeParseFailureDebug({ - projectId, - phase: "revalidate", - agentType: this.type, - resultText, - error: err, - batch, - }); - cleanupHome(grokHome); - throw err; - } - rawResponses.push({ kind: "json-repair", prompt: repairPrompt, rawText: repairText }); - try { - verdicts = parseRevalidateVerdicts(repairText); - resultText = repairText; - yield { type: "thinking", message: "Grok revalidation JSON repair succeeded" }; - } catch (repairErr) { - const combinedError = jsonRepairFailureError(err, repairErr); - writeParseFailureDebug({ - projectId, - phase: "revalidate", - agentType: this.type, - resultText: formatJsonRepairFailureDebugText(resultText, repairText), - error: combinedError, - batch, - }); - cleanupHome(grokHome); - throw combinedError; - } - } - - const idRepair = yield* runRevalidateIdRepairLoop({ - expected, - verdicts, - initialRawText: resultText, - followUp: async (p) => - runToollessFollowUp({ - sessionId, - grokHome: grokHome!, - projectRoot, - model, - thinkingLevel, - prompt: p, - signal, - }), - agentLabel: "Grok", - }); - verdicts = idRepair.verdicts; - // Prefer the repair loop's complete transcript (includes initial). - const finalRawResponses = - idRepair.rawResponses.length > 0 - ? [ - ...rawResponses.filter((r) => r.kind !== "initial"), - ...idRepair.rawResponses, - ] - : rawResponses; - - const durationMs = Date.now() - startTime; - const costStr = sdkMeta.costUsd != null ? ` $${sdkMeta.costUsd.toFixed(3)}` : ""; - yield { - type: "complete", - message: `Revalidation complete (${(durationMs / 1000).toFixed(1)}s, ${verdicts.length} verdicts${costStr})`, - }; - cleanupHome(grokHome); - return { - verdicts, - meta: { - durationMs, - ...sdkMeta, - }, - rawResponses: finalRawResponses, - repairAttempts: idRepair.repairAttempts, - }; + return { + verdicts: idRepair.verdicts, + meta: { + durationMs, + ...sdkMeta, + }, + rawResponses: [...rawResponses, ...idRepair.rawResponses], + repairAttempts: idRepair.repairAttempts, + }; + } finally { + cleanupHome(grokHome); + } } } From 9b0d77cdae2a93eca035cf4beb9cd46a48ed4606 Mon Sep 17 00:00:00 2001 From: ayush-that Date: Wed, 12 Aug 2026 02:27:39 +0530 Subject: [PATCH 3/8] remove local review notes file from the repo --- .gitignore | 3 +++ .review-grok-agent.md | 33 --------------------------------- 2 files changed, 3 insertions(+), 33 deletions(-) delete mode 100644 .review-grok-agent.md diff --git a/.gitignore b/.gitignore index 96d46dbb..96339d85 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ linear-* plans/ .pnpm-store/ .env* + +# local review notes +.review-grok-agent.md diff --git a/.review-grok-agent.md b/.review-grok-agent.md deleted file mode 100644 index 68ad56d2..00000000 --- a/.review-grok-agent.md +++ /dev/null @@ -1,33 +0,0 @@ -# Thermo-nuclear review: Grok Build agent (feat/grok-build-agent) - -Reviewer subagent + deslop pass. Status after fixes. - -## Summary - -Solid local harness peer of Claude/Codex/Pi for process/revalidate. First review found 4 bugs / 3 suggestions / 1 nit. Deslop + fixes landed before merge to fork main. - -## Issues (original → status) - -| # | Severity | Topic | Status | -|---|----------|--------|--------| -| 1 | bug | `init`/`setup` hard-fail: resolveModelRoute always threw for grok | **fixed** — grok resolves synthetic XAI route; OAuth skips HTTP verify | -| 2 | bug | Sandbox path unbrokered for XAI/grok binary | **fixed** — early hard-fail with clear local-only message | -| 3 | bug | Temp GROK_HOME leak without try/finally | **fixed** — investigate/revalidate wrap body in try/finally | -| 4 | bug | Banner JSON used lastIndexOf `{` (breaks nested objects) | **fixed** — brace-match extractJsonObject + unit test | -| 5 | suggestion | Toolless follow-ups still had tools + maxTurns 2 | **fixed** — TOOLLESS=read_file, maxTurns 1 | -| 6 | suggestion | Preflight accepted `which grok` without auth | **fixed** — requires XAI_API_KEY or auth.json | -| 7 | suggestion | Weak tests; model-picker expected pi for xai/grok | **fixed** — harness test + parse/route tests | -| 8 | nit | Shell + XAI key + network under read-only sandbox | **open** (acceptable residual; revalidation needs git shell) | - -## Deslop - -- Removed dead no-op onProgress bridge -- Trimmed AI-style file comments -- Fixed error path that swallowed structured Grok failures -- Cleared SIGKILL timer on process close -- Simplified revalidate rawResponses assembly - -## Tests (green) - -- processor: grok-build (4), registry (5) -- cli: model-route (10), model-picker (6), agent-defaults (2) From 6ab2ab47f82f3657e90353c0e74c34842aeb3143 Mon Sep 17 00:00:00 2001 From: ayush-that Date: Wed, 12 Aug 2026 02:29:50 +0530 Subject: [PATCH 4/8] strip excess comments from Grok Build agent wiring --- .../src/auth/ensure-connected-workspace.ts | 2 -- packages/deepsec/src/auth/model-picker.ts | 6 ------ packages/deepsec/src/auth/model-route.ts | 6 ------ .../deepsec/src/commands/sandbox-process.ts | 2 -- packages/deepsec/src/preflight.ts | 4 ---- packages/deepsec/src/resolve-agent-type.ts | 1 - .../processor/src/__tests__/grok-build.test.ts | 3 --- packages/processor/src/agents/grok-build.ts | 18 +----------------- 8 files changed, 1 insertion(+), 41 deletions(-) diff --git a/packages/deepsec/src/auth/ensure-connected-workspace.ts b/packages/deepsec/src/auth/ensure-connected-workspace.ts index 9845de77..08c95cae 100644 --- a/packages/deepsec/src/auth/ensure-connected-workspace.ts +++ b/packages/deepsec/src/auth/ensure-connected-workspace.ts @@ -133,7 +133,6 @@ export async function ensureConnectedWorkspace( assertSandboxCredential({ env }); const resolvedRoutes: ResolvedModelRoute[] = []; for (const agentType of options.agentTypes) { - // Grok Build uses XAI_API_KEY / grok login; verify that path before route resolve. if (isGrokAgent(agentType)) { assertAgentCredential(agentType); } @@ -157,7 +156,6 @@ export async function ensureConnectedWorkspace( ); if (!reuseModel) { for (const resolved of resolvedRoutes) { - // Skip HTTP probe when Grok auth is OAuth-only (no API key). if (resolved.route.provider === "xai" && !resolved.credential) continue; await (deps.verifyModelRoute ?? verifyModelRouteWithFetch)(resolved); } diff --git a/packages/deepsec/src/auth/model-picker.ts b/packages/deepsec/src/auth/model-picker.ts index e7eed5f7..18c38d42 100644 --- a/packages/deepsec/src/auth/model-picker.ts +++ b/packages/deepsec/src/auth/model-picker.ts @@ -139,7 +139,6 @@ function strongest(results: BenchmarkResult[], modelId: string): BenchmarkResult } function configuredModel(result: BenchmarkResult): string { - // Pi needs the provider/model gateway id. Grok Build takes the bare model id. if (result.harness === "pi") return result.modelId; if (result.harness === "grok") { return result.modelId.startsWith("xai/") @@ -177,8 +176,6 @@ function canonicalHarness(value: string | undefined): ModelHarness | undefined { } function compatibleHarness(route: ModelRoute, requested?: string): ModelHarness | undefined { - // Grok Build authenticates via XAI_API_KEY / grok login, independent of - // the Vercel AI Gateway model route. Prefer an explicit grok request. const requestedHarness = canonicalHarness(requested); if (requestedHarness === "grok") return "grok"; if (route.mode === "direct") { @@ -283,9 +280,6 @@ export async function promptForModelSelection(options: { const benchmark = await fetchBenchmarkResults(options.fetchImpl); const requiredHarness = compatibleHarness(options.route, options.agent); const recommendations = buildRecommendedModelChoices(benchmark.results); - // DeepSecBench currently scores Grok models under the Pi harness (AI - // Gateway). When the operator asked for Grok Build, remap those rows so - // they still appear with the native harness + bare model id. const choices = recommendations .map((choice) => { if ( diff --git a/packages/deepsec/src/auth/model-route.ts b/packages/deepsec/src/auth/model-route.ts index defb2fb5..8d75aacd 100644 --- a/packages/deepsec/src/auth/model-route.ts +++ b/packages/deepsec/src/auth/model-route.ts @@ -74,8 +74,6 @@ export function modelRouteCompatibilityError( route: ModelRoute, agentType: string, ): string | undefined { - // Grok resolves its own XAI route inside resolveModelRoute; any stored - // gateway/custom config is ignored for this harness (not an error). if (isGrokAgent(agentType)) return undefined; if (route.mode === "custom" && agentType !== "pi") { return `Custom model routes require --agent pi (received ${agentType})`; @@ -89,7 +87,6 @@ export function modelRouteCompatibilityError( return undefined; } -/** Synthetic route for Grok Build (XAI_API_KEY or local grok login). */ function resolveGrokModelRoute(env: NodeJS.ProcessEnv): ResolvedModelRoute { const credentialEnv = "XAI_API_KEY"; const credential = env[credentialEnv] ?? ""; @@ -132,8 +129,6 @@ export async function resolveModelRoute( options: ResolveModelRouteOptions, ): Promise { const env = options.env ?? process.env; - // Grok Build authenticates via XAI_API_KEY or `grok login`, independent of - // the selected gateway/direct/custom route stored for other harnesses. if (isGrokAgent(options.agentType)) { return resolveGrokModelRoute(env); } @@ -283,7 +278,6 @@ export async function verifyModelRouteWithFetch( route: ResolvedModelRoute, fetchImpl: typeof fetch = fetch, ): Promise { - // Grok OAuth (`grok login`) has no API key to probe; skip HTTP verification. if (route.route.provider === "xai" && !route.credential) return; const endpoint = modelsEndpoint(route); diff --git a/packages/deepsec/src/commands/sandbox-process.ts b/packages/deepsec/src/commands/sandbox-process.ts index bcb0f3ab..dff319be 100644 --- a/packages/deepsec/src/commands/sandbox-process.ts +++ b/packages/deepsec/src/commands/sandbox-process.ts @@ -148,8 +148,6 @@ export async function sandboxCommand(subcommand: string, opts: SandboxOpts) { const projectId = resolveProjectId(opts.projectId); const config = buildConfig(subcommand as SandboxSubcommand, projectId, opts); - // Grok Build is local-only until sandbox brokering installs the CLI and - // injects XAI credentials into the microVM network policy. if (config.agentType === "grok" || config.agentType === "grok-build") { console.error( `Sandbox mode does not support --agent grok yet.\n` + diff --git a/packages/deepsec/src/preflight.ts b/packages/deepsec/src/preflight.ts index 5b653c24..be933773 100644 --- a/packages/deepsec/src/preflight.ts +++ b/packages/deepsec/src/preflight.ts @@ -187,10 +187,6 @@ function hasLocalPiAgent(): boolean { return existsSync(join(piHome, "auth.json")); } -/** - * Grok Build: XAI_API_KEY, or a real auth.json from `grok login`. - * `which grok` alone is not enough (logged-out CLI would pass preflight). - */ function hasLocalGrokAgent(): boolean { if (process.env.XAI_API_KEY) return true; const homes = [process.env.GROK_HOME, join(homedir(), ".grok")].filter( diff --git a/packages/deepsec/src/resolve-agent-type.ts b/packages/deepsec/src/resolve-agent-type.ts index bbfdf19e..3c6f6bfe 100644 --- a/packages/deepsec/src/resolve-agent-type.ts +++ b/packages/deepsec/src/resolve-agent-type.ts @@ -11,7 +11,6 @@ import { getConfig } from "@deepsec/core"; export function resolveAgentType(provided: string | undefined): string { const resolved = provided ?? getConfig()?.defaultAgent ?? "codex"; if (resolved === "claude") return "claude-agent-sdk"; - // Accept both short and long forms for the Grok Build harness. if (resolved === "grok-build") return "grok"; return resolved; } diff --git a/packages/processor/src/__tests__/grok-build.test.ts b/packages/processor/src/__tests__/grok-build.test.ts index 76595173..ce25e4e6 100644 --- a/packages/processor/src/__tests__/grok-build.test.ts +++ b/packages/processor/src/__tests__/grok-build.test.ts @@ -57,8 +57,6 @@ describe("Grok Build agent", () => { }); it("makeIsolatedGrokHome creates a config and mirrors auth when available", () => { - // Seed a fake user auth so we exercise the mirror path even if the - // developer machine has no real grok login in this environment. const seed = fs.mkdtempSync(path.join(os.tmpdir(), "deepsec-grok-seed-")); homes.push(seed); fs.writeFileSync(path.join(seed, "auth.json"), JSON.stringify({ token: "test" }), { @@ -71,7 +69,6 @@ describe("Grok Build agent", () => { homes.push(home); expect(fs.existsSync(path.join(home, "config.toml"))).toBe(true); expect(fs.existsSync(path.join(home, "auth.json"))).toBe(true); - // No skills / plugins directory copied in. expect(fs.existsSync(path.join(home, "skills"))).toBe(false); } finally { if (prev === undefined) delete process.env.GROK_HOME; diff --git a/packages/processor/src/agents/grok-build.ts b/packages/processor/src/agents/grok-build.ts index 4c5187d3..623a779f 100644 --- a/packages/processor/src/agents/grok-build.ts +++ b/packages/processor/src/agents/grok-build.ts @@ -39,18 +39,11 @@ import type { SetupTaskParams, } from "./types.js"; -/** - * Grok Build coding-agent backend: headless `grok -p` with a restricted - * tool allowlist. Auth is XAI_API_KEY or mirrored `~/.grok/auth.json`. - */ - const DEFAULT_MODEL = "grok-4.5"; const DEFAULT_THINKING_LEVEL = "xhigh"; -// Internal Grok tool ids (not Claude's Read/Grep names). const INVESTIGATE_TOOLS = "read_file,grep,list_dir,run_terminal_cmd"; const SETUP_TOOLS = "read_file,grep,list_dir"; -// Follow-ups (JSON repair / refusal) should not re-open the tool loop. const TOOLLESS = "read_file"; const GROK_ENV_ALLOWLIST = new Set([ @@ -117,7 +110,6 @@ interface GrokRunOptions { signal?: AbortSignal; onProgress?: (progress: AgentProgress) => void; grokHome?: string; - /** Keep GROK_HOME after the run so --resume can find the session. */ keepHome?: boolean; } @@ -146,10 +138,6 @@ function resolveGrokBinary(): string { return "grok"; } -/** - * Minimal GROK_HOME without the operator's skills/MCP plugins (those - * inflate the system prompt). Mirrors auth.json when present. - */ export function makeIsolatedGrokHome(): string { const home = fs.mkdtempSync(path.join(os.tmpdir(), "deepsec-grok-home-")); fs.writeFileSync( @@ -178,7 +166,6 @@ export function makeIsolatedGrokHome(): string { return home; } -/** Exported for tests. */ export function buildGrokEnv(grokHome: string): Record { const env: Record = {}; for (const [k, v] of Object.entries(process.env)) { @@ -232,7 +219,6 @@ function extractJsonObject(text: string): string | undefined { return undefined; } -/** Exported for tests. */ export function parseGrokStdout(stdout: string): GrokJsonResult { const trimmed = stdout.trim(); if (!trimmed) throw new Error("Grok produced empty stdout"); @@ -264,7 +250,6 @@ function metaFromGrokJson(raw: GrokJsonResult): Partial { return meta; } -/** One headless `grok -p` turn. Returns final JSON text + spend meta. */ export async function runGrokHeadless(opts: GrokRunOptions): Promise { const bin = resolveGrokBinary(); const grokHome = opts.grokHome ?? makeIsolatedGrokHome(); @@ -335,7 +320,6 @@ export async function runGrokHeadless(opts: GrokRunOptions): Promise Date: Wed, 12 Aug 2026 02:40:20 +0530 Subject: [PATCH 5/8] revert gitignore review notes entry --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 96339d85..96d46dbb 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,3 @@ linear-* plans/ .pnpm-store/ .env* - -# local review notes -.review-grok-agent.md From 3086863219358189f7039c871d65f83db4bf4e27 Mon Sep 17 00:00:00 2001 From: ayush-that Date: Wed, 12 Aug 2026 02:49:35 +0530 Subject: [PATCH 6/8] fold Grok Build docs into existing models.md patterns --- docs/models.md | 60 +++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/docs/models.md b/docs/models.md index 8d93be83..550e2ccf 100644 --- a/docs/models.md +++ b/docs/models.md @@ -1,6 +1,6 @@ --- title: "Models" -description: "Choose Codex, Claude, or Pi for process and revalidate runs, and compare models under the same workload." +description: "Choose Codex, Claude, Pi, or Grok for process and revalidate runs, and compare models under the same workload." --- deepsec talks to LLMs through interchangeable agent backends: @@ -9,38 +9,10 @@ deepsec talks to LLMs through interchangeable agent backends: |-----------------------------|-----------------------|------------------------------| | `codex` (default) | `gpt-5.5` | `process`, `revalidate` | | `claude` | `claude-opus-4-8` | `process`, `revalidate` | -| `pi` | `zai/glm-5.2` | `process`, `revalidate` | -| `grok` | `grok-4.5` | `process`, `revalidate`, setup | +| `pi` | `zai/glm-5.2` | `process`, `revalidate` | +| `grok` | `grok-4.5` | `process`, `revalidate` | | `claude` (triage) | `claude-sonnet-4-6` | `triage` (Claude-only) | -### Grok Build (`--agent grok`) - -Uses the local [Grok Build](https://grok.com) CLI (`grok`) in headless mode -(`grok -p … --output-format json`). Auth is independent of Vercel AI Gateway: - -```bash -# Option A: API key from https://console.x.ai -export XAI_API_KEY=xai-... - -# Option B: browser / device login once -grok login - -# Then: -npx deepsec init --agent grok --model grok-4.5 -# or, later: -pnpm deepsec process --project-id my-app --agent grok --model grok-4.5 -``` - -Requires the `grok` binary on `PATH` (or `GROK_EXECUTABLE`). Each batch runs -with an isolated `GROK_HOME` (auth mirrored, skills/plugins not loaded), -`--tools read_file,grep,list_dir,run_terminal_cmd`, and `--sandbox read-only` -by default (`DEEPSEC_GROK_SANDBOX` overrides; nested sandbox is off when -`DEEPSEC_INSIDE_SANDBOX=1`). - -**Local only for now.** `deepsec sandbox … --agent grok` exits early: Vercel -Sandbox does not yet install the Grok CLI or broker `XAI_API_KEY` into the -microVM. Use local `process` / `revalidate` / `init` instead. - Interactive one-shot setup recommends five benchmark-backed combinations: GPT-5.6 Sol, Claude Opus 5, Kimi K3, Grok 4.5, and the current DeepSeek entry. Deepsec fetches the latest score, reasoning level, harness, and total run cost @@ -69,7 +41,8 @@ npx deepsec init --yes --model-profile value --output jsonl ``` Direct OpenAI and Anthropic credentials automatically restrict profiles to a -compatible Codex or Claude harness; custom routes restrict them to Pi. +compatible Codex or Claude harness; custom routes restrict them to Pi. Grok +uses `XAI_API_KEY` or a prior `grok login` rather than a gateway route. The built-in backends work with Vercel AI Gateway through the linked workspace's OIDC credential. The model credential route is independent of the @@ -98,6 +71,12 @@ pnpm deepsec process --project-id my-app --agent pi # Pi with an AI SDK / AI Gateway style model id: pnpm deepsec process --project-id my-app --agent pi --model zai/glm-5.2 +# Grok Build CLI (local), default model: +pnpm deepsec process --project-id my-app --agent grok + +# Grok Build CLI, specific model: +pnpm deepsec process --project-id my-app --agent grok --model grok-4.5 + # Triage uses Claude; pass a cheaper model if you want: pnpm deepsec triage --project-id my-app --model claude-haiku-4-5 ``` @@ -127,6 +106,7 @@ The flag maps onto each backend's native dial: |----------|---------------------------------------------| | `codex` | model reasoning effort (`minimal`–`xhigh`) | | `pi` | thinking level (`minimal`–`xhigh`) | +| `grok` | `--reasoning-effort` (`minimal`–`xhigh`) | | `claude` | adaptive-thinking effort (`minimal` → `low`, `xhigh` → `max`) | It applies to the main investigation/revalidation runs only. @@ -193,6 +173,22 @@ Later `process`, `revalidate`, and Sandbox commands resolve the persisted route. Per-command `--ai-provider`, `--ai-base-url`, `--ai-api-key-env`, and repeatable `--ai-header name=value` remain available as Pi runtime overrides. +### Grok Build for local CLI runs + +Grok spawns the local [Grok Build](https://x.ai/cli) CLI (`grok`) headlessly +with the same deepsec prompt/schema as the other backends. Default model is +`grok-4.5`. Auth is `XAI_API_KEY` or a prior `grok login` (the binary must be +on `PATH`, or set `GROK_EXECUTABLE`): + +```bash +export XAI_API_KEY=xai-... +# or: grok login +pnpm deepsec process --project-id my-app --agent grok +``` + +Sandbox mode does not support `--agent grok` yet. Use a local `process` or +`revalidate` run instead. + ### `claude-sonnet-4-6` for `triage` Triage buckets findings into P0/P1/P2/skip without re-reading the code. From 597c5563739cae667ae6384350bbbd7cb3373de8 Mon Sep 17 00:00:00 2001 From: ayush-that Date: Wed, 12 Aug 2026 11:03:22 +0530 Subject: [PATCH 7/8] fix lint and knip for Grok Build agent --- packages/deepsec/src/auth/model-picker.ts | 4 +--- packages/processor/src/agents/grok-build.ts | 14 +++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/deepsec/src/auth/model-picker.ts b/packages/deepsec/src/auth/model-picker.ts index 18c38d42..d0589ae2 100644 --- a/packages/deepsec/src/auth/model-picker.ts +++ b/packages/deepsec/src/auth/model-picker.ts @@ -141,9 +141,7 @@ function strongest(results: BenchmarkResult[], modelId: string): BenchmarkResult function configuredModel(result: BenchmarkResult): string { if (result.harness === "pi") return result.modelId; if (result.harness === "grok") { - return result.modelId.startsWith("xai/") - ? result.modelId.slice("xai/".length) - : result.model; + return result.modelId.startsWith("xai/") ? result.modelId.slice("xai/".length) : result.model; } return result.model; } diff --git a/packages/processor/src/agents/grok-build.ts b/packages/processor/src/agents/grok-build.ts index 623a779f..acda0bf6 100644 --- a/packages/processor/src/agents/grok-build.ts +++ b/packages/processor/src/agents/grok-build.ts @@ -73,7 +73,7 @@ const GROK_ENV_ALLOWLIST = new Set([ "RUST_BACKTRACE", ]); -export interface GrokJsonResult { +interface GrokJsonResult { text?: string; stopReason?: string; sessionId?: string; @@ -142,7 +142,7 @@ export function makeIsolatedGrokHome(): string { const home = fs.mkdtempSync(path.join(os.tmpdir(), "deepsec-grok-home-")); fs.writeFileSync( path.join(home, "config.toml"), - ['[ui]', 'permission_mode = "dontAsk"', "", "[cli]", "auto_update = false", ""].join("\n"), + ["[ui]", 'permission_mode = "dontAsk"', "", "[cli]", "auto_update = false", ""].join("\n"), { mode: 0o600 }, ); @@ -193,14 +193,14 @@ function extractJsonObject(text: string): string | undefined { if (start < 0) return undefined; let depth = 0; let inString = false; - let escape = false; + let escaped = false; for (let i = start; i < text.length; i++) { const ch = text[i]; if (inString) { - if (escape) { - escape = false; + if (escaped) { + escaped = false; } else if (ch === "\\") { - escape = true; + escaped = true; } else if (ch === '"') { inString = false; } @@ -250,7 +250,7 @@ function metaFromGrokJson(raw: GrokJsonResult): Partial { return meta; } -export async function runGrokHeadless(opts: GrokRunOptions): Promise { +async function runGrokHeadless(opts: GrokRunOptions): Promise { const bin = resolveGrokBinary(); const grokHome = opts.grokHome ?? makeIsolatedGrokHome(); const ownHome = opts.grokHome === undefined; From 19e72c948815beb22c1937b5047a28d6c2fca3a7 Mon Sep 17 00:00:00 2001 From: ayush-that Date: Wed, 12 Aug 2026 23:20:14 +0530 Subject: [PATCH 8/8] default Grok Build agent to grok-4.6 --- docs/models.md | 6 +++--- packages/deepsec/src/__tests__/agent-defaults.test.ts | 4 ++-- packages/deepsec/src/agent-defaults.ts | 2 +- packages/processor/src/agents/grok-build.ts | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/models.md b/docs/models.md index 550e2ccf..9b9cc743 100644 --- a/docs/models.md +++ b/docs/models.md @@ -10,7 +10,7 @@ deepsec talks to LLMs through interchangeable agent backends: | `codex` (default) | `gpt-5.5` | `process`, `revalidate` | | `claude` | `claude-opus-4-8` | `process`, `revalidate` | | `pi` | `zai/glm-5.2` | `process`, `revalidate` | -| `grok` | `grok-4.5` | `process`, `revalidate` | +| `grok` | `grok-4.6` | `process`, `revalidate` | | `claude` (triage) | `claude-sonnet-4-6` | `triage` (Claude-only) | Interactive one-shot setup recommends five benchmark-backed combinations: @@ -75,7 +75,7 @@ pnpm deepsec process --project-id my-app --agent pi --model zai/glm-5.2 pnpm deepsec process --project-id my-app --agent grok # Grok Build CLI, specific model: -pnpm deepsec process --project-id my-app --agent grok --model grok-4.5 +pnpm deepsec process --project-id my-app --agent grok --model grok-4.6 # Triage uses Claude; pass a cheaper model if you want: pnpm deepsec triage --project-id my-app --model claude-haiku-4-5 @@ -177,7 +177,7 @@ repeatable `--ai-header name=value` remain available as Pi runtime overrides. Grok spawns the local [Grok Build](https://x.ai/cli) CLI (`grok`) headlessly with the same deepsec prompt/schema as the other backends. Default model is -`grok-4.5`. Auth is `XAI_API_KEY` or a prior `grok login` (the binary must be +`grok-4.6`. Auth is `XAI_API_KEY` or a prior `grok login` (the binary must be on `PATH`, or set `GROK_EXECUTABLE`): ```bash diff --git a/packages/deepsec/src/__tests__/agent-defaults.test.ts b/packages/deepsec/src/__tests__/agent-defaults.test.ts index 96624de0..de9744ad 100644 --- a/packages/deepsec/src/__tests__/agent-defaults.test.ts +++ b/packages/deepsec/src/__tests__/agent-defaults.test.ts @@ -9,8 +9,8 @@ describe("defaultModelForAgent", () => { expect(defaultModelForAgent("codex")).toBe("gpt-5.5"); expect(defaultModelForAgent("pi")).toBe("zai/glm-5.2"); expect(defaultModelForAgent("claude-agent-sdk")).toBe("claude-opus-4-8"); - expect(defaultModelForAgent("grok")).toBe("grok-4.5"); - expect(defaultModelForAgent("grok-build")).toBe("grok-4.5"); + expect(defaultModelForAgent("grok")).toBe("grok-4.6"); + expect(defaultModelForAgent("grok-build")).toBe("grok-4.6"); }); it("uses the model persisted for the configured harness", () => { diff --git a/packages/deepsec/src/agent-defaults.ts b/packages/deepsec/src/agent-defaults.ts index aca22b3e..6b873aec 100644 --- a/packages/deepsec/src/agent-defaults.ts +++ b/packages/deepsec/src/agent-defaults.ts @@ -16,7 +16,7 @@ export function defaultModelForAgent(agentType: string): string { return "zai/glm-5.2"; case "grok": case "grok-build": - return "grok-4.5"; + return "grok-4.6"; default: return "claude-opus-4-8"; } diff --git a/packages/processor/src/agents/grok-build.ts b/packages/processor/src/agents/grok-build.ts index acda0bf6..f176120c 100644 --- a/packages/processor/src/agents/grok-build.ts +++ b/packages/processor/src/agents/grok-build.ts @@ -39,7 +39,7 @@ import type { SetupTaskParams, } from "./types.js"; -const DEFAULT_MODEL = "grok-4.5"; +const DEFAULT_MODEL = "grok-4.6"; const DEFAULT_THINKING_LEVEL = "xhigh"; const INVESTIGATE_TOOLS = "read_file,grep,list_dir,run_terminal_cmd";