diff --git a/docs/automode-classifier-flow.md b/docs/automode-classifier-flow.md index fcbb2fc..e24b31f 100644 --- a/docs/automode-classifier-flow.md +++ b/docs/automode-classifier-flow.md @@ -265,11 +265,13 @@ No classifier model/API key available; auto mode fails closed. Classifier calls use: ```text -temperature: 0 -maxTokens: 700 +maxTokens: 1200 signal: ctx.signal ``` +The classifier does not force a temperature, because some providers reject the +parameter. Provider defaults are used instead. + ## Parsing the classifier result The parser accepts JSON returned directly, JSON inside a fenced code block, or the first JSON-looking object in the response. @@ -287,7 +289,7 @@ If parsing fails, the action is blocked with this reason: Classifier response was not valid decision JSON; auto mode fails closed. ``` -If the model call throws, the action is blocked with a classifier failure message. +If the model call throws or returns a provider error, the action is blocked with a classifier failure message. ## State, UI, and denial history diff --git a/docs/observability-logging.md b/docs/observability-logging.md index aed3f59..48e5e8f 100644 --- a/docs/observability-logging.md +++ b/docs/observability-logging.md @@ -86,7 +86,7 @@ Written only for classifier-routed actions, and only when `classifierIo: true`. Each `attempts[]` entry is `{ attempt, response?, parsed?, error?, durationMs }`: -- `response` — `{ stopReason, text, model, timestamp, usage }`, the raw model output and provider-reported usage for that attempt. +- `response` — `{ stopReason, text, model, timestamp, usage, errorMessage? }`, the raw model output and provider-reported usage for that attempt, including provider-reported errors. - `parsed` — the decision parsed from the response, or absent if it did not parse. - `error` — present when the call threw (network/auth); `response` is then absent. diff --git a/extensions/auto-mode/classifier.ts b/extensions/auto-mode/classifier.ts index 4d04889..06cd1b7 100644 --- a/extensions/auto-mode/classifier.ts +++ b/extensions/auto-mode/classifier.ts @@ -65,7 +65,7 @@ export type ClassifierCompletionFn = ( headers?: Record; signal?: AbortSignal; maxTokens: number; - temperature: number; + temperature?: number; }, ) => Promise; @@ -125,8 +125,8 @@ export function parseClassifierDecision( * * Safety is preserved: an "allow" is only returned when a response actually * parses to a valid allow decision. If every attempt fails to parse, the - * function fails closed with a block decision. Thrown errors (network/auth) - * are not retried — they fail closed immediately, matching prior behavior. + * function fails closed with a block decision. Thrown or provider-reported + * errors are not retried — they fail closed immediately. */ export async function classifyWithRetry( completeFn: ClassifierCompletionFn, @@ -141,7 +141,7 @@ export async function classifyWithRetry( ): Promise { const maxAttempts = options.maxAttempts ?? 2; const maxTokens = options.maxTokens ?? 1200; - const temperature = options.temperature ?? 0; + const temperature = options.temperature; const onAttempt = options.onAttempt; let lastReason = "Classifier response was not valid decision JSON; auto mode fails closed."; @@ -157,7 +157,7 @@ export async function classifyWithRetry( headers: classifier.headers, signal, maxTokens, - temperature, + ...(temperature === undefined ? {} : { temperature }), }, ); } catch (error) { @@ -174,7 +174,10 @@ export async function classifyWithRetry( }; } const durationMs = Date.now() - started; - const decision = parseClassifierDecision(response); + const providerError = response.stopReason === "error"; + const decision = providerError + ? undefined + : parseClassifierDecision(response); onAttempt?.({ attempt: attempt + 1, response: { @@ -183,10 +186,22 @@ export async function classifyWithRetry( model: response.model, timestamp: response.timestamp, usage: response.usage, + ...(response.errorMessage === undefined + ? {} + : { errorMessage: response.errorMessage }), }, parsed: decision, durationMs, }); + if (providerError) { + const errorMessage = response.errorMessage || + "Classifier model returned an error response."; + return { + decision: "block", + tier: "none", + reason: `Classifier failed; auto mode fails closed: ${errorMessage}`, + }; + } if (decision) return decision; lastReason = response.stopReason === "length" diff --git a/extensions/auto-mode/types.ts b/extensions/auto-mode/types.ts index 1799562..4ed8c2c 100644 --- a/extensions/auto-mode/types.ts +++ b/extensions/auto-mode/types.ts @@ -98,6 +98,7 @@ export type ClassifierIoAttempt = { model: string; timestamp: number; usage: AssistantMessage["usage"]; + errorMessage?: string; }; parsed?: ClassificationDecision; error?: string; diff --git a/tests/auto-mode.test.ts b/tests/auto-mode.test.ts index 6586949..69af227 100644 --- a/tests/auto-mode.test.ts +++ b/tests/auto-mode.test.ts @@ -394,14 +394,14 @@ const VALID_ALLOW = '{"decision":"allow","tier":"allow","reason":"read-only"}'; const GARBAGE = "and I'm ready to go. I'll start by listing the ability to ability to ability to"; function fakeComplete(responses: AssistantMessage[]) { - const calls: Array<{ maxTokens: number; temperature: number }> = []; + const calls: Array<{ maxTokens: number; temperature?: number }> = []; let i = 0; const fn = async ( _model: unknown, _options: unknown, - callOptions: { maxTokens: number; temperature: number }, + callOptions: { maxTokens: number; temperature?: number }, ): Promise => { - calls.push({ maxTokens: callOptions.maxTokens, temperature: callOptions.temperature }); + calls.push({ ...callOptions }); const res = responses[i]; i += 1; return res; @@ -419,6 +419,21 @@ test("classifyWithRetry returns a valid decision on the first attempt without re ); assert.equal(decision.decision, "allow"); assert.equal(calls.length, 1); + assert.equal(calls[0]?.maxTokens, 1200); + assert.equal(Object.hasOwn(calls[0] ?? {}, "temperature"), false); +}); + +test("classifyWithRetry forwards an explicitly configured temperature", async () => { + const { fn, calls } = fakeComplete([assistantWith(VALID_ALLOW)]); + const decision = await classifyWithRetry( + fn, + { model: { provider: "test", id: "x" } }, + { systemPrompt: "s", messages: [] }, + undefined, + { temperature: 0 }, + ); + assert.equal(decision.decision, "allow"); + assert.equal(calls[0]?.temperature, 0); }); test("classifyWithRetry recovers when the first response is garbage and the second is valid", async () => { @@ -478,6 +493,47 @@ test("classifyWithRetry fails closed immediately without retrying when complete assert.equal(calls, 1); }); +test("classifyWithRetry surfaces provider-reported errors without retrying", async () => { + const response = { + ...assistantWith("", "error"), + errorMessage: "Unsupported parameter: temperature", + }; + const { fn, calls } = fakeComplete([response, assistantWith(VALID_ALLOW)]); + const attempts: ClassifierIoAttempt[] = []; + const decision = await classifyWithRetry( + fn, + { model: { provider: "test", id: "x" } }, + { systemPrompt: "s", messages: [] }, + undefined, + { onAttempt: (attempt) => attempts.push(attempt) }, + ); + assert.equal(decision.decision, "block"); + assert.match(decision.reason, /Unsupported parameter: temperature/); + assert.equal(calls.length, 1); + assert.equal(attempts[0]?.response?.errorMessage, "Unsupported parameter: temperature"); +}); + +test("classifyWithRetry fails closed on an error stopReason with an empty message and valid allow JSON", async () => { + const response = { + ...assistantWith(VALID_ALLOW, "error"), + errorMessage: "", + }; + const { fn, calls } = fakeComplete([response, assistantWith(VALID_ALLOW)]); + const attempts: ClassifierIoAttempt[] = []; + const decision = await classifyWithRetry( + fn, + { model: { provider: "test", id: "x" } }, + { systemPrompt: "s", messages: [] }, + undefined, + { onAttempt: (attempt) => attempts.push(attempt) }, + ); + assert.equal(decision.decision, "block"); + assert.match(decision.reason, /Classifier model returned an error response/); + assert.equal(calls.length, 1); + assert.equal(attempts[0]?.parsed, undefined); + assert.equal(attempts[0]?.response?.errorMessage, ""); +}); + test("tool_call hook blocks permissions.deny before deterministic checks and classifier", async () => { const pattern = parseToolPattern("bash(git push --force*)"); assert.ok(pattern);