Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions docs/automode-classifier-flow.md
Original file line number Diff line number Diff line change
Expand Up @@ -265,11 +265,13 @@ No classifier model/API key available; auto mode fails closed.
Classifier calls use:

```text
temperature: 0
maxTokens: 700
maxTokens: 1200
signal: ctx.signal
```

The classifier does not force a temperature, because some providers reject the
parameter. Provider defaults are used instead.

## Parsing the classifier result

The parser accepts JSON returned directly, JSON inside a fenced code block, or the first JSON-looking object in the response.
Expand All @@ -287,7 +289,7 @@ If parsing fails, the action is blocked with this reason:
Classifier response was not valid decision JSON; auto mode fails closed.
```

If the model call throws, the action is blocked with a classifier failure message.
If the model call throws or returns a provider error, the action is blocked with a classifier failure message.

## State, UI, and denial history

Expand Down
2 changes: 1 addition & 1 deletion docs/observability-logging.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ Written only for classifier-routed actions, and only when `classifierIo: true`.

Each `attempts[]` entry is `{ attempt, response?, parsed?, error?, durationMs }`:

- `response` — `{ stopReason, text, model, timestamp, usage }`, the raw model output and provider-reported usage for that attempt.
- `response` — `{ stopReason, text, model, timestamp, usage, errorMessage? }`, the raw model output and provider-reported usage for that attempt, including provider-reported errors.
- `parsed` — the decision parsed from the response, or absent if it did not parse.
- `error` — present when the call threw (network/auth); `response` is then absent.

Expand Down
27 changes: 21 additions & 6 deletions extensions/auto-mode/classifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ export type ClassifierCompletionFn = (
headers?: Record<string, string>;
signal?: AbortSignal;
maxTokens: number;
temperature: number;
temperature?: number;
},
) => Promise<AssistantMessage>;

Expand Down Expand Up @@ -125,8 +125,8 @@ export function parseClassifierDecision(
*
* Safety is preserved: an "allow" is only returned when a response actually
* parses to a valid allow decision. If every attempt fails to parse, the
* function fails closed with a block decision. Thrown errors (network/auth)
* are not retried — they fail closed immediately, matching prior behavior.
* function fails closed with a block decision. Thrown or provider-reported
* errors are not retried — they fail closed immediately.
*/
export async function classifyWithRetry(
completeFn: ClassifierCompletionFn,
Expand All @@ -141,7 +141,7 @@ export async function classifyWithRetry(
): Promise<ClassificationDecision> {
const maxAttempts = options.maxAttempts ?? 2;
const maxTokens = options.maxTokens ?? 1200;
const temperature = options.temperature ?? 0;
const temperature = options.temperature;
const onAttempt = options.onAttempt;
let lastReason =
"Classifier response was not valid decision JSON; auto mode fails closed.";
Expand All @@ -157,7 +157,7 @@ export async function classifyWithRetry(
headers: classifier.headers,
signal,
maxTokens,
temperature,
...(temperature === undefined ? {} : { temperature }),
},
);
} catch (error) {
Expand All @@ -174,7 +174,10 @@ export async function classifyWithRetry(
};
}
const durationMs = Date.now() - started;
const decision = parseClassifierDecision(response);
const providerError = response.stopReason === "error";
const decision = providerError
? undefined
: parseClassifierDecision(response);
onAttempt?.({
attempt: attempt + 1,
response: {
Expand All @@ -183,10 +186,22 @@ export async function classifyWithRetry(
model: response.model,
timestamp: response.timestamp,
usage: response.usage,
...(response.errorMessage === undefined
? {}
: { errorMessage: response.errorMessage }),
},
parsed: decision,
durationMs,
});
if (providerError) {
const errorMessage = response.errorMessage ||
"Classifier model returned an error response.";
return {
decision: "block",
tier: "none",
reason: `Classifier failed; auto mode fails closed: ${errorMessage}`,
};
}
if (decision) return decision;
lastReason =
response.stopReason === "length"
Expand Down
1 change: 1 addition & 0 deletions extensions/auto-mode/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ export type ClassifierIoAttempt = {
model: string;
timestamp: number;
usage: AssistantMessage["usage"];
errorMessage?: string;
};
parsed?: ClassificationDecision;
error?: string;
Expand Down
62 changes: 59 additions & 3 deletions tests/auto-mode.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -394,14 +394,14 @@ const VALID_ALLOW = '{"decision":"allow","tier":"allow","reason":"read-only"}';
const GARBAGE = "and I'm ready to go. I'll start by listing the ability to ability to ability to";

function fakeComplete(responses: AssistantMessage[]) {
const calls: Array<{ maxTokens: number; temperature: number }> = [];
const calls: Array<{ maxTokens: number; temperature?: number }> = [];
let i = 0;
const fn = async (
_model: unknown,
_options: unknown,
callOptions: { maxTokens: number; temperature: number },
callOptions: { maxTokens: number; temperature?: number },
): Promise<AssistantMessage> => {
calls.push({ maxTokens: callOptions.maxTokens, temperature: callOptions.temperature });
calls.push({ ...callOptions });
const res = responses[i];
i += 1;
return res;
Expand All @@ -419,6 +419,21 @@ test("classifyWithRetry returns a valid decision on the first attempt without re
);
assert.equal(decision.decision, "allow");
assert.equal(calls.length, 1);
assert.equal(calls[0]?.maxTokens, 1200);
assert.equal(Object.hasOwn(calls[0] ?? {}, "temperature"), false);
});

test("classifyWithRetry forwards an explicitly configured temperature", async () => {
const { fn, calls } = fakeComplete([assistantWith(VALID_ALLOW)]);
const decision = await classifyWithRetry(
fn,
{ model: { provider: "test", id: "x" } },
{ systemPrompt: "s", messages: [] },
undefined,
{ temperature: 0 },
);
assert.equal(decision.decision, "allow");
assert.equal(calls[0]?.temperature, 0);
});

test("classifyWithRetry recovers when the first response is garbage and the second is valid", async () => {
Expand Down Expand Up @@ -478,6 +493,47 @@ test("classifyWithRetry fails closed immediately without retrying when complete
assert.equal(calls, 1);
});

test("classifyWithRetry surfaces provider-reported errors without retrying", async () => {
const response = {
...assistantWith("", "error"),
errorMessage: "Unsupported parameter: temperature",
};
const { fn, calls } = fakeComplete([response, assistantWith(VALID_ALLOW)]);
const attempts: ClassifierIoAttempt[] = [];
const decision = await classifyWithRetry(
fn,
{ model: { provider: "test", id: "x" } },
{ systemPrompt: "s", messages: [] },
undefined,
{ onAttempt: (attempt) => attempts.push(attempt) },
);
assert.equal(decision.decision, "block");
assert.match(decision.reason, /Unsupported parameter: temperature/);
assert.equal(calls.length, 1);
assert.equal(attempts[0]?.response?.errorMessage, "Unsupported parameter: temperature");
});

test("classifyWithRetry fails closed on an error stopReason with an empty message and valid allow JSON", async () => {
const response = {
...assistantWith(VALID_ALLOW, "error"),
errorMessage: "",
};
const { fn, calls } = fakeComplete([response, assistantWith(VALID_ALLOW)]);
const attempts: ClassifierIoAttempt[] = [];
const decision = await classifyWithRetry(
fn,
{ model: { provider: "test", id: "x" } },
{ systemPrompt: "s", messages: [] },
undefined,
{ onAttempt: (attempt) => attempts.push(attempt) },
);
assert.equal(decision.decision, "block");
assert.match(decision.reason, /Classifier model returned an error response/);
assert.equal(calls.length, 1);
assert.equal(attempts[0]?.parsed, undefined);
assert.equal(attempts[0]?.response?.errorMessage, "");
});

test("tool_call hook blocks permissions.deny before deterministic checks and classifier", async () => {
const pattern = parseToolPattern("bash(git push --force*)");
assert.ok(pattern);
Expand Down