diff --git a/.claude/skills/sync-models/SKILL.md b/.claude/skills/sync-models/SKILL.md index 0072ba2e10a..072b7f7b9bf 100644 --- a/.claude/skills/sync-models/SKILL.md +++ b/.claude/skills/sync-models/SKILL.md @@ -64,7 +64,19 @@ Both directions share the same source of truth: whatever `https://router.hugging - **Whether it's a reasoning model** (see step 5) 5. **Decide if the model is reasoning-capable** - A model is "reasoning-capable" for chat-ui purposes if it accepts the OpenAI-style `reasoning_effort: low|medium|high` parameter via the HF router and _meaningfully changes its chain-of-thought depth_ in response. Whether that holds depends on **both the model and the providers serving it** — the router is a transparent proxy, so behavior comes from each provider's implementation. Don't decide from the name alone. + The `supportsReasoning` flag controls TWO behaviors, and both must be appropriate before flagging: + + 1. chat-ui renders the thinking-effort dropdown and forwards `reasoning_effort` to the router; + 2. chat-ui echoes the model's prior reasoning back as `reasoning_content` on past assistant messages (cross-turn "preserved thinking" — see `prepareFiles.ts` / `endpointOai.ts`). + + A model qualifies for (1) if it accepts the OpenAI-style `reasoning_effort: low|medium|high` parameter via the HF router and _meaningfully changes its chain-of-thought depth_ in response. Whether that holds depends on **both the model and the providers serving it** — the router is a transparent proxy, so behavior comes from each provider's implementation. Don't decide from the name alone. + + For (2), check the vendor's preserved-thinking / multi-turn guidance, because it can point in either direction: + + - **Flag-strengthening**: the vendor documents that prior `reasoning_content` must or should be passed back in multi-turn or tool-calling flows. Known examples: Moonshot `thinking.keep` (Kimi K2.6+/K3), MiniMax "Interleaved Thinking" ("must preserve the model's thinking content completely"), DeepSeek V4 thinking mode (hard 400 if `reasoning_content` is missing on tool-call turns), Z.ai "Preserved Thinking" (`clear_thinking: false`), Qwen3.6 `preserve_thinking`. + - **Flag-blocking**: the vendor documents that historical thoughts must be STRIPPED across completed turns. Known example: the Gemma family — Google requires removing thoughts from previous turns ("historical model output must only include the final response") while preserving them only inside a single turn's tool loop, which chat-ui handles automatically without the flag. **Do not flag such models even though they emit reasoning and may accept an effort knob** — flagging would make chat-ui echo reasoning the vendor says to strip. + + Vendor doc entry points for the preserved-thinking check: , , , , , plus the model card's own multi-turn/best-practices section (Qwen cards document what their chat template does with historical `` blocks). **Heuristic shortlist (candidates worth verifying):** @@ -141,7 +153,7 @@ Both directions share the same source of truth: whatever `https://router.hugging } ``` - `supportsReasoning` is what makes chat-ui render the Thinking-effort dropdown in the chat footer for that model and forward `reasoning_effort` to the router. `supportsArtifacts` enables the artifacts side panel for the model. + `supportsReasoning` makes chat-ui render the Thinking-effort dropdown in the chat footer, forward `reasoning_effort` to the router, AND echo the model's prior reasoning back as `reasoning_content` on past assistant messages (preserved thinking). Models whose vendor requires stripping historical thoughts (Gemma family) must stay unflagged — see step 5. `supportsArtifacts` enables the artifacts side panel for the model. 9. **Remove deprecated models from both files** Delete the full entry line for every id in the deprecated set (step 3) from **both** `chart/env/prod.yaml` and `chart/env/dev.yaml`. Match on the exact `"id"` value so near-duplicate ids aren't removed by accident, and keep the removal symmetric — the two files must end with the same model set. diff --git a/chart/env/dev.yaml b/chart/env/dev.yaml index 9c5b21adaf3..fcd0fa18bc0 100644 --- a/chart/env/dev.yaml +++ b/chart/env/dev.yaml @@ -111,16 +111,16 @@ envVars: { "id": "moonshotai/Kimi-K2.7-Code", "description": "1T MoE coding agent with persistent thinking across long tool workflows." , "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 98304 } }, { "id": "MiniMaxAI/MiniMax-M3", "description": "Natively multimodal 428B MoE with 1M context for frontier coding and agents." , "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 98304 } }, { "id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "description": "Hybrid Mamba-Transformer 550B MoE with 55B active params for efficient agentic reasoning.", "parameters": { "max_tokens": 98304 }, "supportsReasoning": true, "supportsArtifacts": true }, - { "id": "Qwen/Qwen3.6-27B", "description": "Dense 27B hybrid with DeltaNet attention and 1M context.", "supportsArtifacts": true }, + { "id": "Qwen/Qwen3.6-27B", "description": "Dense 27B hybrid with DeltaNet attention and 1M context.", "supportsReasoning": true, "supportsArtifacts": true }, { "id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "description": "Native multimodal 17B MoE with 16 experts and very long context."}, { "id": "CohereLabs/c4ai-command-r7b-12-2024", "description": "Compact 7B Command R for multilingual RAG, tool use, and agents." }, { "id": "pearl-ai/Gemma-4-31B-it-pearl", "description": "Community Gemma-4-31B variant integrated with Pearl mining for chain-validated inference."}, { "id": "inclusionAI/Ling-2.6-1T", "description": "1T MoE with 50B active params, hybrid MLA-Linear attention, and fast thinking.", "supportsArtifacts": true }, { "id": "deepseek-ai/DeepSeek-V4-Pro", "description": "Frontier 1.6T MoE with 49B active params, hybrid attention, and 1M context.", "parameters": { "max_tokens": 98304 }, "supportsReasoning": true, "supportsArtifacts": true }, - { "id": "deepseek-ai/DeepSeek-V4-Flash", "description": "Compact 284B MoE with 13B active params, hybrid attention, and 1M context.", "parameters": { "max_tokens": 49152 }, "supportsArtifacts": true }, + { "id": "deepseek-ai/DeepSeek-V4-Flash", "description": "Compact 284B MoE with 13B active params, hybrid attention, and 1M context.", "parameters": { "max_tokens": 49152 }, "supportsReasoning": true, "supportsArtifacts": true }, { "id": "deepseek-ai/DeepSeek-V3.2", "description": "Stable 671B MoE with sparse attention for agentic reasoning and long contexts.", "supportsArtifacts": true }, { "id": "stepfun-ai/Step-3.5-Flash", "description": "Sparse 197B MoE agent with multi-token prediction and 256K context.", "supportsArtifacts": true }, - { "id": "Qwen/Qwen3.6-35B-A3B", "description": "Hybrid 35B MoE with 3B active params, DeltaNet attention, and 1M context.", "supportsArtifacts": true }, + { "id": "Qwen/Qwen3.6-35B-A3B", "description": "Hybrid 35B MoE with 3B active params, DeltaNet attention, and 1M context.", "supportsReasoning": true, "supportsArtifacts": true }, { "id": "zai-org/GLM-4.7-Flash", "description": "Compact 30B MoE for agentic coding with 128K context and tool use.", "supportsReasoning": true, "supportsArtifacts": true }, { "id": "moonshotai/Kimi-K2.6", "description": "Native multimodal 1T MoE for long-horizon coding and 300-sub-agent swarms.", "supportsArtifacts": true }, { "id": "MiniMaxAI/MiniMax-M2.7", "description": "Self-evolving 230B MoE agent for frontier coding, reasoning, and tool use." , "supportsReasoning": true, "supportsArtifacts": true }, diff --git a/chart/env/prod.yaml b/chart/env/prod.yaml index ccd320988cb..4893097a4e7 100644 --- a/chart/env/prod.yaml +++ b/chart/env/prod.yaml @@ -121,16 +121,16 @@ envVars: { "id": "moonshotai/Kimi-K2.7-Code", "description": "1T MoE coding agent with persistent thinking across long tool workflows." , "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 98304 } }, { "id": "MiniMaxAI/MiniMax-M3", "description": "Natively multimodal 428B MoE with 1M context for frontier coding and agents." , "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 98304 } }, { "id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "description": "Hybrid Mamba-Transformer 550B MoE with 55B active params for efficient agentic reasoning.", "parameters": { "max_tokens": 98304 }, "supportsReasoning": true, "supportsArtifacts": true }, - { "id": "Qwen/Qwen3.6-27B", "description": "Dense 27B hybrid with DeltaNet attention and 1M context.", "supportsArtifacts": true, "parameters": { "max_tokens": 49152 } }, + { "id": "Qwen/Qwen3.6-27B", "description": "Dense 27B hybrid with DeltaNet attention and 1M context.", "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 49152 } }, { "id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "description": "Native multimodal 17B MoE with 16 experts and very long context."}, { "id": "CohereLabs/c4ai-command-r7b-12-2024", "description": "Compact 7B Command R for multilingual RAG, tool use, and agents." }, { "id": "pearl-ai/Gemma-4-31B-it-pearl", "description": "Community Gemma-4-31B variant integrated with Pearl mining for chain-validated inference."}, { "id": "inclusionAI/Ling-2.6-1T", "description": "1T MoE with 50B active params, hybrid MLA-Linear attention, and fast thinking.", "supportsArtifacts": true, "parameters": { "max_tokens": 32768 } }, { "id": "deepseek-ai/DeepSeek-V4-Pro", "description": "Frontier 1.6T MoE with 49B active params, hybrid attention, and 1M context.", "parameters": { "max_tokens": 98304 }, "supportsReasoning": true, "supportsArtifacts": true }, - { "id": "deepseek-ai/DeepSeek-V4-Flash", "description": "Compact 284B MoE with 13B active params, hybrid attention, and 1M context.", "parameters": { "max_tokens": 49152 }, "supportsArtifacts": true }, + { "id": "deepseek-ai/DeepSeek-V4-Flash", "description": "Compact 284B MoE with 13B active params, hybrid attention, and 1M context.", "parameters": { "max_tokens": 49152 }, "supportsReasoning": true, "supportsArtifacts": true }, { "id": "deepseek-ai/DeepSeek-V3.2", "description": "Stable 671B MoE with sparse attention for agentic reasoning and long contexts.", "supportsArtifacts": true, "parameters": { "max_tokens": 49152 } }, { "id": "stepfun-ai/Step-3.5-Flash", "description": "Sparse 197B MoE agent with multi-token prediction and 256K context.", "supportsArtifacts": true, "parameters": { "max_tokens": 49152 } }, - { "id": "Qwen/Qwen3.6-35B-A3B", "description": "Hybrid 35B MoE with 3B active params, DeltaNet attention, and 1M context.", "supportsArtifacts": true, "parameters": { "max_tokens": 49152 } }, + { "id": "Qwen/Qwen3.6-35B-A3B", "description": "Hybrid 35B MoE with 3B active params, DeltaNet attention, and 1M context.", "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 49152 } }, { "id": "zai-org/GLM-4.7-Flash", "description": "Compact 30B MoE for agentic coding with 128K context and tool use.", "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 98304 } }, { "id": "moonshotai/Kimi-K2.6", "description": "Native multimodal 1T MoE for long-horizon coding and 300-sub-agent swarms.", "supportsArtifacts": true, "parameters": { "max_tokens": 65536 } }, { "id": "MiniMaxAI/MiniMax-M2.7", "description": "Self-evolving 230B MoE agent for frontier coding, reasoning, and tool use." , "supportsReasoning": true, "supportsArtifacts": true, "parameters": { "max_tokens": 98304 } }, diff --git a/scripts/reasoning-replay-harness.ts b/scripts/reasoning-replay-harness.ts new file mode 100644 index 00000000000..8b3b6e00ba2 --- /dev/null +++ b/scripts/reasoning-replay-harness.ts @@ -0,0 +1,677 @@ +/** + * Compatibility + semantic + speed harness for the reasoning/tool-history + * replay changes. + * + * Sends synthetic conversations to a pinned cohort of router models (see + * PINNED_MODELS) in several shapes and compares acceptance, semantic + * correctness, and streaming speed: + * S1 baseline — flat {role, content} history (previous prod behavior) + * S2 replay — output of prepareMessagesWithFiles({replayToolHistory: true}) + * S3 in-loop — S2 with reasoning_content on a tool-call assistant message + * and `content` omitted (the shape runMcpFlow sends between + * tool rounds) + * P1/P2 — tool-less flow, flat vs reasoning_content attached + * N1/N2 — semantic proof (see below), not just shape acceptance + * + * N1/N2 close the harness's biggest evidentiary gap: S1-S3's stored tool + * result already restates the facts (Paris, 18°C) in the visible assistant + * answer, so a model can answer the follow-up correctly from the flat + * baseline without ever needing replayed history — shape acceptance was + * being mistaken for semantic proof. N1/N2 instead fabricate a tool result + * containing a nonce that appears NOWHERE in visible content, then ask for + * it: only a model that actually received the replayed tool history can + * answer, so N2-nonce-replay containing the nonce is direct proof replay + * works, not just that the provider accepted the payload shape. + * + * Requests stream (like prod) and are repeated REPS times sequentially per + * model/scenario, models in parallel, measuring time-to-first-token and + * generation throughput (approximated from SSE delta chunks). + * + * Ship criteria (both gate the exit code): + * 1. Every model that accepts a baseline scenario must also accept its + * dependent scenarios (see FAMILIES) — shape-acceptance regression. + * 2. Every model that accepts N2-nonce-replay must produce the nonce in + * its answer — semantic regression, independent of (1). + * + * Run it the same way as the other repo scripts (see "populate" in + * package.json), passing this file's path to vite-node. + */ +import { readFileSync } from "fs"; +import { prepareMessagesWithFiles } from "$lib/server/textGeneration/utils/prepareFiles"; +import type { EndpointMessage } from "$lib/server/endpoints/endpoints"; +import type { makeImageProcessor } from "$lib/server/endpoints/images"; +import { MessageToolUpdateType, MessageUpdateType } from "$lib/types/MessageUpdate"; +import { ToolResultStatus } from "$lib/types/Tool"; +import type { OpenAI } from "openai"; + +const REPS = 2; +const REQUEST_TIMEOUT_MS = 90_000; +// Reasoning models can spend 250-450+ tokens thinking before emitting any +// visible content, especially when the answer requires recalling a specific +// fact (the nonce scenarios). A tighter budget was observed cutting content +// to empty (finish_reason: "length") after reasoning alone exhausted it. +const MAX_TOKENS = 400; + +/** + * Pinned to the models this PR's research established a preservation policy + * for (see the PR body's vendor-guidance table), plus two controls, instead + * of "first N from /models": an unpinned population drifts across runs and + * says nothing about whether the models this PR actually targets behave. + * gemma-4-31B-it and Llama-3.1-8B-Instruct are controls — Gemma's vendor + * requires stripping historical thoughts (must stay unflagged, must NOT gain + * cross-turn reasoning_content), Llama has no reasoning mechanism at all. + */ +const PINNED_MODELS = [ + "moonshotai/Kimi-K3", + "moonshotai/Kimi-K2.7-Code", + "MiniMaxAI/MiniMax-M3", + "deepseek-ai/DeepSeek-V4-Flash", + "deepseek-ai/DeepSeek-V4-Pro", + "zai-org/GLM-5.2", + "Qwen/Qwen3.6-27B", + "Qwen/Qwen3.6-35B-A3B", + "google/gemma-4-31B-it", + "meta-llama/Llama-3.1-8B-Instruct", +]; + +/** + * Mirrors each pinned model's real `supportsReasoning` flag (chart/env/*.yaml) + * so the cross-turn scenarios (S2/P2/N2 below) build the same attachReasoning + * value production would actually resolve for that model. Building them with + * a hardcoded `true` for every model would send Gemma and Llama — both + * deliberately unflagged — a cross-turn reasoning_content shape production + * never sends them, which proves nothing about their real (correct) policy + * and could even fail them for the wrong reason. S3-inloop is exempt: the + * in-loop echo it models is evidence-based and ungated in production for + * every model (see runMcpFlow.ts), so it stays unconditioned here too. + */ +const MODEL_SUPPORTS_REASONING: Record = { + "moonshotai/Kimi-K3": true, + "moonshotai/Kimi-K2.7-Code": true, + "MiniMaxAI/MiniMax-M3": true, + "deepseek-ai/DeepSeek-V4-Flash": true, + "deepseek-ai/DeepSeek-V4-Pro": true, + "zai-org/GLM-5.2": true, + "Qwen/Qwen3.6-27B": true, + "Qwen/Qwen3.6-35B-A3B": true, + "google/gemma-4-31B-it": false, + "meta-llama/Llama-3.1-8B-Instruct": false, +}; + +type ChatMessage = OpenAI.Chat.Completions.ChatCompletionMessageParam & { + reasoning_content?: string; +}; + +function loadEnv(): { baseUrl: string; apiKey: string } { + const env = new Map(); + for (const file of [".env", ".env.local"]) { + try { + for (const line of readFileSync(file, "utf-8").split("\n")) { + const match = line.match(/^([A-Z0-9_]+)=(.*)$/); + if (match) env.set(match[1], match[2].replace(/^["']|["']$/g, "")); + } + } catch { + // file optional + } + } + const baseUrl = + process.env.OPENAI_BASE_URL ?? env.get("OPENAI_BASE_URL") ?? "https://router.huggingface.co/v1"; + const apiKey = process.env.OPENAI_API_KEY ?? env.get("OPENAI_API_KEY") ?? ""; + if (!apiKey) throw new Error("No OPENAI_API_KEY found in env or .env.local"); + return { baseUrl: baseUrl.replace(/\/$/, ""), apiKey }; +} + +const TOOLS = [ + { + type: "function" as const, + function: { + name: "get_weather", + description: "Get the current weather for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, + { + type: "function" as const, + function: { + name: "get_forecast", + description: "Get the 7-day forecast for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, +]; + +const SYSTEM = "You are a helpful assistant. Answer concisely."; +const THINK = + "The user wants current weather in Paris. I called get_weather which returned 18°C and sunny, then get_forecast which says sunny all week."; +const ASSISTANT_VISIBLE = + "It's currently 18°C and sunny in Paris, and the forecast says sunny all week."; +const FOLLOW_UP = + "In one short sentence: which city did I ask about, and what temperature did the weather tool report?"; + +/** The stored conversation, exactly as chat-ui would persist it. */ +const storedHistory: EndpointMessage[] = [ + { from: "user", content: "What's the weather in Paris right now? Also check the forecast." }, + { + from: "assistant", + content: THINK + ASSISTANT_VISIBLE, + updates: [ + { + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Call, + uuid: "call-1", + call: { name: "get_weather", parameters: { city: "Paris" } }, + }, + { + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Result, + uuid: "call-1", + result: { + status: ToolResultStatus.Success, + call: { name: "get_weather", parameters: { city: "Paris" } }, + outputs: [{ text: "18°C, sunny, humidity 60%" }], + }, + }, + { + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Call, + uuid: "call-2", + call: { name: "get_forecast", parameters: { city: "Paris" } }, + }, + { + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Result, + uuid: "call-2", + result: { + status: ToolResultStatus.Success, + call: { name: "get_forecast", parameters: { city: "Paris" } }, + outputs: [{ text: "Sunny all week, highs around 20°C" }], + }, + }, + ], + }, + { from: "user", content: FOLLOW_UP }, +]; + +const imageProcessor = (() => { + throw new Error("unused"); +}) as unknown as ReturnType; + +/** + * A nonce that appears ONLY in the fabricated tool result, never in the + * assistant's visible content — the semantic proof scenario's whole point. + * High-entropy enough that a model cannot plausibly guess or hallucinate it. + */ +const NONCE = "Q7M4-XP29"; + +/** + * Same shape as storedHistory, but the tool result carries a fact (the + * nonce) that the visible assistant answer never restates. Flat history + * drops the tool entirely, so a model can only produce the nonce in the + * follow-up if it actually saw replayed tool history — unlike storedHistory + * above, where the visible answer already gives away every fact the + * follow-up asks about. + * + * Framed as a weather station id, not "internal_reference": an early version + * used that field name and a safety-tuned model (Kimi-K3) correctly recalled + * it in its own reasoning_content but then refused to repeat it, reading + * "internal" as "not meant for the user" — a fixture-wording false negative, + * not a replay failure. A station id has no such ambiguity. + */ +const storedNonceHistory: EndpointMessage[] = [ + { from: "user", content: "What's the weather in Paris right now?" }, + { + from: "assistant", + content: "Checking the weather tool.It's sunny and mild in Paris right now.", + updates: [ + { + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Call, + uuid: "nonce-call-1", + call: { name: "get_weather", parameters: { city: "Paris" } }, + }, + { + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Result, + uuid: "nonce-call-1", + result: { + status: ToolResultStatus.Success, + call: { name: "get_weather", parameters: { city: "Paris" } }, + outputs: [{ text: `Sunny, 19°C. station_id=${NONCE}` }], + }, + }, + ], + }, + { + from: "user", + content: + "In one short sentence: what weather station ID did the tool report? It looked like XXXX-XXXX.", + }, +]; + +/** Tool-less two-turn conversation for the plain-flow scenarios. */ +const storedPlainHistory: EndpointMessage[] = [ + { from: "user", content: "If a train travels 120 km in 1.5 hours, what is its average speed?" }, + { + from: "assistant", + content: "120 divided by 1.5 is 80, so 80 km/h.The average speed is 80 km/h.", + }, + { + from: "user", + content: "In one short sentence: what speed did you compute in your previous answer?", + }, +]; + +type Scenario = { + name: string; + messages: ChatMessage[]; + withTools: boolean; + expect: RegExp[]; + /** + * Semantic gate on the final answer text, independent of `expect`: + * "must-contain" fails the scenario (regardless of HTTP-level success) if + * the nonce is absent; "must-not-contain" is informational only (logged, + * never gates) — a sanity check that the flat baseline truly has no way + * to know the nonce. + */ + nonceCheck?: "must-contain" | "must-not-contain"; +}; + +/** Baseline scenario name → scenarios that must not regress against it. */ +const FAMILIES: Record = { + "S1-baseline": ["S2-replay", "S3-inloop"], + "P1-plain": ["P2-reasoning"], +}; + +/** + * Builds the scenario set for one model. `supportsReasoning` should be that + * model's real MODEL_SUPPORTS_REASONING value: it decides attachReasoning for + * every cross-turn scenario (S2/P2/N2) the same way production resolves it, + * so an unflagged model is tested against the shape it will actually receive + * rather than one forced uniformly onto every model. S3-inloop is the one + * exception — see MODEL_SUPPORTS_REASONING's doc comment for why. + */ +async function buildScenarios(supportsReasoning: boolean): Promise { + const toolExpect = [/paris/i, /18/]; + const plainExpect = [/80/]; + const withSystem = (msgs: ChatMessage[]): ChatMessage[] => [ + { role: "system", content: SYSTEM }, + ...msgs, + ]; + + const baseline = withSystem(await prepareMessagesWithFiles(storedHistory, imageProcessor, false)); + const replay = withSystem( + await prepareMessagesWithFiles(storedHistory, imageProcessor, false, { + replayToolHistory: true, + attachReasoning: supportsReasoning, + }) + ); + // S3: attach reasoning_content to the first tool-call assistant message, + // mirroring what runMcpFlow sends between rounds of a live turn. Built + // from `replay` above, but only ever touches the tool-call message (never + // gated by attachReasoning either in this fixture or in production), so + // it's unaffected by whichever attachReasoning value built `replay`. + const inloop: ChatMessage[] = replay.map((m) => + m.role === "assistant" && "tool_calls" in m && m.tool_calls?.[0]?.id === "call10000" + ? { ...m, reasoning_content: "The user wants current weather, calling get_weather first." } + : m + ); + const plain = withSystem( + await prepareMessagesWithFiles(storedPlainHistory, imageProcessor, false) + ); + const plainReasoning = withSystem( + await prepareMessagesWithFiles(storedPlainHistory, imageProcessor, false, { + attachReasoning: supportsReasoning, + }) + ); + const nonceFlat = withSystem( + await prepareMessagesWithFiles(storedNonceHistory, imageProcessor, false) + ); + // The nonce itself lives in the replayed tool RESULT, which is always + // replayed unconditionally (tool replay is never gated by + // attachReasoning) — so this scenario's semantic proof is unaffected by + // supportsReasoning either way; it's threaded through purely so an + // unflagged model isn't sent an unrealistic reasoning_content alongside it. + const nonceReplay = withSystem( + await prepareMessagesWithFiles(storedNonceHistory, imageProcessor, false, { + replayToolHistory: true, + attachReasoning: supportsReasoning, + }) + ); + + return [ + { name: "S1-baseline", messages: baseline, withTools: true, expect: toolExpect }, + { name: "S2-replay", messages: replay, withTools: true, expect: toolExpect }, + { name: "S3-inloop", messages: inloop, withTools: true, expect: toolExpect }, + { name: "P1-plain", messages: plain, withTools: false, expect: plainExpect }, + { name: "P2-reasoning", messages: plainReasoning, withTools: false, expect: plainExpect }, + { + name: "N1-nonce-flat", + messages: nonceFlat, + withTools: true, + expect: [], + nonceCheck: "must-not-contain", + }, + { + name: "N2-nonce-replay", + messages: nonceReplay, + withTools: true, + expect: [], + nonceCheck: "must-contain", + }, + ]; +} + +type RunResult = { + ok: boolean; + ttftMs?: number; + totalMs: number; + genTokens: number; + coherent?: boolean; + /** Only set when scenario.nonceCheck is defined; see that field's doc. */ + nonceOk?: boolean; + note: string; +}; + +/** One streaming request; TTFT = first delta carrying content/reasoning/tool_calls. */ +async function runOne( + baseUrl: string, + apiKey: string, + model: string, + scenario: Scenario +): Promise { + const started = Date.now(); + try { + const res = await fetch(`${baseUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + "X-use-cache": "false", + }, + body: JSON.stringify({ + model, + messages: scenario.messages, + ...(scenario.withTools ? { tools: TOOLS, tool_choice: "auto" } : {}), + temperature: 0, + max_tokens: MAX_TOKENS, + stream: true, + }), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!res.ok || !res.body) { + const body = (await res.text()).slice(0, 300).replace(/\s+/g, " "); + return { + ok: false, + totalMs: Date.now() - started, + genTokens: 0, + note: `HTTP ${res.status}: ${body}`, + }; + } + + let ttftMs: number | undefined; + let genTokens = 0; + let content = ""; + let sawToolCall = false; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + for (const line of lines) { + if (!line.startsWith("data:")) continue; + const payload = line.slice(5).trim(); + if (!payload || payload === "[DONE]") continue; + let delta: Record | undefined; + try { + const parsed = JSON.parse(payload) as { + choices?: Array<{ delta?: Record }>; + }; + delta = parsed.choices?.[0]?.delta; + } catch { + continue; + } + if (!delta) continue; + const text = + (typeof delta.content === "string" ? delta.content : "") + + (typeof delta.reasoning === "string" ? delta.reasoning : "") + + (typeof delta.reasoning_content === "string" ? delta.reasoning_content : "") + + (typeof delta.reasoning_text === "string" ? delta.reasoning_text : ""); + const hasToolCall = Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0; + if (hasToolCall) sawToolCall = true; + if (text.length > 0 || hasToolCall) { + ttftMs ??= Date.now() - started; + genTokens += 1; + } + if (typeof delta.content === "string") content += delta.content; + } + } + const totalMs = Date.now() - started; + content = content.trim(); + if (!content && !sawToolCall && genTokens === 0) { + return { ok: false, ttftMs, totalMs, genTokens, note: "empty response" }; + } + const coherent = + content.length > 0 ? scenario.expect.every((re) => re.test(content)) : undefined; + // A tool-calls-only response (no text) never satisfies a nonce check: + // the semantic proof requires the fact to appear in the model's actual + // answer, not just that some request happened to succeed. + const nonceOk = scenario.nonceCheck + ? scenario.nonceCheck === "must-contain" + ? content.includes(NONCE) + : !content.includes(NONCE) + : undefined; + return { + ok: true, + ttftMs, + totalMs, + genTokens, + coherent, + nonceOk, + note: sawToolCall && !content ? "answered with tool_calls" : content.slice(0, 80), + }; + } catch (err) { + return { + ok: false, + totalMs: Date.now() - started, + genTokens: 0, + note: err instanceof Error ? `${err.name}: ${err.message.slice(0, 200)}` : String(err), + }; + } +} + +const median = (values: number[]): number | undefined => { + if (values.length === 0) return undefined; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +}; + +type ScenarioStats = { + scenario: string; + ok: boolean; + okCount: number; + repCount: number; + coherent?: boolean; + /** Only meaningful when the scenario has nonceCheck: "must-contain". */ + nonceOk?: boolean; + ttftMs?: number; + tokPerSec?: number; + totalMs?: number; + note: string; +}; + +async function runModel( + baseUrl: string, + apiKey: string, + model: string, + scenarios: Scenario[] +): Promise { + const stats: ScenarioStats[] = []; + for (const scenario of scenarios) { + const runs: RunResult[] = []; + for (let rep = 0; rep < REPS; rep += 1) { + runs.push(await runOne(baseUrl, apiKey, model, scenario)); + } + const okRuns = runs.filter((r) => r.ok); + const speedRuns = okRuns.filter((r) => r.ttftMs !== undefined && r.genTokens > 1); + // Require every repetition to pass: a scenario that succeeds once and + // fails once (e.g. a request landing on a provider variant with + // different schema validation) is a real half-failure, not a + // compatibility pass. okCount/repCount surface the partial case + // distinctly instead of rounding it up to "ok". + // nonceOk is likewise strict: every successful rep must satisfy the + // nonce check, not just one out of REPS. + const nonceRuns = okRuns.filter((r) => r.nonceOk !== undefined); + stats.push({ + scenario: scenario.name, + ok: okRuns.length === runs.length, + okCount: okRuns.length, + repCount: runs.length, + // Aggregate across every judged successful rep (like nonceOk): one + // incoherent rep is a sampled semantic failure even if another rep + // happened to answer well. + coherent: okRuns.some((r) => r.coherent !== undefined) + ? okRuns.every((r) => r.coherent !== false) + : undefined, + nonceOk: nonceRuns.length > 0 ? nonceRuns.every((r) => r.nonceOk) : undefined, + ttftMs: median(speedRuns.map((r) => r.ttftMs ?? 0)), + tokPerSec: median( + speedRuns.map( + (r) => (r.genTokens - 1) / Math.max(0.001, (r.totalMs - (r.ttftMs ?? 0)) / 1000) + ) + ), + totalMs: median(okRuns.map((r) => r.totalMs)), + note: (okRuns[0] ?? runs[0]).note, + }); + } + return stats; +} + +async function main() { + const { baseUrl, apiKey } = loadEnv(); + console.log(`Base URL: ${baseUrl}`); + + const modelsRes = await fetch(`${baseUrl}/models`, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + if (!modelsRes.ok) throw new Error(`GET /models failed: HTTP ${modelsRes.status}`); + const modelsJson = (await modelsRes.json()) as { data?: Array<{ id: string }> }; + const availableIds = new Set((modelsJson.data ?? []).map((m) => m.id)); + const models = PINNED_MODELS.filter((id) => availableIds.has(id)); + const missing = PINNED_MODELS.filter((id) => !availableIds.has(id)); + if (missing.length > 0) { + console.log(`Skipping pinned models no longer on the router: ${missing.join(", ")}`); + } + // An empty cohort (model churn, a misconfigured endpoint) would otherwise + // fall through to zero regressions/zero semantic failures and print + // SHIPPABLE without a single request sent — an indeterminate run must + // fail loudly, not look identical to a clean pass. + if (models.length === 0) { + console.error( + `No pinned models are available on the router (checked ${PINNED_MODELS.length}); nothing was tested.` + ); + process.exit(1); + } + + // Scenarios are built per model (attachReasoning depends on each model's + // real supportsReasoning flag — see buildScenarios's doc comment), so log + // representative payload sizes for both variants once instead of per model. + for (const supportsReasoning of [true, false]) { + const sample = await buildScenarios(supportsReasoning); + console.log(`--- payload sizes (supportsReasoning=${supportsReasoning}) ---`); + for (const scenario of sample) { + console.log(`${scenario.name}: payload ${JSON.stringify(scenario.messages).length} chars`); + } + } + console.log(`\nTesting ${models.length} models, ${REPS} reps per scenario:\n`); + + const results = await Promise.all( + models.map(async (model) => ({ + model, + stats: await runModel( + baseUrl, + apiKey, + model, + await buildScenarios(MODEL_SUPPORTS_REASONING[model] ?? false) + ), + })) + ); + + let regressions = 0; + let semanticFailures = 0; + const fmt = (n?: number) => (n === undefined ? " —" : String(Math.round(n)).padStart(4)); + console.log("\n=== RESULTS (median over reps; ttft ms | ~tok/s | total ms) ==="); + for (const { model, stats } of results) { + const regressed = Object.entries(FAMILIES).some(([base, dependents]) => { + const baseStat = stats.find((s) => s.scenario === base); + if (!baseStat?.ok) return false; + return dependents.some((name) => { + const dep = stats.find((s) => s.scenario === name); + if (!dep?.ok) return true; + // A dependent scenario that technically succeeded but answered + // incoherently, when the baseline it's compared against did not, + // is a real regression: the request was accepted but the replayed + // history degraded the answer. Only compares against a coherent + // baseline — an already-incoherent baseline says nothing about + // whether replay made things worse. + return baseStat.coherent !== false && dep.coherent === false; + }); + }); + // Semantic gate: N2-nonce-replay succeeding at the HTTP level is not + // enough — the nonce must actually appear in the answer, or replay + // isn't proven to work, only that the provider accepted the shape. + // nonceOk is independent of overall scenario.ok on purpose: runModel + // already scopes it to only the reps that succeeded (nonceRuns), so a + // rep that failed at the HTTP level (timeout, 5xx) must never mask a + // DIFFERENT rep that succeeded and directly demonstrated the nonce is + // missing — that observed failure is real signal either way. + // The scenario must ALSO have succeeded overall: if every rep timed out + // or errored, nonceRuns is empty and nonceOk is undefined (neither true + // nor false), which must not read as "no semantic failure" — the sole + // proof that replay works semantically was never obtained, so treat an + // unresolved nonce scenario as a failure too, not a silent pass. + const nonceScenario = stats.find((s) => s.scenario === "N2-nonce-replay"); + const semanticFailure = Boolean( + nonceScenario && (nonceScenario.nonceOk === false || !nonceScenario.ok) + ); + if (regressed) regressions += 1; + if (semanticFailure) semanticFailures += 1; + console.log(`${regressed || semanticFailure ? "❌" : "✅"} ${model}`); + for (const s of stats) { + const partial = !s.ok && s.okCount > 0; + const nonceFailed = s.nonceOk === false; + const status = !s.ok + ? `${partial ? `FLAKY(${s.okCount}/${s.repCount})` : "FAIL"}${nonceFailed ? ", no nonce" : ""}` + : nonceFailed + ? "OK(no nonce)" + : `OK${s.coherent === false ? "(incoherent)" : ""}`; + console.log( + ` ${s.scenario.padEnd(16)} ${status.padEnd(14)} ttft ${fmt(s.ttftMs)} | ${fmt( + s.tokPerSec + )} tok/s | total ${fmt(s.totalMs)}${s.ok && !nonceFailed ? "" : ` ${s.note}`}` + ); + } + } + + console.log( + `\n${regressions === 0 && semanticFailures === 0 ? "SHIPPABLE" : "NOT SHIPPABLE"}: ` + + `${regressions} model(s) shape-regressed, ${semanticFailures} model(s) failed the nonce semantic proof.` + ); + process.exit(regressions === 0 && semanticFailures === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/src/lib/server/endpoints/endpoints.ts b/src/lib/server/endpoints/endpoints.ts index 6a5cf20240f..0dbc0db3708 100644 --- a/src/lib/server/endpoints/endpoints.ts +++ b/src/lib/server/endpoints/endpoints.ts @@ -25,6 +25,8 @@ export interface EndpointParameters { provider?: string; /** Optional thinking-effort, forwarded as OpenAI `reasoning_effort` when set */ reasoningEffort?: "low" | "medium" | "high"; + /** Per-model user override for reasoning; wins over the model's supportsReasoning flag in both directions */ + reasoningOverride?: boolean; } export type TextGenerationStreamOutputSimplified = TextGenerationStreamOutput & { diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts index fcd5198a393..3259396a576 100644 --- a/src/lib/server/endpoints/openai/endpointOai.ts +++ b/src/lib/server/endpoints/openai/endpointOai.ts @@ -174,10 +174,26 @@ export async function endpointOai( abortSignal, provider, reasoningEffort, + reasoningOverride, }) => { - // Format messages for the chat API, handling multimodal content if supported + // Format messages for the chat API, handling multimodal content if supported. + // attachReasoning re-attaches persisted reasoning as reasoning_content on + // past assistant turns (preserved-thinking models condition on it). The + // per-user reasoning override wins in both directions, else the model's + // capability flag decides, mirroring reasoning_effort forwarding, so + // strict non-reasoning backends never see the extra field; tool replay + // stays off here since this path never declares tools. + // currentProducerModel is this call's own resolved model: when invoked + // directly for a pinned conversation it's the only model that has ever + // produced a turn here, and when invoked as a router candidate (the + // "omni" alias resolves a candidate before delegating here) it's that + // resolved candidate — either way it gates reasoning_content to history + // this same model actually produced. let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = - await prepareMessagesWithFiles(messages, imageProcessor, isMultimodal ?? model.multimodal); + await prepareMessagesWithFiles(messages, imageProcessor, isMultimodal ?? model.multimodal, { + attachReasoning: reasoningOverride ?? Boolean(model.supportsReasoning), + currentProducerModel: model.id ?? model.name, + }); // Normalize preprompt and handle empty values const normalizedPreprompt = typeof preprompt === "string" ? preprompt.trim() : ""; diff --git a/src/lib/server/endpoints/openai/openAIChatToTextGenerationStream.ts b/src/lib/server/endpoints/openai/openAIChatToTextGenerationStream.ts index 17ad14bc102..b89bb4b894a 100644 --- a/src/lib/server/endpoints/openai/openAIChatToTextGenerationStream.ts +++ b/src/lib/server/endpoints/openai/openAIChatToTextGenerationStream.ts @@ -53,6 +53,7 @@ export async function* openAIChatToTextGenerationStream( const delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta & { reasoning?: string; reasoning_content?: string; + reasoning_text?: string; } = choices?.[0]?.delta ?? {}; const content: string = delta.content ?? ""; const reasoning: string = @@ -60,7 +61,9 @@ export async function* openAIChatToTextGenerationStream( ? (delta.reasoning as string) : typeof delta?.reasoning_content === "string" ? (delta.reasoning_content as string) - : ""; + : typeof delta?.reasoning_text === "string" + ? (delta.reasoning_text as string) + : ""; const last = choices?.[0]?.finish_reason === "stop" || choices?.[0]?.finish_reason === "length"; // if the last token is a stop and the tool buffer is not empty, yield it as a generated_text @@ -174,6 +177,7 @@ export async function* openAIChatToTextGenerationSingle( const message: NonNullable["message"] & { reasoning?: string; reasoning_content?: string; + reasoning_text?: string; } = completion.choices?.[0]?.message ?? {}; let content: string = message?.content || ""; // Provider-dependent reasoning shapes (non-streaming) @@ -182,7 +186,9 @@ export async function* openAIChatToTextGenerationSingle( ? (message.reasoning as string) : typeof message?.reasoning_content === "string" ? (message.reasoning_content as string) - : ""; + : typeof message?.reasoning_text === "string" + ? (message.reasoning_text as string) + : ""; if (r && r.length > 0) { content = `${r}` + content; } diff --git a/src/lib/server/textGeneration/generate.ts b/src/lib/server/textGeneration/generate.ts index 909236f565b..c21c165781f 100644 --- a/src/lib/server/textGeneration/generate.ts +++ b/src/lib/server/textGeneration/generate.ts @@ -24,6 +24,7 @@ export async function* generate( forceMultimodal, provider, reasoningEffort, + reasoningOverride, locals, abortController, }: GenerateContext, @@ -63,6 +64,7 @@ export async function* generate( abortSignal: abortController.signal, provider, reasoningEffort, + reasoningOverride, }); for await (const output of stream) { diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index b49d3304163..a8e3b54facf 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -78,6 +78,7 @@ async function* textGenerationWithoutTitle( forceTools: ctx.forceTools, provider: ctx.provider, reasoningEffort: ctx.reasoningEffort, + reasoningOverride: ctx.reasoningOverride, locals: ctx.locals, preprompt, abortSignal: ctx.abortController.signal, diff --git a/src/lib/server/textGeneration/mcp/runMcpFlow.ts b/src/lib/server/textGeneration/mcp/runMcpFlow.ts index d06ba348a6e..73ec83f2b57 100644 --- a/src/lib/server/textGeneration/mcp/runMcpFlow.ts +++ b/src/lib/server/textGeneration/mcp/runMcpFlow.ts @@ -37,6 +37,7 @@ export type RunMcpFlowContext = Pick< | "forceTools" | "provider" | "reasoningEffort" + | "reasoningOverride" | "locals" > & { messages: EndpointMessage[] }; @@ -58,6 +59,7 @@ export async function* runMcpFlow({ forceTools, provider, reasoningEffort, + reasoningOverride, locals, preprompt, abortSignal, @@ -357,7 +359,21 @@ export async function* runMcpFlow({ let messagesOpenAI: ChatCompletionMessageParam[] = await prepareMessagesWithFiles( messages, imageProcessor, - mmEnabled + mmEnabled, + { + replayToolHistory: true, + // Cross-turn reasoning echo: the per-user override wins in both + // directions, else the capability flag decides. The in-loop echo + // below stays evidence-based (the model just emitted it). + attachReasoning: + reasoningOverride ?? + Boolean((targetModel as unknown as { supportsReasoning?: boolean }).supportsReasoning), + // The model resolved for THIS turn. Under the "omni" router alias a + // prior turn in the same conversation can have been produced by a + // different model (per-message routing, no user action needed); this + // gates reasoning_content to only replay onto its own producer. + currentProducerModel: candidateModelId ?? targetModel.id ?? targetModel.name, + } ); const userTimezone = (locals as unknown as { timezone?: string })?.timezone; const toolPreprompt = buildToolPreprompt(oaTools, userTimezone); @@ -455,6 +471,11 @@ export async function* runMcpFlow({ // Track whether we're inside a block when the upstream streams // provider-specific reasoning tokens (e.g. `reasoning` or `reasoning_content`). let thinkOpen = false; + // Leading whitespace-only reasoning deltas that arrived before the block + // opened (thinkOpen still false, so a blank chunk wouldn't otherwise open + // one). Held here and flushed once a non-blank delta opens the block, so + // the persisted trace stays byte-exact instead of silently dropping them. + let pendingReasoningWhitespace = ""; let truncatedToolCallRetries = 0; if (resolvedRoute && candidateModelId) { @@ -478,6 +499,9 @@ export async function* runMcpFlow({ lastAssistantContent = ""; streamedContent = false; + // Discard any whitespace-only reasoning buffered but never flushed by a + // non-blank delta last round — it never became part of a real trace. + pendingReasoningWhitespace = ""; const completionRequest: ChatCompletionCreateParamsStreaming = { ...completionBase, @@ -572,23 +596,40 @@ export async function* runMcpFlow({ return ""; })(); - // Provider-dependent reasoning fields (e.g., `reasoning` or `reasoning_content`). + // Provider-dependent reasoning fields (`reasoning`, `reasoning_content`, + // or `reasoning_text`). + const deltaFields = delta as unknown as { + reasoning?: unknown; + reasoning_content?: unknown; + reasoning_text?: unknown; + }; const deltaReasoning: string = - typeof (delta as unknown as Record)?.reasoning === "string" - ? ((delta as unknown as { reasoning?: string }).reasoning as string) - : typeof (delta as unknown as Record)?.reasoning_content === "string" - ? ((delta as unknown as { reasoning_content?: string }).reasoning_content as string) - : ""; + typeof deltaFields?.reasoning === "string" + ? deltaFields.reasoning + : typeof deltaFields?.reasoning_content === "string" + ? deltaFields.reasoning_content + : typeof deltaFields?.reasoning_text === "string" + ? deltaFields.reasoning_text + : ""; // Merge reasoning + content into a single combined token stream, mirroring // the OpenAI adapter so the UI can auto-detect blocks. let combined = ""; - if (deltaReasoning.trim().length > 0) { - if (!thinkOpen) { - combined += "" + deltaReasoning; + // Whitespace-only deltas still count once a think block is open + // (paragraph breaks are part of the byte-exact trace); non-blank + // text is only required to OPEN a block, so stray leading + // whitespace can't create empty think blocks on its own — but it + // must not be discarded either, so it's buffered until a non-blank + // delta arrives and flushed into the opening of the block. + if (deltaReasoning.length > 0) { + if (thinkOpen) { + combined += deltaReasoning; + } else if (deltaReasoning.trim().length > 0) { + combined += "" + pendingReasoningWhitespace + deltaReasoning; + pendingReasoningWhitespace = ""; thinkOpen = true; } else { - combined += deltaReasoning; + pendingReasoningWhitespace += deltaReasoning; } } @@ -720,16 +761,36 @@ export async function* runMcpFlow({ function: { name: call.name, arguments: call.arguments }, })); - // Avoid sending content back to the model alongside tool_calls - // to prevent confusing follow-up reasoning. Strip any think blocks. + // Move content out of `content` and echo it back as + // `reasoning_content`: preserved-thinking models (e.g. Kimi K2/K3) + // condition their next tool round on prior reasoning and degrade + // when it's dropped; other providers ignore the field. + const thinkParts: string[] = []; const assistantContentForToolMsg = lastAssistantContent.replace( - /[\s\S]*?(?:<\/think>|$)/g, - "" + /([\s\S]*?)(?:<\/think>|$)/g, + (_match, inner: string) => { + thinkParts.push(inner); + return ""; + } ); - const assistantToolMessage: ChatCompletionMessageParam = { + // Trim only to TEST for emptiness — the joined value itself must stay + // byte-exact once it's echoed back and persisted: vendors documenting + // preserved thinking (e.g. Z.ai's "must return the complete, + // unmodified reasoning_content") can condition on or cache against the + // exact bytes, so stripping whitespace here would send a corrupted + // trace on the next round/turn. + const reasoningForToolMsg = thinkParts.join("\n"); + // Omit `content` entirely when nothing visible remains — some + // OpenAI-compatible backends 400 on empty text next to tool_calls. + const assistantToolMessage: ChatCompletionMessageParam & { reasoning_content?: string } = { role: "assistant", - content: assistantContentForToolMsg, tool_calls: toolCalls, + ...(assistantContentForToolMsg.trim().length > 0 + ? { content: assistantContentForToolMsg } + : {}), + ...(reasoningForToolMsg.trim().length > 0 + ? { reasoning_content: reasoningForToolMsg } + : {}), }; const exec = executeToolCalls({ @@ -741,6 +802,11 @@ export async function* runMcpFlow({ toPrimitive, processToolOutput, abortSignal, + // Persisted on the round's first Call update so history replay + // can re-attach this round's reasoning and preamble text to its + // own message instead of moving them onto the final answer. + roundReasoning: reasoningForToolMsg, + roundContent: assistantContentForToolMsg, }); let toolMsgCount = 0; let toolRunCount = 0; diff --git a/src/lib/server/textGeneration/mcp/toolInvocation.spec.ts b/src/lib/server/textGeneration/mcp/toolInvocation.spec.ts index 79f7b280411..1d72a6c6237 100644 --- a/src/lib/server/textGeneration/mcp/toolInvocation.spec.ts +++ b/src/lib/server/textGeneration/mcp/toolInvocation.spec.ts @@ -23,7 +23,7 @@ vi.mock("../../logger", () => ({ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, })); -const { executeToolCalls } = await import("./toolInvocation"); +const { executeToolCalls, isValidJsonObject } = await import("./toolInvocation"); const SERVERS = [{ name: "hf", url: "https://example.test/mcp" }]; const MAPPING = { do_thing: { fnName: "do_thing", server: "hf", tool: "do_thing" } }; @@ -189,3 +189,28 @@ describe("executeToolCalls argument handling", () => { expect(toolMessages[1].content).toBe("ok"); }); }); + +describe("isValidJsonObject", () => { + it("accepts a well-formed JSON object", () => { + expect(isValidJsonObject('{"city":"Paris"}')).toBe(true); + expect(isValidJsonObject("{}")).toBe(true); + }); + + it("rejects malformed or truncated JSON", () => { + // The exact failure mode this guards: a model streams a truncated + // arguments string, which must never be persisted as argumentsRaw and + // later replayed as an invalid historical tool_calls.function.arguments. + expect(isValidJsonObject('{"city":"Pari')).toBe(false); + expect(isValidJsonObject("")).toBe(false); + expect(isValidJsonObject("not json at all")).toBe(false); + }); + + it("rejects valid JSON that isn't an object", () => { + // Tool-call arguments must be an object; arrays/primitives/null are + // syntactically valid JSON but never a valid arguments shape. + expect(isValidJsonObject("[1,2,3]")).toBe(false); + expect(isValidJsonObject("null")).toBe(false); + expect(isValidJsonObject('"a string"')).toBe(false); + expect(isValidJsonObject("42")).toBe(false); + }); +}); diff --git a/src/lib/server/textGeneration/mcp/toolInvocation.ts b/src/lib/server/textGeneration/mcp/toolInvocation.ts index 05e15bf56fc..5d4de4223d0 100644 --- a/src/lib/server/textGeneration/mcp/toolInvocation.ts +++ b/src/lib/server/textGeneration/mcp/toolInvocation.ts @@ -43,6 +43,10 @@ export interface ExecuteToolCallsParams { }; abortSignal?: AbortSignal; toolTimeoutMs?: number; + /** Reasoning that led to this round of calls; persisted on the round's first Call update. */ + roundReasoning?: string; + /** Visible text streamed before this round's calls; persisted on the round's first Call update. */ + roundContent?: string; } export interface ToolCallExecutionResult { @@ -55,6 +59,24 @@ export type ToolExecutionEvent = | { type: "update"; update: MessageUpdate } | { type: "complete"; summary: ToolCallExecutionResult }; +/** + * Whether a string is valid, parseable JSON encoding an object. Guards + * argumentsRaw persistence: a model can stream a truncated or otherwise + * malformed `arguments` string, which `parseArgs` already tolerates for the + * live tool call (falling back to `{}`), but persisting that malformed + * string as argumentsRaw would later replay invalid JSON as a historical + * tool_calls.function.arguments — some providers validate that field and + * would reject the whole continuation, not just this one call. + */ +export function isValidJsonObject(raw: string): boolean { + try { + const parsed: unknown = JSON.parse(raw); + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed); + } catch { + return false; + } +} + const serverMap = (servers: McpServerConfig[]): Map => { const map = new Map(); for (const server of servers) { @@ -75,6 +97,8 @@ export async function* executeToolCalls({ processToolOutput, abortSignal, toolTimeoutMs, + roundReasoning, + roundContent, }: ExecuteToolCallsParams): AsyncGenerator { const effectiveTimeoutMs = toolTimeoutMs ?? getMcpToolTimeoutMs(); const toolMessages: ChatCompletionMessageParam[] = []; @@ -106,7 +130,7 @@ export async function* executeToolCalls({ return { call, argsObj, paramsClean, uuid: randomUUID() }; }); - for (const p of prepared) { + for (const [index, p] of prepared.entries()) { yield { type: "update", update: { @@ -114,6 +138,17 @@ export async function* executeToolCalls({ subtype: MessageToolUpdateType.Call, uuid: p.uuid, call: { name: p.call.name, parameters: p.paramsClean }, + ...(p.call.id?.trim() ? { originalId: p.call.id } : {}), + ...(p.call.arguments?.trim() && isValidJsonObject(p.call.arguments) + ? { argumentsRaw: p.call.arguments } + : {}), + ...(index === 0 && roundReasoning?.trim() ? { reasoning: roundReasoning } : {}), + // Preamble text is trimmed (unlike reasoning, which stays + // byte-exact): replay compares it against the trim-normalized + // visible text from splitReasoning, so persisting leading + // whitespace would break the dedup match and duplicate the + // preamble in replayed history. + ...(index === 0 && roundContent?.trim() ? { content: roundContent.trim() } : {}), }, }; yield { diff --git a/src/lib/server/textGeneration/types.ts b/src/lib/server/textGeneration/types.ts index e71daabe3dd..43467d568e6 100644 --- a/src/lib/server/textGeneration/types.ts +++ b/src/lib/server/textGeneration/types.ts @@ -21,6 +21,8 @@ export interface TextGenerationContext { provider?: string; /** Optional thinking-effort override forwarded as `reasoning_effort` to OpenAI-compatible endpoints */ reasoningEffort?: "low" | "medium" | "high"; + /** Per-model user override for reasoning; wins over the model's supportsReasoning flag in both directions */ + reasoningOverride?: boolean; /** Per-model user override for artifacts; wins over the model's supportsArtifacts flag in both directions */ artifactsOverride?: boolean; locals: App.Locals | undefined; diff --git a/src/lib/server/textGeneration/utils/prepareFiles.spec.ts b/src/lib/server/textGeneration/utils/prepareFiles.spec.ts new file mode 100644 index 00000000000..4f6ca16a1b4 --- /dev/null +++ b/src/lib/server/textGeneration/utils/prepareFiles.spec.ts @@ -0,0 +1,832 @@ +import { describe, expect, it } from "vitest"; +import { prepareMessagesWithFiles } from "./prepareFiles"; +import type { EndpointMessage } from "$lib/server/endpoints/endpoints"; +import type { makeImageProcessor } from "$lib/server/endpoints/images"; +import { + MessageToolUpdateType, + MessageUpdateType, + type MessageUpdate, +} from "$lib/types/MessageUpdate"; +import { ToolResultStatus } from "$lib/types/Tool"; + +const imageProcessor = (() => { + throw new Error("imageProcessor should not be called in these tests"); +}) as unknown as ReturnType; + +const callUpdate = (uuid: string, name: string, parameters: Record) => + ({ + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Call, + uuid, + call: { name, parameters }, + }) satisfies MessageUpdate; + +const resultUpdate = (uuid: string, name: string, text: string) => + ({ + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Result, + uuid, + result: { + status: ToolResultStatus.Success, + call: { name, parameters: {} }, + outputs: [{ text }], + }, + }) satisfies MessageUpdate; + +const errorUpdate = (uuid: string, message: string) => + ({ + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Error, + uuid, + message, + }) satisfies MessageUpdate; + +describe("prepareMessagesWithFiles tool history replay", () => { + it("keeps flat {role, content} messages when replay is off", async () => { + const messages: EndpointMessage[] = [ + { from: "user", content: "hi" }, + { + from: "assistant", + content: "hello", + updates: [callUpdate("u1", "get_weather", { city: "Paris" })], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false); + expect(prepared).toEqual([ + { role: "user", content: "hi" }, + { role: "assistant", content: "hello" }, + ]); + }); + + it("expands tool rounds into assistant/tool pairs and splits reasoning", async () => { + const messages: EndpointMessage[] = [ + { from: "user", content: "weather in Paris?" }, + { + from: "assistant", + content: "need the toolIt is 18°C and sunny in Paris.", + updates: [ + callUpdate("u1", "get_weather", { city: "Paris" }), + resultUpdate("u1", "get_weather", "18°C, sunny"), + callUpdate("u2", "get_forecast", { city: "Paris" }), + resultUpdate("u2", "get_forecast", "sunny all week"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + + expect(prepared).toEqual([ + { role: "user", content: "weather in Paris?" }, + { + role: "assistant", + tool_calls: [ + { + id: "u10000000", + type: "function", + function: { name: "get_weather", arguments: JSON.stringify({ city: "Paris" }) }, + }, + ], + }, + { role: "tool", tool_call_id: "u10000000", content: "18°C, sunny" }, + { + role: "assistant", + tool_calls: [ + { + id: "u20000000", + type: "function", + function: { name: "get_forecast", arguments: JSON.stringify({ city: "Paris" }) }, + }, + ], + }, + { role: "tool", tool_call_id: "u20000000", content: "sunny all week" }, + { + role: "assistant", + content: "It is 18°C and sunny in Paris.", + reasoning_content: "need the tool", + }, + ]); + // Mistral-family templates require exactly nine alphanumeric chars + for (const m of prepared) { + if (m.role === "tool") expect(m.tool_call_id).toMatch(/^[a-zA-Z0-9]{9}$/); + } + // tool-call messages must not carry a content key at all + const withToolCalls = prepared.filter((m) => "tool_calls" in m); + for (const message of withToolCalls) { + expect("content" in message).toBe(false); + } + }); + + it("groups parallel calls of one round into a single assistant message", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "done", + updates: [ + callUpdate("a", "search", { q: "x" }), + callUpdate("b", "search", { q: "y" }), + resultUpdate("a", "search", "res-a"), + errorUpdate("b", "timeout"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared).toEqual([ + { + role: "assistant", + tool_calls: [ + { + id: "a00000000", + type: "function", + function: { name: "search", arguments: '{"q":"x"}' }, + }, + { + id: "b00000000", + type: "function", + function: { name: "search", arguments: '{"q":"y"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "a00000000", content: "res-a" }, + { role: "tool", tool_call_id: "b00000000", content: "Error: timeout" }, + { role: "assistant", content: "done" }, + ]); + }); + + it("marks calls without a persisted outcome as interrupted instead of empty success", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "", + updates: [callUpdate("u1", "get_weather", { city: "Paris" })], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared[1]).toEqual({ + role: "tool", + tool_call_id: "u10000000", + content: "Error: interrupted before a result was recorded", + }); + }); + + it("omits the trailing assistant message entirely when a turn was interrupted before any final text or reasoning", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "", + updates: [ + callUpdate("u1", "get_weather", { city: "Paris" }), + resultUpdate("u1", "get_weather", "18°C"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + // No trailing { role: "assistant", content: "" } — just the tool round. + expect(prepared).toEqual([ + { + role: "assistant", + tool_calls: [ + { + id: "u10000000", + type: "function", + function: { name: "get_weather", arguments: JSON.stringify({ city: "Paris" }) }, + }, + ], + }, + { role: "tool", tool_call_id: "u10000000", content: "18°C" }, + ]); + }); + + it("omits an all-empty plain assistant message (no tool calls, no text, no reasoning) entirely", async () => { + const messages: EndpointMessage[] = [ + { from: "user", content: "hi" }, + { from: "assistant", content: "" }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared).toEqual([{ role: "user", content: "hi" }]); + }); + + it("degrades the oldest turns to flat messages once the replay budget is spent", async () => { + // Each turn carries ~7×8k of tool output, so two turns exceed the 100k + // budget: the newest keeps its tool history, the oldest goes flat. + const bigTurn = (prefix: string): EndpointMessage => ({ + from: "assistant", + content: `${prefix} done`, + updates: Array.from({ length: 7 }, (_, i) => [ + callUpdate(`${prefix}${i}`, "search", { q: String(i) }), + resultUpdate(`${prefix}${i}`, "search", "x".repeat(8000)), + ]).flat(), + }); + const messages: EndpointMessage[] = [ + bigTurn("old"), + { from: "user", content: "next" }, + bigTurn("new"), + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared[0]).toEqual({ role: "assistant", content: "old done" }); + expect(prepared[1]).toEqual({ role: "user", content: "next" }); + expect(prepared.filter((m) => m.role === "tool")).toHaveLength(7); + }); + + it("dedups a round preamble persisted with leading whitespace", async () => { + // The pre-tool stream often starts with newlines after a think block; a + // Call update persisted untrimmed must still match the trim-normalized + // visible text, or the preamble replays twice. + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "plan\n\nLet me check that.\n\nHere is the answer.", + updates: [ + { + ...callUpdate("u1", "get_weather", { city: "Paris" }), + reasoning: "plan", + content: "\n\nLet me check that.", + }, + resultUpdate("u1", "get_weather", "18°C"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const roundMessage = prepared[0] as { content?: string }; + const finalMessage = prepared.at(-1) as { content?: string }; + expect(roundMessage.content).toBe("Let me check that."); + expect(finalMessage.content).toContain("Here is the answer."); + expect(finalMessage.content).not.toContain("Let me check that."); + }); + + it("re-attaches persisted round reasoning to its own tool-call message", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: + "round one reasoningfinal reasoningIt is 18°C in Paris.", + updates: [ + { + ...callUpdate("u1", "get_weather", { city: "Paris" }), + reasoning: "round one reasoning", + }, + resultUpdate("u1", "get_weather", "18°C"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared).toEqual([ + { + role: "assistant", + tool_calls: [ + { + id: "u10000000", + type: "function", + function: { name: "get_weather", arguments: '{"city":"Paris"}' }, + }, + ], + reasoning_content: "round one reasoning", + }, + { role: "tool", tool_call_id: "u10000000", content: "18°C" }, + { + role: "assistant", + content: "It is 18°C in Paris.", + reasoning_content: "final reasoning", + }, + ]); + }); + + it("does not delete an unrelated final reasoning block during round dedup, only the positionally-matching one", async () => { + // The round's persisted reasoning ("Need weather forecast") doesn't + // exactly match either extracted block, so exact match misses. + // It DOES contain "Need weather" (the round's own, positionally-first, + // mismatched-by-formatting block) as a substring, but it also contains + // "weather" (the unrelated final block) as a substring — only the first + // must be removed; the final block must survive. + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "Need weatherweatherFinal text.", + updates: [ + { + ...callUpdate("u1", "get_weather", { city: "Paris" }), + reasoning: "Need weather forecast", + }, + resultUpdate("u1", "get_weather", "18°C"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const finalMessage = prepared.at(-1) as { content?: string; reasoning_content?: string }; + expect(finalMessage.content).toBe("Final text."); + expect(finalMessage.reasoning_content).toBe("weather"); + }); + + it("never expands an older turn when a newer turn already fell back to flat", async () => { + // The newest turn alone exceeds the 100k budget, so it goes flat; the + // older turn must then go flat too, even though it would fit on its own. + const turn = (prefix: string, calls: number): EndpointMessage => ({ + from: "assistant", + content: `${prefix} done`, + updates: Array.from({ length: calls }, (_, i) => [ + callUpdate(`${prefix}${i}`, "search", { q: String(i) }), + resultUpdate(`${prefix}${i}`, "search", "x".repeat(8000)), + ]).flat(), + }); + const messages: EndpointMessage[] = [ + turn("old", 2), + { from: "user", content: "next" }, + turn("new", 14), + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared).toEqual([ + { role: "assistant", content: "old done" }, + { role: "user", content: "next" }, + { role: "assistant", content: "new done" }, + ]); + }); + + it("omits an empty flat fallback once the budget is exhausted", async () => { + // The newest turn alone exceeds the budget, forcing every turn flat; + // the older turn was interrupted before any visible text, so its flat + // shape would be {role: assistant, content: ""} — it must be omitted + // like the replay and plain branches already do, not emitted. + const interrupted: EndpointMessage = { + from: "assistant", + content: "", + updates: [callUpdate("i1", "search", { q: "x" })], + }; + const huge: EndpointMessage = { + from: "assistant", + content: "new done", + updates: Array.from({ length: 14 }, (_, i) => [ + callUpdate(`new${i}`, "search", { q: String(i) }), + resultUpdate(`new${i}`, "search", "x".repeat(8000)), + ]).flat(), + }; + const messages: EndpointMessage[] = [interrupted, { from: "user", content: "next" }, huge]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared).toEqual([ + { role: "user", content: "next" }, + { role: "assistant", content: "new done" }, + ]); + }); + + it("does not pull final-answer text matching an unstreamed preamble before the tools", async () => { + // A preamble persisted on the Call update but never merged into stored + // content (it arrived in the same delta as the first tool_calls entry) + // is not a prefix of the visible text. Identical text inside the final + // answer must stay where it is — mild duplication is acceptable, + // reordering the conversation is not. + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "The answer is 42. Let me check. Done.", + updates: [ + { ...callUpdate("u1", "search", { q: "x" }), content: "Let me check." }, + resultUpdate("u1", "search", "42"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared[0]).toMatchObject({ role: "assistant", content: "Let me check." }); + expect(prepared.at(-1)).toEqual({ + role: "assistant", + content: "The answer is 42. Let me check. Done.", + }); + }); + + it("attachReasoning splits reasoning out but never emits tool messages", async () => { + const messages: EndpointMessage[] = [ + { from: "user", content: "hi" }, + { + from: "assistant", + content: "inline partfinal answer", + reasoning: "stored part", + updates: [ + callUpdate("u1", "get_weather", { city: "Paris" }), + resultUpdate("u1", "get_weather", "18°C"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: true, + }); + expect(prepared).toEqual([ + { role: "user", content: "hi" }, + { + role: "assistant", + content: "final answer", + reasoning_content: "stored part\ninline part", + }, + ]); + }); + + it("attachReasoning leaves assistant turns without reasoning untouched", async () => { + const messages: EndpointMessage[] = [{ from: "assistant", content: "plain answer" }]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: true, + }); + expect(prepared).toEqual([{ role: "assistant", content: "plain answer" }]); + expect("reasoning_content" in prepared[0]).toBe(false); + }); + + it("omits an interrupted reasoning-only turn entirely when attachReasoning is off, instead of a phantom empty message", async () => { + // content is only a think block (no visible text ever streamed) and + // attachReasoning is false, so wantsReasoning is false: the plain + // {role: assistant, content: visible} path would otherwise emit an + // empty-content message with nothing else attached. + const messages: EndpointMessage[] = [ + { from: "user", content: "hi" }, + { from: "assistant", content: "only reasoning, no answer" }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: false, + }); + expect(prepared).toEqual([{ role: "user", content: "hi" }]); + }); + + it("attachReasoning spends the same replay budget, oldest turns first, without leaking in the fallback", async () => { + // Two turns of ~60k reasoning exceed the 100k budget: the newest keeps + // reasoning_content, the oldest falls back to a -stripped flat + // shape (not the raw string) so models that must never see historical + // thoughts (e.g. Gemma) don't get them just because the budget ran out. + const bigReasoningTurn = (n: number): EndpointMessage => ({ + from: "assistant", + content: `${"x".repeat(60_000)}answer ${n}`, + }); + const messages: EndpointMessage[] = [bigReasoningTurn(1), bigReasoningTurn(2)]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: true, + }); + expect("reasoning_content" in prepared[0]).toBe(false); + expect(prepared[0]).toEqual({ role: "assistant", content: "answer 1" }); + expect(prepared[1]).toMatchObject({ role: "assistant", content: "answer 2" }); + expect("reasoning_content" in prepared[1]).toBe(true); + }); + + it("replayToolHistory with attachReasoning disabled keeps tool pairs but drops reasoning", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "secret chaindone", + updates: [callUpdate("u1", "search", { q: "x" }), resultUpdate("u1", "search", "res")], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + attachReasoning: false, + }); + expect(prepared.filter((m) => m.role === "tool")).toHaveLength(1); + expect(prepared.at(-1)).toEqual({ role: "assistant", content: "done" }); + }); + + it("keeps reasoning bytes exact (no trimming) while still dropping whitespace-only parts", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: " leading and trailing space done", + updates: [], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + // the whitespace-only second block is dropped, but the first block's + // surrounding spaces survive verbatim in the echoed value + expect(prepared).toEqual([ + { + role: "assistant", + content: "done", + reasoning_content: " leading and trailing space ", + }, + ]); + }); + + it("keeps a round's preamble text on its own tool-call message instead of the final answer", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + // message.content is purely the model's own streamed tokens (round + // preamble(s) + final answer), never the tool's own output text. + content: "Let me check that.It is 18°C and sunny in Paris.", + updates: [ + { + ...callUpdate("u1", "get_weather", { city: "Paris" }), + content: "Let me check that.", + }, + resultUpdate("u1", "get_weather", "18°C, sunny"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared).toEqual([ + { + role: "assistant", + tool_calls: [ + { + id: "u10000000", + type: "function", + function: { name: "get_weather", arguments: '{"city":"Paris"}' }, + }, + ], + content: "Let me check that.", + }, + { role: "tool", tool_call_id: "u10000000", content: "18°C, sunny" }, + { role: "assistant", content: "It is 18°C and sunny in Paris." }, + ]); + }); + + it("omits content on the tool-call message when no preamble was persisted (pre-existing messages)", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "It is 18°C and sunny in Paris.", + updates: [ + callUpdate("u1", "get_weather", { city: "Paris" }), + resultUpdate("u1", "get_weather", "18°C"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const toolCallMsg = prepared.find((m) => "tool_calls" in m); + expect(toolCallMsg && "content" in toolCallMsg).toBe(false); + }); + + it("strips from the replayToolHistory budget fallback instead of leaking it raw", async () => { + const bigTurn = (n: number): EndpointMessage => ({ + from: "assistant", + content: `${"x".repeat(60_000)}answer ${n}`, + updates: [ + callUpdate(`c${n}`, "search", { q: String(n) }), + resultUpdate(`c${n}`, "search", "x".repeat(60_000)), + ], + }); + const messages: EndpointMessage[] = [bigTurn(1), bigTurn(2)]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + // turn 1 (oldest) fell back to flat because turn 2 alone (~68k of + // reasoning + capped tool output) already spends most of the 100k + // budget: it must be plain content with no reasoning_content leaking + // through, and no raw tag either. + const flatCandidates = prepared.filter( + (m) => m.role === "assistant" && !("tool_calls" in m) && !("reasoning_content" in m) + ); + expect(flatCandidates).toHaveLength(1); + expect(flatCandidates[0].content).toBe("answer 1"); + expect(flatCandidates[0].content).not.toContain(""); + }); + + it("attaches persisted message.reasoning alongside extracted think blocks", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "inline partfinal answer", + reasoning: "stored part", + updates: [], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + expect(prepared).toEqual([ + { + role: "assistant", + content: "final answer", + reasoning_content: "stored part\ninline part", + }, + ]); + }); + + it("strips historical content even with attachReasoning disabled (Gemma-style models)", async () => { + // Some vendors (Gemma) document that historical thoughts must be + // stripped across completed turns. Previously, attachReasoning:false + // fell through to raw message.content, leaking inline text. + const messages: EndpointMessage[] = [ + { from: "assistant", content: "private prior thoughtFinal answer" }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: false, + }); + expect(prepared).toEqual([{ role: "assistant", content: "Final answer" }]); + }); + + it("strips historical content with no options passed at all", async () => { + const messages: EndpointMessage[] = [ + { from: "assistant", content: "thoughtdone" }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false); + expect(prepared).toEqual([{ role: "assistant", content: "done" }]); + }); + + it("strips an empty block even when nothing survives to attach as reasoning", async () => { + const messages: EndpointMessage[] = [{ from: "assistant", content: "Hello" }]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: true, + }); + expect(prepared).toEqual([{ role: "assistant", content: "Hello" }]); + }); + + it("suppresses reasoning_content for a message produced by a different router-resolved model", async () => { + // Under the "omni" router alias, each turn can be produced by a + // different model. Reasoning is conditioned on its own producer, so a + // message routed to model A must not have its reasoning replayed when + // the current turn targets model B. + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "model A's private reasoninganswer from A", + routerMetadata: { route: "r", model: "model-a" }, + }, + ]; + const suppressed = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: true, + currentProducerModel: "model-b", + }); + expect(suppressed).toEqual([{ role: "assistant", content: "answer from A" }]); + + const allowed = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: true, + currentProducerModel: "model-a", + }); + expect(allowed).toEqual([ + { + role: "assistant", + content: "answer from A", + reasoning_content: "model A's private reasoning", + }, + ]); + }); + + it("treats a message with no routerMetadata as same-producer (the common pinned-model case)", async () => { + const messages: EndpointMessage[] = [ + { from: "assistant", content: "reasoninganswer" }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + attachReasoning: true, + currentProducerModel: "any-model", + }); + expect(prepared).toEqual([ + { role: "assistant", content: "answer", reasoning_content: "reasoning" }, + ]); + }); + + it("gates replayed tool-round reasoning by producer, but always replays tool calls/results", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "done", + routerMetadata: { route: "r", model: "model-a" }, + updates: [ + { + ...callUpdate("u1", "get_weather", { city: "Paris" }), + reasoning: "model A reasoning", + }, + resultUpdate("u1", "get_weather", "18°C"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + currentProducerModel: "model-b", + }); + // Tool calls and results are protocol-neutral and always replay. + expect(prepared).toEqual([ + { + role: "assistant", + tool_calls: [ + { + id: "u10000000", + type: "function", + function: { name: "get_weather", arguments: '{"city":"Paris"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "u10000000", content: "18°C" }, + { role: "assistant", content: "done" }, + ]); + // No reasoning_content anywhere, since the producer doesn't match. + expect(prepared.some((m) => "reasoning_content" in m)).toBe(false); + }); + + it("replays the persisted raw arguments string instead of reserializing sanitized parameters", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "done", + updates: [ + { + ...callUpdate("u1", "search", { query: "x" }), + argumentsRaw: '{"query":{"city":"Paris","units":"metric"},"images":["image_1"]}', + }, + resultUpdate("u1", "search", "res"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const callMessage = prepared[0] as { tool_calls?: Array<{ function: { arguments: string } }> }; + expect(callMessage.tool_calls?.[0]?.function.arguments).toBe( + '{"query":{"city":"Paris","units":"metric"},"images":["image_1"]}' + ); + }); + + it("falls back to sanitized parameters when argumentsRaw was not persisted (legacy messages)", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "done", + updates: [callUpdate("u1", "search", { q: "x" }), resultUpdate("u1", "search", "res")], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const callMessage = prepared[0] as { tool_calls?: Array<{ function: { arguments: string } }> }; + expect(callMessage.tool_calls?.[0]?.function.arguments).toBe('{"q":"x"}'); + }); + + it("falls back to sanitized parameters when a persisted argumentsRaw is not valid JSON", async () => { + // toolInvocation.ts already guards this at write time, but replay must + // never trust a persisted argumentsRaw blindly at its own read + // boundary — defense in depth against a future write path or + // otherwise-corrupted data. Invalid JSON here must never reach the + // outgoing tool_calls.function.arguments, since a provider that + // validates that field could reject the whole continuation. + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "done", + updates: [ + { ...callUpdate("u1", "search", { q: "x" }), argumentsRaw: '{"q":"unterminated' }, + resultUpdate("u1", "search", "res"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const callMessage = prepared[0] as { tool_calls?: Array<{ function: { arguments: string } }> }; + expect(callMessage.tool_calls?.[0]?.function.arguments).toBe('{"q":"x"}'); + expect(() => JSON.parse(callMessage.tool_calls?.[0]?.function.arguments ?? "")).not.toThrow(); + }); + + it("falls back to sanitized parameters when a persisted argumentsRaw is valid JSON but not an object", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "done", + updates: [ + { ...callUpdate("u1", "search", { q: "x" }), argumentsRaw: "[1,2,3]" }, + resultUpdate("u1", "search", "res"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const callMessage = prepared[0] as { tool_calls?: Array<{ function: { arguments: string } }> }; + expect(callMessage.tool_calls?.[0]?.function.arguments).toBe('{"q":"x"}'); + }); + + it("still emits the normalized tool_call_id even when the original provider id is persisted", async () => { + const messages: EndpointMessage[] = [ + { + from: "assistant", + content: "done", + updates: [ + { ...callUpdate("u1", "search", { q: "x" }), originalId: "call_abc123XYZ" }, + resultUpdate("u1", "search", "res"), + ], + }, + ]; + const prepared = await prepareMessagesWithFiles(messages, imageProcessor, false, { + replayToolHistory: true, + }); + const callMessage = prepared[0] as { tool_calls?: Array<{ id: string }> }; + expect(callMessage.tool_calls?.[0]?.id).toBe("u10000000"); + }); +}); diff --git a/src/lib/server/textGeneration/utils/prepareFiles.ts b/src/lib/server/textGeneration/utils/prepareFiles.ts index bc2a2260b06..927e5804123 100644 --- a/src/lib/server/textGeneration/utils/prepareFiles.ts +++ b/src/lib/server/textGeneration/utils/prepareFiles.ts @@ -3,20 +3,331 @@ import type { EndpointMessage } from "$lib/server/endpoints/endpoints"; import type { OpenAI } from "openai"; import { TEXT_MIME_ALLOWLIST } from "$lib/constants/mime"; import type { makeImageProcessor } from "$lib/server/endpoints/images"; +import { + MessageToolUpdateType, + MessageUpdateType, + type MessageToolCallUpdate, + type MessageToolErrorUpdate, + type MessageToolResultUpdate, + type MessageUpdate, +} from "$lib/types/MessageUpdate"; +import { ToolResultStatus } from "$lib/types/Tool"; +import { isValidJsonObject } from "$lib/server/textGeneration/mcp/toolInvocation"; + +type ChatMessageParam = OpenAI.Chat.Completions.ChatCompletionMessageParam; + +/** + * Assistant message extended with the reasoning echo field consumed by + * preserved-thinking backends (e.g. Kimi K2/K3). Providers that don't know the + * field ignore it. + */ +type AssistantReplayMessage = ChatMessageParam & { reasoning_content?: string }; + +/** Cap replayed tool outputs so old turns can't flood the context window. */ +const MAX_REPLAYED_TOOL_OUTPUT_CHARS = 8000; + +/** + * Cumulative cap on the expanded replay payload across the whole history. + * Turns are budgeted newest-first; older turns that don't fit fall back to + * the flat {role, content} shape the request used before replay existed, so a + * long tool-heavy conversation can't outgrow a context window it used to fit. + */ +const REPLAY_HISTORY_BUDGET_CHARS = 100_000; + +/** + * Normalize a persisted update uuid into a provider-safe tool_call_id. + * Mistral-family chat templates require exactly nine alphanumeric characters, + * a shape every other provider also accepts; the persisted uuid is only a + * correlation key, so the id just has to pair calls with results consistently. + */ +function toToolCallId(uuid: string, used: Set): string { + const alnum = uuid.replace(/[^a-zA-Z0-9]/g, "") || "toolcall0"; + let candidate = (alnum + "0".repeat(9)).slice(0, 9); + for (let salt = 1; used.has(candidate); salt += 1) { + const suffix = String(salt); + candidate = (alnum + "0".repeat(9)).slice(0, 9 - suffix.length) + suffix; + } + used.add(candidate); + return candidate; +} + +const isToolCallUpdate = (u: MessageUpdate): u is MessageToolCallUpdate => + u.type === MessageUpdateType.Tool && u.subtype === MessageToolUpdateType.Call; +const isToolResultUpdate = (u: MessageUpdate): u is MessageToolResultUpdate => + u.type === MessageUpdateType.Tool && u.subtype === MessageToolUpdateType.Result; +const isToolErrorUpdate = (u: MessageUpdate): u is MessageToolErrorUpdate => + u.type === MessageUpdateType.Tool && u.subtype === MessageToolUpdateType.Error; + +/** + * Whether a historical message's own producer (its persisted + * `routerMetadata.model`, set for messages generated through the "omni" + * router alias) matches the model about to consume the request. A message + * with no routerMetadata was produced by whatever single model the + * conversation is pinned to — the common case — and is always treated as + * same-producer since there is nothing to contradict. Reasoning is + * conditioned on the producing model's own prior thoughts; attaching one + * model's reasoning_content to a different model's turn is unverified and + * skipped rather than risked. Tool call/result replay is protocol-neutral + * (a `tool` message is just data) and is never gated by this. + */ +function reasoningProducerMatches( + message: EndpointMessage, + currentProducerModel?: string +): boolean { + const producer = message.routerMetadata?.model; + return !producer || !currentProducerModel || producer === currentProducerModel; +} + +/** + * Split `` blocks out of assistant text, merging them with the + * separately persisted `message.reasoning` when present. Parts are returned + * individually so replay can pair round reasoning back to its tool round. + * Parts are filtered by whether they're non-blank but kept byte-exact + * otherwise: vendors documenting preserved thinking can require the + * reasoning payload sent back unmodified, so trimming must only decide + * whether a part counts as empty, never change what gets echoed. + */ +function splitReasoning( + content: string, + storedReasoning?: string +): { visible: string; parts: string[] } { + const thinkParts: string[] = []; + const visible = content.replace(/([\s\S]*?)(?:<\/think>|$)/g, (_match, inner: string) => { + thinkParts.push(inner); + return ""; + }); + const parts = [storedReasoning ?? "", ...thinkParts].filter((part) => part.trim().length > 0); + return { visible: visible.trim(), parts }; +} + +/** Strip `` blocks without collecting them; used for the degraded + * fallback shape so a budget cutoff never leaks raw reasoning as visible + * content, including to models whose vendor requires historical thoughts + * to be stripped (e.g. Gemma) regardless of the replay budget. */ +function stripThink(content: string): string { + return content.replace(/[\s\S]*?(?:<\/think>|$)/g, "").trim(); +} + +/** + * Rebuild a past assistant turn from its persisted tool updates so follow-up + * requests see the tool calls and their outputs instead of a flat text + * summary. Rounds are inferred from update order — a Call update arriving + * after any Result/Error starts a new round, matching how the live loop emits + * one batch of calls per completion round. Each call's `tool_call_id` is + * always the normalized id from toToolCallId, even though the original + * provider-issued id may also be persisted (see MessageToolCallUpdate): + * emitting it unconditionally keeps one code path and satisfies every + * provider's id-shape requirements, including Mistral-family templates. + */ +function replayAssistantTurn( + message: EndpointMessage, + includeReasoning: boolean +): AssistantReplayMessage[] { + const updates = message.updates ?? []; + const { visible, parts } = splitReasoning(message.content, message.reasoning); + // `parts` holds every recovered reasoning block across the whole turn, and + // `remainingVisible` the full visible text: when tools ran, the + // FinalAnswer handler merges the pre-tool stream into content, so earlier + // rounds' think blocks and preamble text survive there too, concatenated + // in chronological order. Rounds whose Call update persisted its own + // `reasoning`/`content` (written by the live loop since those fields + // existed) reclaim their piece below; whatever remains belongs to the + // final answer. + const remainingParts = [...parts]; + let remainingVisible = visible; + // null, not an empty-content message, when a turn was interrupted before + // producing any final text or reasoning (e.g. aborted mid-tool-call): an + // empty trailing `{role: "assistant", content: ""}` with nothing else + // attached represents an assistant turn that never happened, and strict + // providers can reject it outright. + const buildFinalMessage = (): AssistantReplayMessage | null => { + const reasoning = remainingParts.join("\n"); + const content = remainingVisible.trim(); + const hasReasoning = includeReasoning && reasoning.length > 0; + if (content.length === 0 && !hasReasoning) return null; + return { + role: "assistant", + content, + ...(hasReasoning ? { reasoning_content: reasoning } : {}), + }; + }; + + const callUpdates = updates.filter(isToolCallUpdate); + if (callUpdates.length === 0) { + const finalMessage = buildFinalMessage(); + return finalMessage ? [finalMessage] : []; + } + + const outputsByUuid = new Map(); + const rounds: MessageToolCallUpdate[][] = []; + let round: MessageToolCallUpdate[] = []; + let roundHasOutcome = false; + for (const update of updates) { + if (isToolCallUpdate(update)) { + if (roundHasOutcome && round.length > 0) { + rounds.push(round); + round = []; + roundHasOutcome = false; + } + round.push(update); + } else if (isToolResultUpdate(update)) { + const result = update.result; + const firstOutput = + result.status === ToolResultStatus.Success ? result.outputs[0] : undefined; + outputsByUuid.set( + update.uuid, + result.status === ToolResultStatus.Success + ? typeof firstOutput?.text === "string" + ? firstOutput.text + : JSON.stringify(firstOutput ?? "") + : `Error: ${result.message}` + ); + roundHasOutcome = true; + } else if (isToolErrorUpdate(update)) { + outputsByUuid.set(update.uuid, `Error: ${update.message}`); + roundHasOutcome = true; + } + } + if (round.length > 0) { + rounds.push(round); + } + + const usedIds = new Set(); + const idByUuid = new Map(callUpdates.map((u) => [u.uuid, toToolCallId(u.uuid, usedIds)])); + + const replayed: AssistantReplayMessage[] = []; + for (const callsInRound of rounds) { + // Reasoning and preamble text persisted on the round's Call update go + // back on this round's message and are deduped out of the final + // answer's blocks. Kept byte-exact (trim only tests for emptiness): see + // splitReasoning for why reasoning fidelity matters; the same + // unmodified-echo principle is applied to preamble text for symmetry + // and so the dedup below matches reliably. + const roundReasoning = includeReasoning + ? (callsInRound.find((u) => u.reasoning?.trim())?.reasoning ?? "") + : ""; + if (roundReasoning) { + const exact = remainingParts.indexOf(roundReasoning); + if (exact !== -1) { + remainingParts.splice(exact, 1); + } else if (remainingParts.length > 0 && roundReasoning.includes(remainingParts[0])) { + // Positional fallback only, never a scan of the whole array: parts + // are chronologically ordered and rounds are processed oldest-first, + // so the earliest still-unconsumed part is the only one that can be + // attributed to this round when exact match fails (e.g. formatting + // drift). A LATER part merely being a substring of this round's + // reasoning is coincidence, not evidence it belongs here — removing + // it would silently delete an unrelated (and possibly the final + // answer's own) reasoning block. + remainingParts.splice(0, 1); + } + } + // Visible text streamed before this round's calls (e.g. "Let me check + // that."): rounds are processed oldest-first, matching the + // chronological order text was concatenated into `message.content`, so + // removing a matched prefix keeps the remainder correctly ordered for + // the final message. Trimmed on both sides (persistence trims too) + // because `remainingVisible` comes from splitReasoning trim-normalized; + // visible text has no byte-exactness requirement, unlike reasoning. + const roundContent = (callsInRound.find((u) => u.content?.trim())?.content ?? "").trim(); + if (roundContent && remainingVisible.startsWith(roundContent)) { + // Prefix-only, deliberately: rounds consume the visible text in + // chronological order, so a streamed preamble is always the next + // prefix. A persisted preamble that is NOT a prefix was never merged + // into stored content (content arriving in the same delta as the + // first tool_calls entry is suppressed from the stream), so a deeper + // indexOf match could only hit identical text belonging to the final + // answer — removing that would reorder the conversation. The failure + // mode of not matching is mild duplication, which is safer. + remainingVisible = remainingVisible.slice(roundContent.length).trimStart(); + } + // `content` is included only when a preamble was actually persisted + // (messages recorded before this field existed have none); omitted + // otherwise since some OpenAI-compatible backends reject empty text + // next to tool_calls with a 400. + // Arguments prefer the persisted raw JSON string the model actually + // sent (argumentsRaw): the sanitized fallback only keeps top-level + // primitive params (nested values and file payloads are deliberately + // kept out of storage), so it can under-represent the real call. + // Legacy updates without argumentsRaw, and any argumentsRaw that + // somehow isn't valid JSON (toolInvocation.ts already guards this at + // write time, but a replayed value must never be trusted blindly at + // its own read boundary — belt and suspenders against a future write + // path or manipulated data), fall back to the sanitized form. + replayed.push({ + role: "assistant", + tool_calls: callsInRound.map((u) => ({ + id: idByUuid.get(u.uuid) ?? u.uuid, + type: "function" as const, + function: { + name: u.call.name, + arguments: + u.argumentsRaw && isValidJsonObject(u.argumentsRaw) + ? u.argumentsRaw + : JSON.stringify(u.call.parameters ?? {}), + }, + })), + ...(roundContent.trim().length > 0 ? { content: roundContent } : {}), + ...(roundReasoning ? { reasoning_content: roundReasoning } : {}), + }); + for (const u of callsInRound) { + // A call with no persisted outcome means the run was aborted + // mid-execution; say so instead of fabricating an empty success. + const output = outputsByUuid.has(u.uuid) + ? (outputsByUuid.get(u.uuid) ?? "") + : "Error: interrupted before a result was recorded"; + replayed.push({ + role: "tool", + tool_call_id: idByUuid.get(u.uuid) ?? u.uuid, + content: + output.length > MAX_REPLAYED_TOOL_OUTPUT_CHARS + ? output.slice(0, MAX_REPLAYED_TOOL_OUTPUT_CHARS) + "\n[...truncated]" + : output, + }); + } + } + const finalMessage = buildFinalMessage(); + if (finalMessage) replayed.push(finalMessage); + return replayed; +} /** * Prepare chat messages for OpenAI-compatible multimodal payloads. * - Processes images via the provided imageProcessor (resize/convert) when multimodal is enabled. * - Injects text-file content into the user message text. * - Leaves messages untouched when no files or multimodal disabled. + * - Historical assistant `` blocks are always stripped from outgoing + * content, whether or not any reasoning option below is set: raw think + * markup must never be replayed as visible text to any model. + * - With `replayToolHistory`, expands past assistant turns into their + * assistant/tool message pairs (from persisted updates) and re-attaches + * reasoning as `reasoning_content` instead of inline `` text. + * - With `attachReasoning`, only the reasoning half: assistant turns stay + * flat but carry `reasoning_content`, for tool-less requests where replayed + * tool messages would be undefined behavior (no `tools` in the request). + * Callers gate it on the model's reasoning capability; with + * `replayToolHistory` it defaults to on unless explicitly disabled. + * - `currentProducerModel` additionally gates reasoning_content per message: + * a message routed (via the router alias) to a different model than the + * one about to consume this request has its reasoning suppressed, since + * reasoning is conditioned on the producing model's own prior thoughts. + * Messages with no routerMetadata (the common pinned-model case) are + * unaffected. Tool call/result replay is protocol-neutral and always + * proceeds regardless of producer. */ export async function prepareMessagesWithFiles( messages: EndpointMessage[], imageProcessor: ReturnType, - isMultimodal: boolean + isMultimodal: boolean, + options?: { + replayToolHistory?: boolean; + attachReasoning?: boolean; + currentProducerModel?: string; + } ): Promise { - return Promise.all( - messages.map(async (message) => { + type ReplayCandidate = { replay: AssistantReplayMessage[]; flat: ChatMessageParam }; + const prepared = await Promise.all( + messages.map(async (message): Promise => { if (message.from === "user" && message.files && message.files.length > 0) { const { imageParts, textContent } = await prepareFiles( imageProcessor, @@ -31,14 +342,87 @@ export async function prepareMessagesWithFiles( if (imageParts.length > 0 && isMultimodal) { const parts = [{ type: "text" as const, text: messageText }, ...imageParts]; - return { role: message.from, content: parts }; + return [{ role: message.from, content: parts }]; } - return { role: message.from, content: messageText }; + return [{ role: message.from, content: messageText }]; } - return { role: message.from, content: message.content }; + if (message.from === "assistant") { + const wantsReasoning = + (options?.replayToolHistory + ? (options?.attachReasoning ?? true) + : Boolean(options?.attachReasoning)) && + reasoningProducerMatches(message, options?.currentProducerModel); + + if (options?.replayToolHistory) { + // The budget-fallback `flat` must still strip , not just + // drop the reasoning_content/tool-replay enrichments: the raw + // string leaks reasoning as visible content to every model that + // falls back to it, including ones (e.g. Gemma) whose vendor + // requires historical thoughts to be stripped regardless of the + // replay budget. + const flat: ChatMessageParam = { + role: "assistant", + content: stripThink(message.content), + }; + return { + replay: replayAssistantTurn(message, wantsReasoning), + flat, + }; + } + const { visible, parts } = splitReasoning(message.content, message.reasoning); + const reasoning = parts.join("\n"); + if (!wantsReasoning || reasoning.length === 0) { + // Either nothing to attach, or attachment is disabled/gated: + // either way `visible` (think-stripped) is the correct shape, + // never the raw `message.content` — but a turn interrupted + // before any visible text (or one whose only content was + // reasoning this call is gated from attaching) must not replay + // as a phantom `{role: assistant, content: ""}` with nothing + // else attached; omit it entirely instead. + return visible.length > 0 ? [{ role: "assistant", content: visible }] : []; + } + // Candidate, not a plain array, so the reasoning payload goes + // through the same newest-first budget as tool replay. The + // fallback keeps the same stripped `visible` text and just + // drops reasoning_content, for the same reason as above. + return { + replay: [{ role: "assistant", content: visible, reasoning_content: reasoning }], + flat: { role: "assistant", content: visible }, + }; + } + return [{ role: message.from, content: message.content }]; }) ); + + // Spend the replay budget newest-first so recent turns keep their full + // tool history and older ones degrade to the pre-replay flat shape. The + // degradation is monotonic: once any turn falls back to flat, every older + // turn does too, so the model never sees rich history for a stale turn + // while the turn it is continuing from is plain prose. + let budget = REPLAY_HISTORY_BUDGET_CHARS; + let exhausted = false; + const resolved: ChatMessageParam[][] = new Array(prepared.length); + for (let i = prepared.length - 1; i >= 0; i -= 1) { + const entry = prepared[i]; + if (Array.isArray(entry)) { + resolved[i] = entry; + continue; + } + const cost = JSON.stringify(entry.replay).length; + if (!exhausted && cost <= budget) { + budget -= cost; + resolved[i] = entry.replay; + } else { + exhausted = true; + // Same phantom-turn guard as the replay and plain branches: an + // interrupted turn whose stripped content is empty must be omitted, + // not sent as {role: "assistant", content: ""}. + const flatContent = typeof entry.flat.content === "string" ? entry.flat.content : ""; + resolved[i] = flatContent.trim().length > 0 ? [entry.flat] : []; + } + } + return resolved.flat(); } async function prepareFiles( diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts index 15041c4031a..cfe5d0441f3 100644 --- a/src/lib/types/MessageUpdate.ts +++ b/src/lib/types/MessageUpdate.ts @@ -65,6 +65,36 @@ interface MessageToolUpdateBase { export interface MessageToolCallUpdate extends MessageToolUpdateBase { call: ToolCall; + /** + * Reasoning that led to this round of calls (set on the round's first call + * update). Lets history replay re-attach reasoning to the right assistant + * message; absent on messages persisted before this field existed. + */ + reasoning?: string; + /** + * Visible text the model streamed before this round's tool calls (set on + * the round's first call update), e.g. "Let me check that." Lets history + * replay keep the preamble on its own round's assistant message instead of + * moving it after the tool results; absent on messages persisted before + * this field existed. + */ + content?: string; + /** + * Original provider-issued tool_call id and raw JSON arguments string, as + * sent by the model (set on every Call update; argumentsRaw only when it + * validates as JSON — a malformed string is never persisted here, since + * replaying invalid JSON in a historical tool call could get the whole + * continuation rejected by providers that validate the field). Replay + * uses argumentsRaw when present for byte-accurate arguments instead of + * reserializing the sanitized primitive parameters; the emitted + * tool_call_id is always the + * normalized one regardless (see toToolCallId in prepareFiles.ts), so + * originalId is captured for future fidelity but not replayed as-is. + * Absent on messages persisted before this field existed, or if the + * provider's response omitted an id. + */ + originalId?: string; + argumentsRaw?: string; } export interface MessageToolResultUpdate extends MessageToolUpdateBase { diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 03a1be786cd..9b653a7b936 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -703,6 +703,7 @@ export async function POST({ request, locals, params, getClientAddress }) { (userSettings?.reasoningOverrides?.[model.id] ?? model.supportsReasoning) ? userSettings?.reasoningEffortOverrides?.[model.id] : undefined, + reasoningOverride: userSettings?.reasoningOverrides?.[model.id], // Artifacts aren't provider-determined, so the per-model user // override applies on HuggingChat too artifactsOverride: userSettings?.artifactsOverrides?.[model.id], @@ -865,6 +866,36 @@ export async function PATCH({ request, locals, params }) { ...(values.model !== undefined && { model: values.model }), }; + // Switching the pinned model (e.g. the retired-model recovery banner) means + // every prior assistant message was actually produced by the OLD model, not + // the new one. Those messages have no routerMetadata.model of their own — + // it's only ever stamped for the "omni" router alias — so history-replay's + // same-producer check would otherwise default to treating them as + // same-producer as the newly selected model and attach the old model's + // reasoning_content to a turn it never produced. Backfill the retiring + // model's id onto messages that don't already carry producer metadata so + // the mismatch is recorded before it's lost. + if (values.model !== undefined && values.model !== conv.model) { + const retiredModel = conv.model; + const messagesForSave = conv.messages.map((msg) => + msg.from === "assistant" && !msg.routerMetadata?.model + ? { + ...msg, + routerMetadata: { + route: msg.routerMetadata?.route ?? "", + model: retiredModel, + provider: msg.routerMetadata?.provider, + }, + } + : msg + ); + await collections.conversations.updateOne( + { _id: convId }, + { $set: { ...updateValues, messages: messagesForSave } } + ); + return new Response(); + } + await collections.conversations.updateOne( { _id: convId,