Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 56 additions & 1 deletion src/proxy.tool-forwarding.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,9 @@ describe("tool forwarding", () => {
body: JSON.stringify({
model: "moonshot/kimi-k2.6",
stream: true,
messages: [{ role: "user", content: "What time is it in Chicago right now? Use the tool." }],
messages: [
{ role: "user", content: "What time is it in Chicago right now? Use the tool." },
],
tools: [
{
type: "function",
Expand Down Expand Up @@ -286,4 +288,57 @@ describe("tool forwarding", () => {
);
expect(finishReasons).toContain("tool_calls");
});

it("suppresses assistant content when upstream marks finish_reason as tool_calls", async () => {
upstreamResponse = {
id: "chatcmpl-tool-finish-reason",
object: "chat.completion",
created: Math.floor(Date.now() / 1000),
model: "moonshot/kimi-k2.6",
choices: [
{
index: 0,
message: {
role: "assistant",
content: "I should look up Barcelona's next match before replying.",
},
finish_reason: "tool_calls",
},
],
usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
};

const res = await fetch(`${proxy.baseUrl}/v1/chat/completions`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model: "moonshot/kimi-k2.6",
stream: false,
messages: [{ role: "user", content: "When is Barcelona's next match?" }],
tools: [
{
type: "function",
function: {
name: "web_search",
description: "Search the web",
parameters: { type: "object" },
},
},
],
}),
});

expect(res.status).toBe(200);
const json = (await res.json()) as {
choices?: Array<{
finish_reason?: string | null;
message?: {
content?: string;
tool_calls?: unknown[];
};
}>;
};
expect(json.choices?.[0]?.finish_reason).toBe("tool_calls");
expect(json.choices?.[0]?.message?.content).toBe("");
});
});
19 changes: 14 additions & 5 deletions src/proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5069,14 +5069,19 @@ async function proxyRequest(
// Process each choice (usually just one)
if (rsp.choices && Array.isArray(rsp.choices)) {
for (const choice of rsp.choices) {
const endsWithToolCalls = choice.finish_reason === "tool_calls";
// Some OpenAI-compatible providers include planning prose in content
// alongside tool_calls. Tool execution only needs tool_calls, so do
// not forward that prose to chat channels.
// alongside tool_calls, or mark the turn as a tool-call turn via
// finish_reason before exposing the tool_calls array at the same
// object shape. Tool execution only needs tool_calls, so do not
// forward that prose to chat channels.
const toolCalls = choice.message?.tool_calls ?? choice.delta?.tool_calls;
// Strip thinking tokens (Kimi <|...|> and standard <think> tags)
const rawContent = choice.message?.content ?? choice.delta?.content ?? "";
const content =
toolCalls && toolCalls.length > 0 ? "" : stripThinkingTokens(rawContent);
endsWithToolCalls || (toolCalls && toolCalls.length > 0)
? ""
: stripThinkingTokens(rawContent);
const role = choice.message?.role ?? choice.delta?.role ?? "assistant";
const index = choice.index ?? 0;

Expand Down Expand Up @@ -5170,7 +5175,7 @@ async function proxyRequest(
delta: {},
logprobs: null,
finish_reason:
toolCalls && toolCalls.length > 0
endsWithToolCalls || (toolCalls && toolCalls.length > 0)
? "tool_calls"
: (choice.finish_reason ?? "stop"),
},
Expand Down Expand Up @@ -5300,6 +5305,7 @@ async function proxyRequest(
try {
const parsed = JSON.parse(responseBody.toString()) as {
choices?: Array<{
finish_reason?: string | null;
message?: {
content?: string;
tool_calls?: unknown[];
Expand All @@ -5311,7 +5317,10 @@ async function proxyRequest(
const message = choice.message;
if (!message || typeof message.content !== "string") continue;

if (Array.isArray(message.tool_calls) && message.tool_calls.length > 0) {
if (
choice.finish_reason === "tool_calls" ||
(Array.isArray(message.tool_calls) && message.tool_calls.length > 0)
) {
if (message.content !== "") {
message.content = "";
changed = true;
Expand Down
Loading