From beca83482df87ed411e34a254bb3b5c8a719a02f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Jul 2026 04:05:34 -0400 Subject: [PATCH] fix(gateway): separate streamed chat completion finish chunks (#115718) Emit every assistant content chunk with a null finish reason and rely on the canonical empty terminal choice for normal and error streams. Refs: #83203 Co-authored-by: niuma996 <244209349+niuma996@users.noreply.github.com> --- src/gateway/openai-http.test.ts | 72 ++++++++++++++++++++++++++++++--- src/gateway/openai-http.ts | 9 +---- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/gateway/openai-http.test.ts b/src/gateway/openai-http.test.ts index 3de4163d1a46..e75dd56a3160 100644 --- a/src/gateway/openai-http.test.ts +++ b/src/gateway/openai-http.test.ts @@ -2010,6 +2010,49 @@ describe("OpenAI-compatible HTTP API (e2e)", () => { expect(finishChoice?.finish_reason).toBe("stop"); }); + it.each([ + { name: "successful completion", fail: false, expected: "hello" }, + { name: "internal agent error", fail: true, expected: "Error: internal error" }, + ])( + "separates streamed content from the terminal finish for an official SDK $name", + async ({ fail, expected }) => { + agentCommand.mockClear(); + if (fail) { + agentCommand.mockRejectedValueOnce(new Error("private upstream failure")); + } else { + agentCommand.mockImplementationOnce((async (opts: unknown) => + buildAssistantDeltaResult({ + opts, + emit: emitAgentEvent, + deltas: ["he", "llo"], + text: expected, + })) as never); + } + + const stream = await createOpenAiChatClient(enabledPort).chat.completions.create({ + model: "openclaw", + messages: [{ role: "user", content: "Return a complete streamed response." }], + stream: true, + }); + const choices: Array<{ + delta: { content?: string | null }; + finish_reason: string | null; + }> = []; + for await (const chunk of stream) { + choices.push(...chunk.choices); + } + + const contentChoices = choices.filter((choice) => typeof choice.delta.content === "string"); + expect(contentChoices.map((choice) => choice.delta.content).join("")).toBe(expected); + expect(contentChoices.every((choice) => choice.finish_reason === null)).toBe(true); + + const terminalChoices = choices.filter((choice) => choice.finish_reason === "stop"); + expect(terminalChoices).toHaveLength(1); + expect(terminalChoices[0]?.delta).toEqual({}); + expect(choices.at(-1)).toEqual(terminalChoices[0]); + }, + ); + it("streams SSE chunks when stream=true", async () => { const port = enabledPort; try { @@ -2045,6 +2088,18 @@ describe("OpenAI-compatible HTTP API (e2e)", () => { .filter((v): v is string => typeof v === "string") .join(""); expect(allContent).toBe("hello"); + const contentChoices = jsonChunks + .flatMap((chunk) => (chunk.choices as Array> | undefined) ?? []) + .filter( + (choice) => + typeof (choice.delta as Record | undefined)?.content === "string", + ); + expect(contentChoices.every((choice) => choice.finish_reason === null)).toBe(true); + const stopChoices = jsonChunks + .flatMap((chunk) => (chunk.choices as Array> | undefined) ?? []) + .filter((choice) => choice.finish_reason === "stop"); + expect(stopChoices).toHaveLength(1); + expect(stopChoices[0]?.delta).toEqual({}); const usageChunks = jsonChunks.filter((c) => "usage" in c); expect(usageChunks).toHaveLength(0); } @@ -2383,12 +2438,19 @@ describe("OpenAI-compatible HTTP API (e2e)", () => { const errorChunks = errorData .filter((d) => d !== "[DONE]") .map((d) => JSON.parse(d) as Record); - const stopChoice = errorChunks - .flatMap((c) => (c.choices as Array> | undefined) ?? []) - .find((choice) => choice.finish_reason === "stop"); - expect((stopChoice?.delta as Record | undefined)?.content).toBe( - "Error: internal error", + const choices = errorChunks.flatMap( + (chunk) => (chunk.choices as Array> | undefined) ?? [], ); + const errorContentChoice = choices.find( + (choice) => + (choice.delta as Record | undefined)?.content === + "Error: internal error", + ); + expect(errorContentChoice?.finish_reason).toBeNull(); + const stopChoices = choices.filter((choice) => choice.finish_reason === "stop"); + expect(stopChoices).toHaveLength(1); + expect(stopChoices[0]?.delta).toEqual({}); + expect(choices.at(-1)).toEqual(stopChoices[0]); } } finally { // shared server diff --git a/src/gateway/openai-http.ts b/src/gateway/openai-http.ts index f0c2f3b78e05..f21e34a665b0 100644 --- a/src/gateway/openai-http.ts +++ b/src/gateway/openai-http.ts @@ -283,7 +283,7 @@ function writeAssistantRoleChunk(res: ServerResponse, params: { runId: string; m function writeAssistantContentChunk( res: ServerResponse, - params: { runId: string; model: string; content: string; finishReason: "stop" | null }, + params: { runId: string; model: string; content: string }, ) { writeSse(res, { id: params.runId, @@ -294,7 +294,7 @@ function writeAssistantContentChunk( { index: 0, delta: { content: params.content }, - finish_reason: params.finishReason, + finish_reason: null, }, ], }); @@ -1292,7 +1292,6 @@ export async function handleOpenAiHttpRequest( runId, model, content, - finishReason: null, }); return; } @@ -1370,7 +1369,6 @@ export async function handleOpenAiHttpRequest( runId, model, content: commentary, - finishReason: null, }); } } @@ -1400,7 +1398,6 @@ export async function handleOpenAiHttpRequest( runId, model, content, - finishReason: null, }); } requestFinalize(); @@ -1436,9 +1433,7 @@ export async function handleOpenAiHttpRequest( runId, model, content, - finishReason: "stop", }); - wroteStopChunk = true; finalUsage = { prompt_tokens: 0, completion_tokens: 0,