diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index bf06354351f5..41596733251e 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -1488,7 +1488,6 @@ packages/agent-core/src/harness/session/session.ts 1 packages/agent-core/src/harness/session/tool-result-pairing.ts 17 packages/agent-core/src/harness/session/uuid.ts 1 packages/agent-core/src/harness/utils/truncate.ts 1 -packages/agent-core/src/reasoning.ts 1 packages/agent-core/src/turn-interruption.ts 1 packages/ai/src/api-registry.ts 3 packages/ai/src/env-api-keys.ts 10 diff --git a/packages/agent-core/src/agent-loop.test.ts b/packages/agent-core/src/agent-loop.test.ts index c02076ebd0a1..b0325c43f075 100644 --- a/packages/agent-core/src/agent-loop.test.ts +++ b/packages/agent-core/src/agent-loop.test.ts @@ -3629,7 +3629,7 @@ describe("agentLoop thinking state", () => { name: "disables reasoning after leaving Fable", initialModel: { ...model, id: "claude-fable-5", thinkingLevelMap: { off: "low" } }, nextModel: model, - expected: ["low", undefined], + expected: ["low", "off"], }, { name: "uses Fable's low fallback after entering Fable", diff --git a/packages/agent-core/src/reasoning.test.ts b/packages/agent-core/src/reasoning.test.ts index 90fab943b4e7..32824392bb02 100644 --- a/packages/agent-core/src/reasoning.test.ts +++ b/packages/agent-core/src/reasoning.test.ts @@ -27,39 +27,45 @@ describe("resolveAgentReasoningOption", () => { expect(resolveAgentReasoningOption(makeModel({ off: "low" }), "off")).toBe("low"); }); - it.each([undefined, null, "none"])("disables reasoning when off maps to %s", (offFallback) => { - expect(resolveAgentReasoningOption(makeModel({ off: offFallback }), "off")).toBeUndefined(); + it.each([undefined, "none"])("preserves explicit off when off maps to %s", (offFallback) => { + expect(resolveAgentReasoningOption(makeModel({ off: offFallback }), "off")).toBe("off"); + }); + + it("leaves unsupported off mapping to the transport", () => { + expect(resolveAgentReasoningOption(makeModel({ off: null }), "off")).toBeUndefined(); }); it("preserves enabled thinking levels", () => { expect(resolveAgentReasoningOption(makeModel({ off: "low" }), "high")).toBe("high"); }); - it("preserves explicit off for Sonnet 5 on Anthropic Messages routes", () => { - expect( - resolveAgentReasoningOption(makeModel(undefined, { id: "claude-sonnet-5" }), "off"), - ).toBe("off"); + it.each( + ["claude-sonnet-5", "anthropic.claude-opus-5"].flatMap((id) => + ([undefined, null, "none", "low"] as const).map((off) => ({ id, off })), + ), + )("retains the native $id exception with off=$off", ({ id, off }) => { + expect(resolveAgentReasoningOption(makeModel({ off }, { id }), "off")).toBe( + off === "low" ? "low" : "off", + ); }); - it("uses the route-owned Sonnet 5 off mapping when provided", () => { - expect( - resolveAgentReasoningOption( - makeModel({ off: "low" }, { id: "anthropic.claude-sonnet-5" }), - "off", - ), - ).toBe("low"); - }); - - it.each(["anthropic-messages", "bedrock-converse-stream"] as const)( - "maps explicit off to low for canonical Fable aliases on %s", - (api) => { + it.each( + (["anthropic-messages", "bedrock-converse-stream"] as const).flatMap((api) => + ([undefined, null] as const).map((off) => ({ api, off })), + ), + )( + "maps explicit off to low for canonical Fable aliases on $api with off=$off", + ({ api, off }) => { expect( resolveAgentReasoningOption( - makeModel(undefined, { - id: "production-deployment", - api, - params: { canonicalModelId: "claude-fable-5" }, - }), + makeModel( + { off }, + { + id: "production-deployment", + api, + params: { canonicalModelId: "claude-fable-5" }, + }, + ), "off", ), ).toBe("low"); diff --git a/packages/agent-core/src/reasoning.ts b/packages/agent-core/src/reasoning.ts index 7571f7350e49..cc6dedb7ef67 100644 --- a/packages/agent-core/src/reasoning.ts +++ b/packages/agent-core/src/reasoning.ts @@ -7,21 +7,6 @@ import { } from "@openclaw/llm-core"; import type { ThinkingLevel } from "./types.js"; -type EnabledThinkingLevel = Exclude, "off">; - -const ENABLED_THINKING_LEVELS = new Set([ - "minimal", - "low", - "medium", - "high", - "xhigh", - "max", -]); - -function isEnabledThinkingLevel(value: unknown): value is EnabledThinkingLevel { - return ENABLED_THINKING_LEVELS.has(value as EnabledThinkingLevel); -} - export function resolveAgentReasoningOption( model: Model, thinkingLevel: ThinkingLevel, @@ -35,11 +20,20 @@ export function resolveAgentReasoningOption( resolveClaudeFable5ModelIdentity(model) ? "low" : undefined); - if (isEnabledThinkingLevel(offFallback)) { - return offFallback; + switch (offFallback) { + case "minimal": + case "low": + case "medium": + case "high": + case "xhigh": + case "max": + return offFallback; + default: + // Unsupported off keeps transport defaults; native Sonnet/Opus retain their off contract. + return model.thinkingLevelMap?.off !== null || + (model.api === "anthropic-messages" && + (resolveClaudeSonnet5ModelIdentity(model) || resolveClaudeOpus5ModelIdentity(model))) + ? "off" + : undefined; } - return model.api === "anthropic-messages" && - (resolveClaudeSonnet5ModelIdentity(model) || resolveClaudeOpus5ModelIdentity(model)) - ? "off" - : undefined; } diff --git a/src/agents/openai-thinking-contract.test.ts b/src/agents/openai-thinking-contract.test.ts index 2f1b2a534c0c..a53440cbdfd5 100644 --- a/src/agents/openai-thinking-contract.test.ts +++ b/src/agents/openai-thinking-contract.test.ts @@ -1,4 +1,6 @@ // Verifies session thinking levels reach OpenAI and Codex Responses transports. +import { createServer } from "node:http"; +import { createLlmRuntime } from "@openclaw/ai"; import { Agent, type StreamFn } from "openclaw/plugin-sdk/agent-core"; import { createAssistantMessageEventStream, @@ -9,6 +11,7 @@ import { streamSimple, } from "openclaw/plugin-sdk/llm"; import { describe, expect, it } from "vitest"; +import { resolveEmbeddedAgentStreamFn } from "./embedded-agent-runner/stream-resolution.js"; type ResponsesModel = Model<"openai-responses"> | Model<"openai-chatgpt-responses">; @@ -36,6 +39,73 @@ const codexTestToken = [ ].join("."); describe("OpenAI thinking contract", () => { + it.each( + (["qwen", "qwen-chat-template"] as const).flatMap((thinkingFormat) => + (["managed", "direct"] as const).flatMap((transport) => + ([undefined, null, "low", "none"] as const).flatMap((offFallback) => + ([undefined, "off", "high"] as const).map((thinkingLevel) => ({ + thinkingFormat, + transport, + offFallback, + thinkingLevel, + })), + ), + ), + ), + )( + "honors Agent $thinkingLevel with off=$offFallback over $transport $thinkingFormat HTTP", + async ({ thinkingFormat, transport, offFallback, thinkingLevel }) => { + const payload = await captureHttpProviderPayload({ + api: "openai-completions", + thinkingFormat, + transport, + thinkingLevelMap: { off: offFallback }, + thinkingLevel, + mode: "agent", + }); + const thinking = thinkingFormat === "qwen" ? payload : payload.chat_template_kwargs; + expect(thinking).toMatchObject({ + enable_thinking: + thinkingLevel === "high" || + offFallback === "low" || + (offFallback === null && transport === "managed"), + }); + }, + ); + + it("preserves explicit Agent off when a managed Responses request asks for a summary", async () => { + for (const thinkingLevel of ["off", "high"] as const) { + const payload = await captureHttpProviderPayload({ + api: "openai-responses", + thinkingLevel, + reasoningSummary: "auto", + mode: "agent", + }); + expect(payload.reasoning).toEqual( + thinkingLevel === "off" ? undefined : { effort: "high", summary: "auto" }, + ); + } + }); + + it("retains standalone managed defaults when no reasoning option is supplied", async () => { + const completions = await captureHttpProviderPayload({ + api: "openai-completions", + thinkingFormat: "qwen-chat-template", + mode: "standalone", + }); + expect(completions.chat_template_kwargs).toMatchObject({ enable_thinking: true }); + for (const reasoningSummary of [undefined, "auto"] as const) { + const responses = await captureHttpProviderPayload({ + api: "openai-responses", + reasoningSummary, + mode: "standalone", + }); + expect(responses.reasoning).toEqual( + reasoningSummary ? { effort: "high", summary: "auto" } : undefined, + ); + } + }); + it.each([ { model: openaiModel, expectedReasoning: "high" }, { model: codexModel, expectedReasoning: "high" }, @@ -58,7 +128,7 @@ describe("OpenAI thinking contract", () => { ); it.each([openaiModel, codexModel])( - "does not forward reasoning when session thinkingLevel is off for $provider/$id", + "preserves explicit off when session thinkingLevel is off for $provider/$id", async (model) => { const capturedOptions: SimpleStreamOptions[] = []; const agent = new Agent({ @@ -71,7 +141,7 @@ describe("OpenAI thinking contract", () => { await agent.prompt("hello"); - expect(capturedOptions.map(({ reasoning }) => reasoning)).toStrictEqual([undefined]); + expect(capturedOptions.map(({ reasoning }) => reasoning)).toStrictEqual(["off"]); }, ); @@ -95,27 +165,132 @@ describe("OpenAI thinking contract", () => { expect(payload.reasoning).toEqual({ effort: "high", summary: "auto" }); }); - it("leaves Codex Responses reasoning absent when agent runtime disables thinking", async () => { - const payload = await captureProviderPayload({ - model: codexModel, - streamFn: streamSimple, - options: { transport: "sse" }, - }); + it.each([undefined, "off"] as const)( + "leaves direct Codex Responses reasoning absent for %s", + async (reasoning) => { + const payload = await captureProviderPayload({ + model: codexModel, + streamFn: streamSimple, + options: { transport: "sse", reasoning }, + }); - expect(payload).not.toHaveProperty("reasoning"); - }); + expect(payload).not.toHaveProperty("reasoning"); + }, + ); - it("keeps OpenAI Responses reasoning explicitly disabled when agent runtime disables thinking", async () => { - const payload = await captureProviderPayload({ - model: openaiModel, - streamFn: streamSimple, - options: {}, - }); + it.each([undefined, "off"] as const)( + "keeps direct OpenAI Responses reasoning disabled for %s", + async (reasoning) => { + const payload = await captureProviderPayload({ + model: openaiModel, + streamFn: streamSimple, + options: { reasoning }, + }); - expect(payload.reasoning).toEqual({ effort: "none" }); - }); + expect(payload.reasoning).toEqual({ effort: "none" }); + }, + ); }); +async function captureHttpProviderPayload(params: { + api: "openai-completions" | "openai-responses"; + thinkingFormat?: "qwen" | "qwen-chat-template"; + transport?: "managed" | "direct"; + thinkingLevelMap?: Model["thinkingLevelMap"]; + thinkingLevel?: "off" | "high"; + reasoningSummary?: "auto"; + mode: "agent" | "standalone"; +}): Promise> { + let payload: Record | undefined; + const server = createServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + }); + request.on("end", () => { + payload = JSON.parse(body) as Record; + const event = + params.api === "openai-completions" + ? { + id: "chatcmpl_thinking", + choices: [ + { index: 0, delta: { role: "assistant", content: "ok" }, finish_reason: "stop" }, + ], + } + : { + type: "response.completed", + response: { id: "resp_thinking", status: "completed", output: [] }, + }; + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end(`data: ${JSON.stringify(event)}\n\ndata: [DONE]\n\n`); + }); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Missing loopback server address"); + } + const model: Model = { + id: params.api === "openai-completions" ? "qwen3.6-27b" : "gpt-5.5", + name: "Thinking contract model", + api: params.api, + provider: "local-thinking", + baseUrl: `http://127.0.0.1:${address.port}/v1`, + reasoning: true, + thinkingLevelMap: params.thinkingLevelMap, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4_096, + ...(params.thinkingFormat + ? { compat: { thinkingFormat: params.thinkingFormat, supportsReasoningEffort: false } } + : {}), + }; + const providerStream = + params.transport === "direct" + ? streamSimple + : resolveEmbeddedAgentStreamFn({ + llmRuntime: createLlmRuntime(), + currentStreamFn: undefined, + model, + sessionId: "thinking-contract", + resolvedApiKey: "synthetic-test-key", + }); + const streamFn: StreamFn = (requestModel, context, options) => + providerStream(requestModel, context, { + ...options, + apiKey: "synthetic-test-key", + ...(params.reasoningSummary ? { reasoningSummary: params.reasoningSummary } : {}), + }); + if (params.mode === "agent") { + const agent = new Agent({ + initialState: { model, thinkingLevel: params.thinkingLevel }, + streamFn, + }); + await agent.prompt("hello"); + expect(agent.state.errorMessage).toBeUndefined(); + } else { + const stream = await streamFn(model, { + messages: [{ role: "user", content: "hello", timestamp: 0 }], + }); + expect((await stream.result()).stopReason).toBe("stop"); + } + if (!payload) { + throw new Error("Provider did not receive a request"); + } + return payload; + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } +} + function createCapturingStreamFn( model: ResponsesModel, capturedOptions: SimpleStreamOptions[],