From 6ec84bc3bd56af188f345ab38492d3bdb79037a3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Jul 2026 03:33:30 -0400 Subject: [PATCH] fix(models): expose custom max and ultra reasoning tiers (#115690) * fix(models): honor custom advanced reasoning levels * fix(ci): restore code mode matrix boundaries --- scripts/code-mode-model-matrix.ts | 2 + src/auto-reply/thinking.test.ts | 45 +++++++++++++++++++++ src/auto-reply/thinking.ts | 38 ++++++++++++----- src/cli/program/register.agent-turn.ts | 5 ++- src/cli/program/register.agent.test.ts | 14 +++++++ test/scripts/code-mode-model-matrix.test.ts | 1 + 6 files changed, 92 insertions(+), 13 deletions(-) diff --git a/scripts/code-mode-model-matrix.ts b/scripts/code-mode-model-matrix.ts index 98f39a8d659d..bd3f81351c4d 100644 --- a/scripts/code-mode-model-matrix.ts +++ b/scripts/code-mode-model-matrix.ts @@ -18,6 +18,8 @@ import { import type { AgentExecEnvelope } from "../src/commands/agent-exec.ts"; import { previewForDevToolLog, redactJsonValueForDevToolLog } from "./lib/dev-tooling-safety.ts"; +export { validateQaEvidenceSummaryJson }; + const execFileAsync = promisify(execFile); const SOURCE_PATH = "scripts/code-mode-model-matrix.ts"; const MATRIX_SCHEMA_VERSION = 1; diff --git a/src/auto-reply/thinking.test.ts b/src/auto-reply/thinking.test.ts index 5fab458f50d1..42169a3586f2 100644 --- a/src/auto-reply/thinking.test.ts +++ b/src/auto-reply/thinking.test.ts @@ -601,6 +601,51 @@ describe("listThinkingLevels", () => { ).toBe(true); }); + it("uses advanced catalog efforts and derives OpenClaw Ultra from Max", () => { + const catalog = [ + { + provider: "myazure", + id: "gpt-5.6-sol", + name: "GPT 5.6 Sol via Azure", + api: "openai-responses", + reasoning: true, + compat: { + supportedReasoningEfforts: ["none", "low", "medium", "high", "xhigh", "max"], + }, + }, + ]; + + expect(listThinkingLevels("myazure", "gpt-5.6-sol", catalog, "openclaw")).toEqual([ + "off", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + "ultra", + ]); + expect( + isThinkingLevelSupported({ + provider: "myazure", + model: "gpt-5.6-sol", + level: "max", + catalog, + agentRuntime: "openclaw", + }), + ).toBe(true); + expect( + isThinkingLevelSupported({ + provider: "myazure", + model: "gpt-5.6-sol", + level: "ultra", + catalog, + agentRuntime: "openclaw", + }), + ).toBe(true); + expect(listThinkingLevels("myazure", "gpt-5.6-sol", catalog, "codex")).not.toContain("ultra"); + }); + it("does not let catalog xhigh compat override binary thinking providers", () => { providerRuntimeMocks.resolveProviderThinkingProfile.mockReturnValue({ levels: [ diff --git a/src/auto-reply/thinking.ts b/src/auto-reply/thinking.ts index eebedf843e53..cbbf9b6b8ca3 100644 --- a/src/auto-reply/thinking.ts +++ b/src/auto-reply/thinking.ts @@ -94,14 +94,6 @@ function resolveThinkingPolicyContext(params: { }; } -function catalogSupportsXHigh(compat: ThinkingCatalogEntry["compat"]): boolean { - const efforts = compat?.supportedReasoningEfforts; - if (!Array.isArray(efforts)) { - return false; - } - return efforts.some((effort) => normalizeThinkLevel(effort) === "xhigh"); -} - function normalizeProfileLevel( level: ProviderThinkingProfile["levels"][number], ): RankedThinkingLevelOption | undefined { @@ -158,6 +150,32 @@ function appendProfileLevel(profile: ResolvedThinkingProfile, id: ThinkLevel) { profile.levels = profile.levels.toSorted((a, b) => a.rank - b.rank); } +const CATALOG_ADVANCED_THINKING_LEVELS = new Set(["adaptive", "xhigh", "max"]); + +function appendCatalogAdvancedThinkingLevels( + profile: ResolvedThinkingProfile, + compat: ThinkingCatalogEntry["compat"], + agentRuntime?: string | null, +) { + const efforts = compat?.supportedReasoningEfforts; + if (!Array.isArray(efforts)) { + return; + } + let supportsMax = false; + for (const effort of efforts) { + const level = normalizeThinkLevel(effort); + if (level && CATALOG_ADVANCED_THINKING_LEVELS.has(level)) { + appendProfileLevel(profile, level); + supportsMax ||= level === "max"; + } + } + const runtime = normalizeOptionalLowercaseString(agentRuntime); + if (supportsMax && (runtime === "openclaw" || runtime === "auto")) { + // Ultra is OpenClaw's orchestration tier; provider requests use Max. + appendProfileLevel(profile, "ultra"); + } +} + /** Resolve supported thinking levels and default for a provider/model pair. */ export function resolveThinkingProfile(params: { provider?: string | null; @@ -215,9 +233,7 @@ export function resolveThinkingProfile(params: { } const profile = buildBaseThinkingProfile(); - if (catalogSupportsXHigh(context.compat)) { - appendProfileLevel(profile, "xhigh"); - } + appendCatalogAdvancedThinkingLevels(profile, context.compat, params.agentRuntime); return profile; } diff --git a/src/cli/program/register.agent-turn.ts b/src/cli/program/register.agent-turn.ts index 6a74bb0da2f7..82ecfbc3c2ce 100644 --- a/src/cli/program/register.agent-turn.ts +++ b/src/cli/program/register.agent-turn.ts @@ -3,6 +3,7 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/st import type { Command } from "commander"; import { formatDocsLink } from "../../../packages/terminal-core/src/links.js"; import { theme } from "../../../packages/terminal-core/src/theme.js"; +import { THINKING_LEVELS_HELP } from "../../auto-reply/thinking.shared.js"; import { formatHelpExamples } from "../help-format.js"; type AgentViaGatewayModule = typeof import("../../commands/agent-via-gateway.js"); @@ -52,7 +53,7 @@ export function registerAgentTurnCommand( .option("--model ", "Model override for this run (provider/model or model id)") .option( "--thinking ", - "Thinking level: off | minimal | low | medium | high | xhigh | adaptive | max where supported", + `Thinking level: ${THINKING_LEVELS_HELP.replaceAll("|", " | ")} where supported`, ) .option("--verbose ", "Persist agent verbose level for the session") .option( @@ -130,7 +131,7 @@ ${theme.muted("Docs:")} ${formatDocsLink("/cli/agent", "docs.openclaw.ai/cli/age .option("--local-model-lean", "Use the reduced local-model tool surface") .option( "--thinking ", - "Thinking level: off | minimal | low | medium | high | xhigh | adaptive | max where supported", + `Thinking level: ${THINKING_LEVELS_HELP.replaceAll("|", " | ")} where supported`, ) .option( "--fallback ", diff --git a/src/cli/program/register.agent.test.ts b/src/cli/program/register.agent.test.ts index d129f97d4c17..273460c77562 100644 --- a/src/cli/program/register.agent.test.ts +++ b/src/cli/program/register.agent.test.ts @@ -102,6 +102,20 @@ describe("agent command registration", () => { return call; } + it("keeps both agent thinking help surfaces aligned with the canonical levels", () => { + const program = new Command(); + registerAgentTurnCommand(program, { agentChannelOptions: "last|telegram|discord" }); + const agent = program.commands.find((command) => command.name() === "agent"); + const exec = agent?.commands.find((command) => command.name() === "exec"); + + expect(agent?.options.find((option) => option.long === "--thinking")?.description).toContain( + "ultra", + ); + expect(exec?.options.find((option) => option.long === "--thinking")?.description).toContain( + "ultra", + ); + }); + it("runs agent command with verbose enabled for --verbose on", async () => { await runCli(["agent", "--message", "hi", "--verbose", "ON", "--json"]); diff --git a/test/scripts/code-mode-model-matrix.test.ts b/test/scripts/code-mode-model-matrix.test.ts index 724848a6ca8d..0bd7545e0604 100644 --- a/test/scripts/code-mode-model-matrix.test.ts +++ b/test/scripts/code-mode-model-matrix.test.ts @@ -11,6 +11,7 @@ import { reserveCodeModeMatrixOutputDir, resolveCodeModeMatrixOutputDir, runCodeModeModelMatrix, + validateQaEvidenceSummaryJson, type CodeModeMatrixCellResult, } from "../../scripts/code-mode-model-matrix.ts"; import type { AgentExecEnvelope } from "../../src/commands/agent-exec.ts";