diff --git a/docs/architecture.md b/docs/architecture.md index d8403f3c..fa84da68 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -824,7 +824,12 @@ the levers live in the chat template, reached through respects that: an off-list knob value rounds UP onto the declared list and a value above the ceiling rides the ceiling (`snap_reasoning_effort`) — asking for more effort than the model - declares never falls back to a lower default tier. + declares never falls back to a lower default tier. The knob's + `none` position is forwarded verbatim when the model declares an + explicit `none` level (gpt-5.1+, grok-4.3) — omitting it there would + leave a reasoning-on server default (e.g. gpt-5.5's `medium`) in + charge of a knob that promises off — and omitted otherwise; `none` + is never a snap target for other positions. `default_reasoning_effort` only catches values the ordinal snap cannot rank (custom strings). Declare values that match the template's documented vocabulary: for DeepSeek-V4, which officially diff --git a/tests/test_effort_ladder.py b/tests/test_effort_ladder.py index cec3f76b..5bfbc3b5 100644 --- a/tests/test_effort_ladder.py +++ b/tests/test_effort_ladder.py @@ -86,6 +86,24 @@ class TestNativeAnthropicLane: assert eff["low"] == "low" assert eff["max"] == "max" + def test_sonnet_5_registry_row(self) -> None: + """claude-sonnet-5: adaptive + full effort ladder incl. xhigh/max — + every knob level above none is a distinct wire behavior.""" + eff = _as_map(effort_ladder_for_model("anthropic", "claude-sonnet-5", None)) + assert eff["none"] == "adaptive" + assert eff["minimal"] == "low" # rounds up onto declared levels + assert eff["low"] == "low" + assert eff["xhigh"] == "xhigh" + assert eff["max"] == "max" + + def test_sonnet_4_6_xhigh_rides_max(self) -> None: + """Sonnet 4.6 declares (low, medium, high, max) — no xhigh, so the + knob's xhigh snaps up onto max rather than down onto high.""" + eff = _as_map(effort_ladder_for_model("anthropic", "claude-sonnet-4-6", None)) + assert eff["high"] == "high" + assert eff["xhigh"] == "max" + assert eff["max"] == "max" + def test_manual_budget_ladder(self) -> None: """Budgets are monotone over the whole knob domain.""" caps = ModelCapabilities(thinking_mode="manual") @@ -136,10 +154,12 @@ class TestFlatParamLanes: def test_xai_projects_flat_only(self) -> None: """grok-4.3 declares values (none/low/medium/high, default low); - knob positions above the ceiling ride the ceiling (high), and the - declared "none" is never a snap target.""" + knob positions above the ceiling ride the ceiling (high). The + declared "none" IS forwarded for the knob's off position (xAI + documents it as disabling reasoning) but is never a snap target + for other positions.""" eff = _as_map(effort_ladder_for_model("xai", "grok-4.3", None)) - assert eff["none"] == "default" # resolve_ never forwards "none" + assert eff["none"] == "none" # explicit disable, declared by grok assert eff["minimal"] == "low" assert eff["low"] == "low" assert eff["high"] == "high" @@ -160,10 +180,37 @@ class TestFlatParamLanes: }, ) ) - assert eff["none"] == "default" # not "off" — there is no toggle + assert eff["none"] == "none" # flat channel, not an "off" toggle assert eff["medium"] == "medium" assert all("+" not in v and v not in ("on", "off") for v in eff.values()) + def test_openai_gpt55_registry_row(self) -> None: + """gpt-5.5 declares none/low/medium/high/xhigh with default medium: + knob none sends the explicit "none" level (server default is + MEDIUM, so omission would not disable), max rides the xhigh + ceiling, minimal rounds up to low.""" + eff = _as_map(effort_ladder_for_model("openai", "gpt-5.5", None)) + assert eff["none"] == "none" + assert eff["minimal"] == "low" + assert eff["xhigh"] == "xhigh" + assert eff["max"] == "xhigh" + + def test_openai_o3_registry_row(self) -> None: + """o-series (except o1-mini) accept low/medium/high; no declared + "none" level, so the knob's off position omits the param.""" + eff = _as_map(effort_ladder_for_model("openai", "o3", None)) + assert eff["none"] == "default" + assert eff["minimal"] == "low" + assert eff["medium"] == "medium" + assert eff["xhigh"] == eff["max"] == "high" + + def test_openai_codex_max_has_xhigh(self) -> None: + """gpt-5.1-codex-max must not prefix-fall onto the gpt-5.1 row + (which lacks xhigh) — xhigh reaches the wire verbatim.""" + eff = _as_map(effort_ladder_for_model("openai", "gpt-5.1-codex-max", None)) + assert eff["xhigh"] == "xhigh" + assert eff["max"] == "xhigh" + def test_anthropic_effort_applies_even_with_thinking_mode_none(self) -> None: """output_config gates on supports_effort alone at request time.""" caps = ModelCapabilities( diff --git a/tests/test_effort_ladder_wire_parity.py b/tests/test_effort_ladder_wire_parity.py index 2d6f28c6..29c0f3c4 100644 --- a/tests/test_effort_ladder_wire_parity.py +++ b/tests/test_effort_ladder_wire_parity.py @@ -60,10 +60,11 @@ class Shape: model: str = "m" -# Real registry rows for the two lanes whose defaults carry effort values — +# Real registry rows for the lanes whose defaults carry effort values — # parity should cover what ships, not only synthetic shapes. _GEMINI_CAPS = create_provider("google").get_capabilities("gemini-3-flash") _GROK_CAPS = create_provider("xai").get_capabilities("grok-4.3") +_GPT55_CAPS = create_provider("openai").get_capabilities("gpt-5.5") SHAPES: tuple[Shape, ...] = ( # -- anthropic-compatible (vLLM /v1/messages): template channel only -- @@ -159,12 +160,14 @@ SHAPES: tuple[Shape, ...] = ( ), # -- commercial flat lanes -- Shape( - "openai-flat", + # Real registry row: none/low/medium/high/xhigh, default medium. + # Knob none must send the EXPLICIT "none" level (omission would + # leave the server default medium reasoning on); knob max rides + # the xhigh ceiling. + "openai-gpt-5.5", "openai", - ModelCapabilities( - reasoning_effort_values=("low", "medium", "high"), - default_reasoning_effort="medium", - ), + _GPT55_CAPS, + model="gpt-5.5", ), Shape("google-default", "google", _GEMINI_CAPS, model="gemini-3-flash"), Shape( diff --git a/tests/test_providers.py b/tests/test_providers.py index 2033333f..6bc5ba55 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -2091,12 +2091,13 @@ class TestOpenAIParameterGating: assert kwargs["reasoning_effort"] == "high" def test_gpt51_temperature_when_effort_none(self) -> None: - """GPT-5.1: temperature only when reasoning_effort='none'.""" + """GPT-5.1: temperature only when reasoning_effort='none'; the + declared "none" level is forwarded explicitly (knob = off).""" caps = self.provider.get_capabilities("gpt-5.1") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert kwargs["temperature"] == 0.7 - assert "reasoning_effort" not in kwargs # "none" is skipped + assert kwargs["reasoning_effort"] == "none" def test_gpt51_no_temperature_when_reasoning_active(self) -> None: """GPT-5.1: no temperature when reasoning is active.""" @@ -2106,12 +2107,20 @@ class TestOpenAIParameterGating: assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "high" - def test_o_series_no_temperature_no_reasoning_effort(self) -> None: - """O-series: no temperature, no reasoning_effort.""" + def test_o_series_no_temperature_but_effort_forwarded(self) -> None: + """O-series: no temperature; low/medium/high ARE valid effort + values (all o-series except o1-mini) and the knob reaches them.""" caps = self.provider.get_capabilities("o3") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") assert "temperature" not in kwargs + assert kwargs["reasoning_effort"] == "medium" + + def test_o1_mini_has_no_effort_control(self) -> None: + """o1-mini is the one o-series model without reasoning_effort.""" + caps = self.provider.get_capabilities("o1-mini") + kwargs: dict[str, Any] = {} + apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") assert "reasoning_effort" not in kwargs def test_gpt5_pro_unsupported_effort_falls_back(self) -> None: @@ -2136,7 +2145,7 @@ class TestOpenAIParameterGating: kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert kwargs["temperature"] == 0.7 - assert "reasoning_effort" not in kwargs + assert kwargs["reasoning_effort"] == "none" # declared level, forwarded kwargs2: dict[str, Any] = {} apply_temperature_and_effort(kwargs2, caps, temperature=0.7, reasoning_effort="xhigh") assert "temperature" not in kwargs2 @@ -2160,7 +2169,7 @@ class TestOpenAIParameterGating: kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert kwargs["temperature"] == 0.7 - assert "reasoning_effort" not in kwargs + assert kwargs["reasoning_effort"] == "none" # declared level, forwarded kwargs2: dict[str, Any] = {} apply_temperature_and_effort(kwargs2, caps, temperature=0.7, reasoning_effort="xhigh") assert "temperature" not in kwargs2 @@ -4254,7 +4263,10 @@ class TestResponsesParamBuilding: assert kwargs["reasoning"] == {"effort": "high"} assert "reasoning_effort" not in kwargs - def test_no_reasoning_when_none_effort(self) -> None: + def test_none_effort_sends_declared_none_level(self) -> None: + """gpt-5.4 declares an explicit "none" level — the knob's off + position forwards it rather than omitting (omission would leave + the server default in charge on models like gpt-5.5).""" kwargs = self.provider._build_kwargs( model="gpt-5.4", messages=[{"role": "user", "content": "Hi"}], @@ -4264,7 +4276,7 @@ class TestResponsesParamBuilding: reasoning_effort="none", deferred_names=None, ) - assert "reasoning" not in kwargs + assert kwargs["reasoning"] == {"effort": "none"} def test_store_is_false(self) -> None: kwargs = self.provider._build_kwargs( diff --git a/turnstone/core/providers/_anthropic.py b/turnstone/core/providers/_anthropic.py index 026da786..792e1d05 100644 --- a/turnstone/core/providers/_anthropic.py +++ b/turnstone/core/providers/_anthropic.py @@ -201,13 +201,37 @@ _ANTHROPIC_CAPABILITIES: dict[str, ModelCapabilities] = { supports_pdf=True, supports_reasoning_replay=True, ), - "claude-sonnet-4-6": ModelCapabilities( + # Sonnet 5: adaptive thinking is on by default (explicit adaptive config + # accepted; manual budget_tokens is a 400); unlike Fable 5, an explicit + # thinking={"type": "disabled"} IS accepted. Non-default sampling params + # (temperature/top_p/top_k) are a 400, same as Opus 4.8 / Fable 5. + # Effort: full ladder incl. xhigh + max; API default is high + # (platform.claude.com/docs/en/build-with-claude/effort, 2026-07 check). + "claude-sonnet-5": ModelCapabilities( context_window=1000000, - max_output_tokens=64000, + max_output_tokens=128000, token_param="max_tokens", thinking_mode="adaptive", supports_effort=True, - effort_levels=("low", "medium", "high"), + effort_levels=("low", "medium", "high", "xhigh", "max"), + supports_web_search=True, + supports_tool_search=True, + supports_vision=True, + supports_pdf=True, + supports_temperature=False, + thinking_display="summarized", + supports_reasoning_replay=True, + ), + # Sonnet 4.6: effort ladder tops at max (no xhigh — that level exists + # only on Fable 5 / Sonnet 5 / Opus 4.7+); knob xhigh rides max via the + # ordinal snap. + "claude-sonnet-4-6": ModelCapabilities( + context_window=1000000, + max_output_tokens=128000, + token_param="max_tokens", + thinking_mode="adaptive", + supports_effort=True, + effort_levels=("low", "medium", "high", "max"), supports_web_search=True, supports_tool_search=True, supports_vision=True, diff --git a/turnstone/core/providers/_openai_common.py b/turnstone/core/providers/_openai_common.py index 1d7a24ef..ed908218 100644 --- a/turnstone/core/providers/_openai_common.py +++ b/turnstone/core/providers/_openai_common.py @@ -81,6 +81,18 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_pdf=True, supports_reasoning_replay=True, ), + # GPT-5.1 codex-max — the model xhigh was introduced on; without this + # row it would prefix-match "gpt-5.1" (no xhigh) and the knob's xhigh + # would wrongly cap at high. Responses-only, supports explicit none. + "gpt-5.1-codex-max": ModelCapabilities( + context_window=400000, + max_output_tokens=128000, + reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), + default_reasoning_effort="medium", + supports_vision=True, + supports_pdf=True, + supports_reasoning_replay=True, + ), # GPT-5.2 — adds xhigh "gpt-5.2": ModelCapabilities( context_window=400000, @@ -135,12 +147,14 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_pdf=True, supports_reasoning_replay=True, ), - # GPT-5.5 — 1M context, native tool search, stronger agentic/tool use + # GPT-5.5 — 1M context, native tool search, stronger agentic/tool use. + # Unlike 5.1-5.4 (default none), 5.5 defaults to MEDIUM reasoning per + # developers.openai.com/api/docs/guides/latest-model (2026-07 check). "gpt-5.5": ModelCapabilities( context_window=1050000, max_output_tokens=128000, reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), - default_reasoning_effort="none", + default_reasoning_effort="medium", supports_tool_search=True, supports_vision=True, supports_pdf=True, @@ -164,6 +178,8 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { max_output_tokens=100000, supports_temperature=False, supports_streaming=False, + reasoning_effort_values=("low", "medium", "high"), + default_reasoning_effort="medium", supports_vision=True, supports_pdf=True, supports_reasoning_replay=True, @@ -177,10 +193,15 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_pdf=True, supports_reasoning_replay=True, ), + # low/medium/high on every o-series model EXCEPT o1-mini (Azure + # reasoning guide, 2026-06 revision) — without declared values the + # session knob was silently dropped for these models. "o3": ModelCapabilities( context_window=200000, max_output_tokens=100000, supports_temperature=False, + reasoning_effort_values=("low", "medium", "high"), + default_reasoning_effort="medium", supports_vision=True, supports_pdf=True, supports_reasoning_replay=True, @@ -189,6 +210,8 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { context_window=200000, max_output_tokens=100000, supports_temperature=False, + reasoning_effort_values=("low", "medium", "high"), + default_reasoning_effort="medium", supports_vision=True, supports_pdf=True, supports_reasoning_replay=True, @@ -198,6 +221,8 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { max_output_tokens=100000, supports_temperature=False, supports_streaming=False, + reasoning_effort_values=("low", "medium", "high"), + default_reasoning_effort="medium", supports_vision=True, supports_pdf=True, supports_reasoning_replay=True, @@ -206,6 +231,8 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { context_window=200000, max_output_tokens=100000, supports_temperature=False, + reasoning_effort_values=("low", "medium", "high"), + default_reasoning_effort="medium", supports_vision=True, supports_pdf=True, supports_reasoning_replay=True, diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index d364f353..756b83ff 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -183,9 +183,19 @@ def resolve_reasoning_effort(caps: ModelCapabilities, reasoning_effort: str) -> the declared list, capped at its ceiling (``snap_reasoning_effort``). ``default_reasoning_effort`` is the last resort for values the ordinal snap cannot rank (custom strings on either side). + + The knob's ``"none"`` position means "no reasoning": it is forwarded + verbatim ONLY when the model declares an explicit ``"none"`` level + (gpt-5.1+, grok-4.3) — omitting the param there would leave the + server default (possibly reasoning ON, e.g. gpt-5.5's ``medium``) in + charge of a knob that promises off. Models without a declared + ``"none"`` get the param omitted, and ``"none"`` is never a snap + target for other knob positions. """ - if not caps.reasoning_effort_values or not reasoning_effort or reasoning_effort == "none": + if not caps.reasoning_effort_values or not reasoning_effort: return None + if reasoning_effort == "none": + return "none" if "none" in caps.reasoning_effort_values else None if reasoning_effort in caps.reasoning_effort_values: return reasoning_effort snapped = snap_reasoning_effort(reasoning_effort, caps.reasoning_effort_values)