diff --git a/docs/architecture.md b/docs/architecture.md index bad7ac6b..d8403f3c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -820,13 +820,19 @@ the levers live in the chat template, reached through `effort_param` (e.g. `"reasoning_effort"`), plus optional `reasoning_effort_values` / `default_reasoning_effort` to validate the knob before it reaches the template; without declared values the - knob is forwarded as-is. Only declare values that match the - template's documented vocabulary — snapping an off-list knob to the - default can defeat a real tier. DeepSeek-V4, for example, officially + knob is forwarded as-is. The knob is ordinal, and validation + respects that: an off-list knob value rounds UP onto the declared + list and a value above the ceiling rides the ceiling + (`snap_reasoning_effort`) — asking for more effort than the model + declares never falls back to a lower default tier. + `default_reasoning_effort` only catches values the ordinal snap + cannot rank (custom strings). Declare values that match the + template's documented vocabulary: for DeepSeek-V4, which officially accepts `high`/`max` (Think High is the default thinking tier; - `low`/`medium` alias to `high`, `xhigh` to `max`), so freeform - passthrough matches the contract while a `("low", "medium", "high")` - values list would make Think Max unreachable. To map an undocumented + `low`/`medium` alias to `high`, `xhigh` to `max`), a + `("high", "max")` values list reproduces the official aliasing + exactly — `low`/`medium` round up to `high`, `xhigh` to `max` — + and freeform passthrough matches it too. To map an undocumented template, probe with per-request `chat_template_kwargs` and compare `input_tokens`. Setting `effort_param` also suppresses the flat top-level `reasoning_effort` request param on the diff --git a/tests/test_effort_ladder.py b/tests/test_effort_ladder.py index 88c67b39..cec3f76b 100644 --- a/tests/test_effort_ladder.py +++ b/tests/test_effort_ladder.py @@ -41,7 +41,8 @@ class TestLocalLanes: assert eff["max"] == "on+max" def test_validated_effort_param_shows_snapping(self) -> None: - """Declared values collapse off-list positions onto the default.""" + """Off-list positions round up onto the declared values; above the + ceiling they ride the ceiling — never the (possibly lower) default.""" caps = ModelCapabilities( thinking_mode="manual", thinking_param="enable_thinking", @@ -50,9 +51,10 @@ class TestLocalLanes: default_reasoning_effort="medium", ) eff = _as_map(effort_ladder("openai-compatible", caps)) - assert eff["xhigh"] == "on+medium" - assert eff["max"] == "on+medium" + assert eff["minimal"] == "on+low" assert eff["high"] == "on+high" + assert eff["xhigh"] == "on+high" + assert eff["max"] == "on+high" def test_openai_compatible_flat_param_without_effort_param(self) -> None: caps = ModelCapabilities( @@ -62,7 +64,7 @@ class TestLocalLanes: eff = _as_map(effort_ladder("openai-compatible", caps)) assert eff["none"] == "default" assert eff["high"] == "high" - assert eff["xhigh"] == "medium" + assert eff["xhigh"] == "high" # ceiling, not default def test_adaptive_local_never_off(self) -> None: caps = ModelCapabilities(thinking_mode="adaptive", thinking_param="enable_thinking") @@ -80,19 +82,20 @@ class TestNativeAnthropicLane: ) eff = _as_map(effort_ladder("anthropic", caps)) assert eff["none"] == "adaptive" # thinking on, model decides - assert eff["minimal"] == "adaptive" # unmapped knob level + assert eff["minimal"] == "low" # rounds up onto the declared levels assert eff["low"] == "low" assert eff["max"] == "max" def test_manual_budget_ladder(self) -> None: + """Budgets are monotone over the whole knob domain.""" caps = ModelCapabilities(thinking_mode="manual") eff = _as_map(effort_ladder("anthropic", caps)) assert eff["none"] == "off" - assert eff["low"] == "budget:1024" + assert eff["minimal"] == eff["low"] == "budget:1024" # 1024 = API floor assert eff["medium"] == "budget:4096" assert eff["high"] == "budget:16384" - # Off-map knob levels share the default budget — aliased group. - assert eff["minimal"] == eff["xhigh"] == eff["max"] == "budget:4096" + assert eff["xhigh"] == "budget:32768" + assert eff["max"] == "budget:65536" class TestFlatParamLanes: @@ -133,12 +136,14 @@ class TestFlatParamLanes: def test_xai_projects_flat_only(self) -> None: """grok-4.3 declares values (none/low/medium/high, default low); - off-list knob positions snap to the default.""" + knob positions above the ceiling ride the ceiling (high), and the + declared "none" is never a snap target.""" eff = _as_map(effort_ladder_for_model("xai", "grok-4.3", None)) assert eff["none"] == "default" # resolve_ never forwards "none" + assert eff["minimal"] == "low" assert eff["low"] == "low" assert eff["high"] == "high" - assert eff["xhigh"] == eff["max"] == "low" + assert eff["xhigh"] == eff["max"] == "high" def test_xai_ignores_template_overrides(self) -> None: """XAIProvider subclasses OpenAIResponsesProvider, which drops diff --git a/tests/test_effort_ladder_wire_parity.py b/tests/test_effort_ladder_wire_parity.py index fd051acd..2d6f28c6 100644 --- a/tests/test_effort_ladder_wire_parity.py +++ b/tests/test_effort_ladder_wire_parity.py @@ -40,9 +40,13 @@ from turnstone.core.providers import create_provider from turnstone.core.providers._protocol import ModelCapabilities from turnstone.core.providers.effort_ladder import KNOB_VALUES, effort_ladder -# Big enough that Anthropic manual-mode budgets are never clamped by -# max_tokens (the ladder documents budgets unclamped). -_MAX_TOKENS = 32_000 +# Above the largest manual-mode thinking budget (max: 65536) so the +# request path's budget None: - """Off-list knob snaps to default_reasoning_effort, not sent raw.""" + """Off-list knob rounds up onto the declared values (ceiling-capped), + never sent raw — and never snaps DOWN to the default.""" caps = dataclasses.replace( self._MANUAL_CAPS, effort_param="reasoning_effort", @@ -255,7 +256,7 @@ class TestCompatReasoningControl: ) kwargs = self._stream_kwargs(caps, "xhigh") assert kwargs["extra_body"] == { - "chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "medium"} + "chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "high"} } def test_effort_param_freeform_without_values(self) -> None: diff --git a/tests/test_providers.py b/tests/test_providers.py index b37cc1c3..2033333f 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -218,7 +218,8 @@ class TestOpenAIProvider: assert kwargs["reasoning_effort"] == "medium" def test_effort_param_injects_knob_value(self) -> None: - """effort_param carries the knob into chat_template_kwargs (gpt-oss).""" + """effort_param carries the knob into chat_template_kwargs (gpt-oss); + a knob above the declared ceiling rides the ceiling, not the default.""" caps = ModelCapabilities( thinking_mode="none", effort_param="reasoning_effort", @@ -226,7 +227,7 @@ class TestOpenAIProvider: default_reasoning_effort="medium", ) eb = self.provider._finalize_extra_body(None, caps, "xhigh") - assert eb == {"chat_template_kwargs": {"reasoning_effort": "medium"}} + assert eb == {"chat_template_kwargs": {"reasoning_effort": "high"}} assert self.provider._finalize_extra_body(None, caps, "none") is None def test_caller_extra_params_not_mutated(self) -> None: @@ -2386,11 +2387,20 @@ class TestAnthropicReasoningNone: result = _map_reasoning_to_effort("xhigh", ("low", "medium", "high", "xhigh", "max")) assert result == "xhigh" - def test_map_xhigh_rejected_by_model_without_it(self) -> None: + def test_map_xhigh_snaps_up_through_gap_to_max(self) -> None: + """Levels with a hole (no xhigh) round the knob UP to the next + declared level rather than dropping output_config entirely.""" from turnstone.core.providers._anthropic import _map_reasoning_to_effort result = _map_reasoning_to_effort("xhigh", ("low", "medium", "high", "max")) - assert result is None + assert result == "max" + + def test_map_above_ceiling_rides_ceiling(self) -> None: + from turnstone.core.providers._anthropic import _map_reasoning_to_effort + + assert _map_reasoning_to_effort("max", ("low", "medium", "high")) == "high" + assert _map_reasoning_to_effort("minimal", ("low", "medium", "high")) == "low" + assert _map_reasoning_to_effort("none", ("low", "medium", "high")) is None # =========================================================================== @@ -3696,8 +3706,9 @@ class TestAnthropicPromptCaching: ) assert kwargs["output_config"] == {"effort": "xhigh"} - def test_xhigh_effort_not_applied_to_opus_4_6(self) -> None: - """xhigh is not a valid effort level for Opus 4.6 — should be ignored.""" + def test_xhigh_effort_snaps_to_max_on_opus_4_6(self) -> None: + """Opus 4.6 declares (low, medium, high, max) — a knob of xhigh + rounds up to max instead of silently dropping output_config.""" caps = self.provider.get_capabilities("claude-opus-4-6") kwargs = self.provider._build_thinking_and_kwargs( caps=caps, @@ -3710,7 +3721,7 @@ class TestAnthropicPromptCaching: model="claude-opus-4-6", tools=None, ) - assert "output_config" not in kwargs + assert kwargs["output_config"] == {"effort": "max"} @patch("turnstone.core.providers._anthropic._ensure_anthropic") def test_streaming_message_start_cache_metrics(self, mock_ensure: MagicMock) -> None: diff --git a/turnstone/core/providers/_anthropic.py b/turnstone/core/providers/_anthropic.py index dcfbc76c..026da786 100644 --- a/turnstone/core/providers/_anthropic.py +++ b/turnstone/core/providers/_anthropic.py @@ -21,6 +21,7 @@ from turnstone.core.providers._protocol import ( _join_reasoning_with_cap, _lookup_capabilities, merge_reasoning_template_kwargs, + snap_reasoning_effort, ) from turnstone.core.trajectory import materialize_attachments @@ -78,11 +79,21 @@ _TOOL_SEARCH_TOOL_TYPE = "tool_search_tool_bm25" # bodies byte-identical when a caller threads thinking overrides. _INTERNAL_EXTRA_PARAMS = frozenset({"thinking_budget_tokens"}) -# Manual-mode thinking budgets per effort knob level; knob values outside -# the map (minimal, xhigh, max) fall back to the default budget. Public: -# shared with ``effort_ladder`` so UI projections can't drift from the -# wire. -EFFORT_BUDGET_MAP = {"low": 1024, "medium": 4096, "high": 16384} +# Manual-mode thinking budgets per effort knob level — monotone over the +# whole knob domain (a higher knob position must never buy a smaller +# budget; the request path still clamps below per-request max_tokens). +# minimal shares the 1024 floor with low: the API rejects budgets under +# 1024, so there is no smaller tier to express. Unknown strings (custom +# configs) fall back to the default budget. Public: shared with +# ``effort_ladder`` so UI projections can't drift from the wire. +EFFORT_BUDGET_MAP = { + "minimal": 1024, + "low": 1024, + "medium": 4096, + "high": 16384, + "xhigh": 32768, + "max": 65536, +} DEFAULT_THINKING_BUDGET = 4096 # -- model capabilities ------------------------------------------------------- @@ -242,12 +253,19 @@ def _map_reasoning_to_effort( reasoning_effort: str, valid_levels: tuple[str, ...], ) -> str | None: - """Map turnstone reasoning_effort to Anthropic effort parameter.""" - mapping = {"low": "low", "medium": "medium", "high": "high", "xhigh": "xhigh", "max": "max"} - effort = mapping.get(reasoning_effort) - if effort and effort in valid_levels: - return effort - return None + """Map turnstone reasoning_effort to Anthropic effort parameter. + + Declared levels match verbatim; off-list knob values round up onto + the declared levels, capped at their ceiling (a knob above the + model's top level must ride the top level, not silently drop + output_config). "none"/empty and unrankable strings return None — + no effort param. + """ + if not reasoning_effort or reasoning_effort == "none": + return None + if reasoning_effort in valid_levels: + return reasoning_effort + return snap_reasoning_effort(reasoning_effort, valid_levels) # Anthropic accepts only a closed set of content-block types on the diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index a9c3e9a3..d364f353 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -86,9 +86,10 @@ class ModelCapabilities: # gpt-oss-style templates). Empty = the template has no effort lever, # send nothing. The session knob value is validated against # ``reasoning_effort_values`` when that is non-empty (off-list values - # snap to ``default_reasoning_effort``); with no declared values the - # knob is forwarded as-is and the template is the authority on - # validity. See ``reasoning_template_kwargs``. + # round UP onto the declared list, capped at its ceiling — see + # ``snap_reasoning_effort``); with no declared values the knob is + # forwarded as-is and the template is the authority on validity. + # See ``reasoning_template_kwargs``. effort_param: str = "" supports_effort: bool = False effort_levels: tuple[str, ...] = () @@ -147,15 +148,49 @@ class ModelCapabilities: rerank_separated: bool = False +# The session effort knob is ORDINAL — snapping must respect this order. +# "none" is the disable position and is never a snap target (snapping a +# high knob onto "none" would invert the request into "don't think"). +KNOB_EFFORT_ORDER: tuple[str, ...] = ("none", "minimal", "low", "medium", "high", "xhigh", "max") +_KNOB_RANK: dict[str, int] = {value: rank for rank, value in enumerate(KNOB_EFFORT_ORDER)} + + +def snap_reasoning_effort(reasoning_effort: str, declared: tuple[str, ...]) -> str | None: + """Round an off-list knob value up onto the declared effort levels. + + Returns the smallest declared level ranking >= the knob; when the + knob is above every declared level, the highest declared level (the + ceiling — asking for more effort than exists must not fall to a + lower tier). Declared values outside the knob vocabulary cannot be + ranked and are reachable only by exact match in the caller; "none" + is never a snap target. Returns ``None`` when the knob itself is + unrankable or nothing declared is rankable. + """ + rank = _KNOB_RANK.get(reasoning_effort) + if rank is None or reasoning_effort == "none": + return None + rankable = [(r, v) for v in declared if (r := _KNOB_RANK.get(v)) is not None and v != "none"] + if not rankable: + return None + at_or_above = [(r, v) for r, v in rankable if r >= rank] + return min(at_or_above)[1] if at_or_above else max(rankable)[1] + + def resolve_reasoning_effort(caps: ModelCapabilities, reasoning_effort: str) -> str | None: """Return the validated reasoning effort value, or ``None`` to omit. - Validates against supported values and falls back to model default. + Declared values match verbatim; off-list knob values round UP onto + the declared list, capped at its ceiling (``snap_reasoning_effort``). + ``default_reasoning_effort`` is the last resort for values the + ordinal snap cannot rank (custom strings on either side). """ if not caps.reasoning_effort_values or not reasoning_effort or reasoning_effort == "none": return None if reasoning_effort in caps.reasoning_effort_values: return reasoning_effort + snapped = snap_reasoning_effort(reasoning_effort, caps.reasoning_effort_values) + if snapped: + return snapped if caps.default_reasoning_effort and caps.default_reasoning_effort != "none": return caps.default_reasoning_effort return None @@ -193,9 +228,10 @@ def reasoning_template_kwargs( the native adaptive branch never lets the knob force-disable thinking). The effort value is validated via ``resolve_reasoning_effort`` when the model declares - ``reasoning_effort_values`` (off-list knob values snap to - ``default_reasoning_effort``); with no declared values the knob is - forwarded as-is and the template is the authority on validity. + ``reasoning_effort_values`` (off-list knob values round up onto the + declared list, capped at its ceiling); with no declared values the + knob is forwarded as-is and the template is the authority on + validity. """ updates: dict[str, Any] = {} effort_on = bool(reasoning_effort) and reasoning_effort != "none"