diff --git a/CHANGELOG.md b/CHANGELOG.md index beffad63..844d21fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,10 +45,10 @@ Earlier stable lines (`stable/1.6`, `stable/1.5`) are frozen. accepts its own terminal marker; the chat lane treats a clean end-of-stream with content as completion) — only streams that die mid-generation, deliver nothing, or drop the connection fail, and - those retry before surfacing. Legacy models - that reject streaming requests outright (o1-era) likewise need a - model alias pointing at a current model — the unread - `supports_streaming` capability flag (and its admin tile) is gone. + those retry before surfacing. The unread `supports_streaming` + capability flag (and its admin tile) is gone; the o-series models it + described are dropped from the capability table entirely (see + Removed). - **One turn interface for every model call: `core/model_turn.py` (#827).** Judges (intent + output guard), perception, title generation, compaction, @@ -131,6 +131,21 @@ Earlier stable lines (`stable/1.6`, `stable/1.5`) are frozen. inherit semantics the next time you change the model or a sampling knob in that workstream. New workstreams inherit from the start. +### Removed + +- **O-series and pre-5.4 GPT-5 rows dropped from the OpenAI capability + table.** `o1`, `o1-mini`, `o3`, `o3-mini`, `o3-pro`, `o4-mini`, + `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-pro`, `gpt-5.1`, + `gpt-5.1-codex-max`, `gpt-5.2`, `gpt-5.2-pro`, and `gpt-5.3` no longer + have built-in capability rows — these models are effectively unused in + the field. The table floor is now `gpt-5.4`; the search-api and + audio/STT/TTS rows are unchanged. An alias still pinning one of the + removed ids resolves to the generic commercial defaults (temperature + sent, no declared reasoning-effort vocabulary, 200K window), which + those models may reject — declare the contract on the model + definition's capabilities JSON if you must stay on one, or move to a + current model. + ### Fixed - **OpenAI Responses streaming: truncated and refused responses no longer diff --git a/tests/test_effort_ladder.py b/tests/test_effort_ladder.py index b90ef0ed..034ff448 100644 --- a/tests/test_effort_ladder.py +++ b/tests/test_effort_ladder.py @@ -200,22 +200,25 @@ class TestFlatParamLanes: assert eff["xhigh"] == "xhigh" assert eff["max"] == "xhigh" - def test_openai_o3_registry_row(self) -> None: - """o-series (except o1-mini) accept low/medium/high; no declared - "none" level, so the knob's off position omits the param.""" - eff = _as_map(effort_ladder_for_model("openai", "o3", None)) + def test_openai_always_reasoning_row_snaps_without_none(self) -> None: + """Always-reasoning rows (gpt-5.4-pro: medium/high/xhigh) declare no + "none" level, so the knob's off position omits the param and low + positions snap UP onto the declared floor.""" + eff = _as_map(effort_ladder_for_model("openai", "gpt-5.4-pro", None)) assert eff["none"] == "default" - assert eff["minimal"] == "low" + assert eff["minimal"] == "medium" assert eff["medium"] == "medium" - assert eff["xhigh"] == eff["max"] == "high" - - def test_openai_codex_max_has_xhigh(self) -> None: - """gpt-5.1-codex-max must not prefix-fall onto the gpt-5.1 row - (which lacks xhigh) — xhigh reaches the wire verbatim.""" - eff = _as_map(effort_ladder_for_model("openai", "gpt-5.1-codex-max", None)) - assert eff["xhigh"] == "xhigh" assert eff["max"] == "xhigh" + def test_openai_pro_row_wins_longest_prefix(self) -> None: + """gpt-5.4-pro must not prefix-fall onto the gpt-5.4 row (which + declares "none") — the pro ladder has no off position, so the + longest-prefix row must win or the knob would wrongly omit.""" + eff = _as_map(effort_ladder_for_model("openai", "gpt-5.4-pro", None)) + assert eff["none"] == "default" + base = _as_map(effort_ladder_for_model("openai", "gpt-5.4", None)) + assert base["none"] == "none" + def test_anthropic_effort_applies_even_with_thinking_mode_none(self) -> None: """output_config gates on supports_effort alone at request time.""" caps = ModelCapabilities( diff --git a/tests/test_model_probe.py b/tests/test_model_probe.py index c28e2dc1..977cf101 100644 --- a/tests/test_model_probe.py +++ b/tests/test_model_probe.py @@ -60,11 +60,11 @@ class TestProbeModelEndpoint: @patch("turnstone.core.providers.create_client") def test_target_found(self, mock_cc: MagicMock) -> None: - m1 = _mock_model("gpt-5") + m1 = _mock_model("gpt-5.4") mock_cc.return_value = _mock_client(m1) result = probe_model_endpoint( - "openai", "http://localhost:8000/v1", "key", target_model="gpt-5" + "openai", "http://localhost:8000/v1", "key", target_model="gpt-5.4" ) assert result["model_found"] is True @@ -74,7 +74,7 @@ class TestProbeModelEndpoint: mock_cc.return_value = _mock_client(m1) result = probe_model_endpoint( - "openai", "http://localhost:8000/v1", "key", target_model="gpt-5" + "openai", "http://localhost:8000/v1", "key", target_model="gpt-5.4" ) assert result["model_found"] is False assert result["available_models"] == ["model-a"] @@ -98,7 +98,7 @@ class TestProbeModelEndpoint: @patch("turnstone.core.providers.create_client") def test_server_type_openai(self, mock_cc: MagicMock) -> None: - m = _mock_model("gpt-5") + m = _mock_model("gpt-5.4") mock_cc.return_value = _mock_client(m) result = probe_model_endpoint("openai", "https://api.openai.com/v1", "sk-test") @@ -203,13 +203,13 @@ class TestProbeModelEndpoint: @patch("turnstone.core.providers.create_client") def test_context_window_openai_static_table(self, mock_cc: MagicMock) -> None: """When base_url is api.openai.com and model is known, use static table.""" - m = _mock_model("gpt-5") + m = _mock_model("gpt-5.4") mock_cc.return_value = _mock_client(m) result = probe_model_endpoint( - "openai", "https://api.openai.com/v1", "sk-test", target_model="gpt-5" + "openai", "https://api.openai.com/v1", "sk-test", target_model="gpt-5.4" ) - assert result["context_window"] == 400000 + assert result["context_window"] == 1050000 # --------------------------------------------------------------------------- @@ -219,10 +219,11 @@ class TestProbeModelEndpoint: class TestLookupModelCapabilities: def test_known_openai_model(self) -> None: - caps = lookup_model_capabilities("openai", "gpt-5") + caps = lookup_model_capabilities("openai", "gpt-5.4") assert caps is not None - assert caps["context_window"] == 400000 - assert caps["supports_temperature"] is False + assert caps["context_window"] == 1050000 + # 5.4 accepts temperature (applied only when effort="none") + assert caps["supports_temperature"] is True def test_known_anthropic_model(self) -> None: caps = lookup_model_capabilities("anthropic", "claude-opus-4-6") @@ -235,13 +236,13 @@ class TestLookupModelCapabilities: assert caps is None def test_tuples_converted_to_lists(self) -> None: - caps = lookup_model_capabilities("openai", "gpt-5") + caps = lookup_model_capabilities("openai", "gpt-5.4") assert caps is not None for val in caps.values(): assert not isinstance(val, tuple), f"Found tuple: {val}" def test_reasoning_effort_values_are_list(self) -> None: - caps = lookup_model_capabilities("openai", "gpt-5") + caps = lookup_model_capabilities("openai", "gpt-5.4") assert caps is not None assert isinstance(caps["reasoning_effort_values"], list) assert "medium" in caps["reasoning_effort_values"] @@ -252,7 +253,7 @@ class TestLookupModelCapabilities: def test_invalid_provider_raises(self) -> None: with pytest.raises(ValueError, match="Unknown provider"): - lookup_model_capabilities("bad-provider", "gpt-5") + lookup_model_capabilities("bad-provider", "gpt-5.4") # --------------------------------------------------------------------------- @@ -263,7 +264,7 @@ class TestLookupModelCapabilities: class TestListKnownModels: def test_openai_models(self) -> None: models = list_known_models("openai") - assert "gpt-5" in models + assert "gpt-5.4" in models assert isinstance(models, list) assert models == sorted(models) diff --git a/tests/test_providers.py b/tests/test_providers.py index 7922cf8f..0d73755a 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -2484,58 +2484,51 @@ class TestOpenAIParameterGating: apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert "reasoning_effort" not in kwargs - def test_gpt5_no_temperature_has_reasoning_effort(self) -> None: - """GPT-5 base: no temperature, reasoning_effort sent.""" - caps = lookup_openai_capabilities("gpt-5") + def test_always_reasoning_row_no_temperature_effort_sent(self) -> None: + """Always-reasoning rows (gpt-5.4-pro): no temperature ever, the + knob's effort value reaches the wire.""" + caps = lookup_openai_capabilities("gpt-5.4-pro") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "high" - def test_gpt51_temperature_when_effort_none(self) -> None: - """GPT-5.1: temperature only when reasoning_effort='none'; the + def test_gpt54_temperature_when_effort_none(self) -> None: + """GPT-5.4: temperature only when reasoning_effort='none'; the declared "none" level is forwarded explicitly (knob = off).""" - caps = lookup_openai_capabilities("gpt-5.1") + caps = lookup_openai_capabilities("gpt-5.4") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none") assert kwargs["temperature"] == 0.7 assert kwargs["reasoning_effort"] == "none" - def test_gpt51_no_temperature_when_reasoning_active(self) -> None: - """GPT-5.1: no temperature when reasoning is active.""" - caps = lookup_openai_capabilities("gpt-5.1") + def test_gpt54_no_temperature_when_reasoning_active(self) -> None: + """GPT-5.4: no temperature when reasoning is active.""" + caps = lookup_openai_capabilities("gpt-5.4") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high") assert "temperature" not in kwargs assert kwargs["reasoning_effort"] == "high" - def test_o_series_no_temperature_but_effort_forwarded(self) -> None: - """O-series: no temperature; low/medium/high ARE valid effort - values (all o-series except o1-mini) and the knob reaches them.""" - caps = lookup_openai_capabilities("o3") - kwargs: dict[str, Any] = {} - apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") - assert "temperature" not in kwargs - assert kwargs["reasoning_effort"] == "medium" - - def test_o1_mini_has_no_effort_control(self) -> None: - """o1-mini is the one o-series model without reasoning_effort.""" - caps = lookup_openai_capabilities("o1-mini") + def test_no_effort_vocabulary_row_drops_the_knob(self) -> None: + """A commercial row with an EMPTY effort vocabulary (the search-api + model) drops the session knob — nothing valid to send.""" + caps = lookup_openai_capabilities("gpt-5-search-api") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") assert "reasoning_effort" not in kwargs - def test_gpt5_pro_unsupported_effort_falls_back(self) -> None: - """GPT-5 pro only supports 'high'; unsupported values fall back to default.""" - caps = lookup_openai_capabilities("gpt-5-pro") + def test_pro_row_off_list_effort_snaps_onto_floor(self) -> None: + """gpt-5.4-pro declares medium/high/xhigh; an off-list low value + rounds UP onto the declared floor.""" + caps = lookup_openai_capabilities("gpt-5.4-pro") kwargs: dict[str, Any] = {} - apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium") + apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="low") assert "temperature" not in kwargs - assert kwargs["reasoning_effort"] == "high" # fell back to default + assert kwargs["reasoning_effort"] == "medium" - def test_gpt5_pro_supported_effort_passes_through(self) -> None: - """GPT-5 pro accepts 'high' directly.""" - caps = lookup_openai_capabilities("gpt-5-pro") + def test_pro_row_supported_effort_passes_through(self) -> None: + caps = lookup_openai_capabilities("gpt-5.4-pro") kwargs: dict[str, Any] = {} apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high") assert kwargs["reasoning_effort"] == "high" @@ -3999,7 +3992,7 @@ class TestVisionCapabilities: assert caps.supports_vision is False def test_openai_commercial_supports_vision(self) -> None: - for model in ("gpt-5", "gpt-5-mini", "gpt-5.4", "o3", "o4-mini"): + for model in ("gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-5.6-luna"): caps = lookup_openai_capabilities(model) assert caps.supports_vision is True, f"{model} should support vision" diff --git a/tests/test_providers_document_parts.py b/tests/test_providers_document_parts.py index e9e44866..917da370 100644 --- a/tests/test_providers_document_parts.py +++ b/tests/test_providers_document_parts.py @@ -518,7 +518,7 @@ class TestProviderPdfCapabilities: lookup_openai_capabilities, ) - assert lookup_openai_capabilities("gpt-5").supports_pdf is True + assert lookup_openai_capabilities("gpt-5.4").supports_pdf is True # Unknown / local models stay False (PDF → client-side fallback). assert OPENAI_DEFAULT.supports_pdf is False diff --git a/turnstone/core/providers/_openai_common.py b/turnstone/core/providers/_openai_common.py index 2f276722..b56b89ba 100644 --- a/turnstone/core/providers/_openai_common.py +++ b/turnstone/core/providers/_openai_common.py @@ -28,102 +28,12 @@ log = structlog.get_logger(__name__) # Model capability table # --------------------------------------------------------------------------- +# Table floor: gpt-5.4 (2026-07 pruning) — the o-series and pre-5.4 GPT-5 +# rows were dropped as unused in the field. A legacy commercial id now +# resolves to OPENAI_DEFAULT (generic caps: temperature sent, no declared +# effort vocabulary); anyone still pinning one declares the contract on +# the model definition's capabilities JSON or moves to a current model. OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { - # GPT-5 base — NO temperature support - "gpt-5": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - supports_temperature=False, - reasoning_effort_values=("minimal", "low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - "gpt-5-mini": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - supports_temperature=False, - reasoning_effort_values=("minimal", "low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - "gpt-5-nano": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - supports_temperature=False, - reasoning_effort_values=("minimal", "low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - # GPT-5 pro — high reasoning only, extended output - "gpt-5-pro": ModelCapabilities( - context_window=400000, - max_output_tokens=272000, - supports_temperature=False, - reasoning_effort_values=("high",), - default_reasoning_effort="high", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - # GPT-5.1 — temperature OK when reasoning_effort=none (default) - "gpt-5.1": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - reasoning_effort_values=("none", "low", "medium", "high"), - default_reasoning_effort="none", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - # GPT-5.1 codex-max — the model xhigh was introduced on; without this - # row it would prefix-match "gpt-5.1" (no xhigh) and the knob's xhigh - # would wrongly cap at high. Responses-only, supports explicit none. - "gpt-5.1-codex-max": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - # GPT-5.2 — adds xhigh - "gpt-5.2": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), - default_reasoning_effort="none", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - # GPT-5.2 pro — always-reasoning variant - "gpt-5.2-pro": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - supports_temperature=False, - reasoning_effort_values=("medium", "high", "xhigh"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - # GPT-5.3 — same capabilities as 5.2 (matches gpt-5.3-chat-latest, codex) - "gpt-5.3": ModelCapabilities( - context_window=400000, - max_output_tokens=128000, - reasoning_effort_values=("none", "low", "medium", "high", "xhigh"), - default_reasoning_effort="none", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), # GPT-5.4 — 1M context window, native tool search "gpt-5.4": ModelCapabilities( context_window=1050000, @@ -216,68 +126,6 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { supports_verbosity=True, supports_pro_mode=True, ), - # O-series reasoning models - "o1": ModelCapabilities( - context_window=200000, - max_output_tokens=100000, - supports_temperature=False, - reasoning_effort_values=("low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - "o1-mini": ModelCapabilities( - context_window=128000, - max_output_tokens=65536, - supports_temperature=False, - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - # low/medium/high on every o-series model EXCEPT o1-mini (Azure - # reasoning guide, 2026-06 revision) — without declared values the - # session knob was silently dropped for these models. - "o3": ModelCapabilities( - context_window=200000, - max_output_tokens=100000, - supports_temperature=False, - reasoning_effort_values=("low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - "o3-mini": ModelCapabilities( - context_window=200000, - max_output_tokens=100000, - supports_temperature=False, - reasoning_effort_values=("low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - "o3-pro": ModelCapabilities( - context_window=200000, - max_output_tokens=100000, - supports_temperature=False, - reasoning_effort_values=("low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), - "o4-mini": ModelCapabilities( - context_window=200000, - max_output_tokens=100000, - supports_temperature=False, - reasoning_effort_values=("low", "medium", "high"), - default_reasoning_effort="medium", - supports_vision=True, - supports_pdf=True, - supports_reasoning_replay=True, - ), # Search models — always search on every request, no reasoning_effort "gpt-5-search-api": ModelCapabilities( context_window=400000, diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index 02372f4c..31fd27c1 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -269,7 +269,7 @@ class ModelCapabilities: # user's effort setting always reaches the wire, and the serving # box is the authority on what it means. Commercial rows leave # this False: there, an empty values list means the model has no - # effort control at all (o1-mini) and the param must be omitted. + # effort control at all (legacy o1-mini-era models) and the param must be omitted. effort_passthrough: bool = False supports_web_search: bool = False supports_tool_search: bool = False