feat(providers): drop o-series and pre-5.4 GPT-5 capability rows

The OpenAI commercial capability table floor is now gpt-5.4: o1,
o1-mini, o3, o3-mini, o3-pro, o4-mini, gpt-5, gpt-5-mini, gpt-5-nano,
gpt-5-pro, gpt-5.1, gpt-5.1-codex-max, gpt-5.2, gpt-5.2-pro, and
gpt-5.3 are effectively unused in the field. The gpt-5-search-api row
(different product surface) and the audio/STT/TTS rows stay.

A legacy id now resolves to OPENAI_DEFAULT (temperature sent, no
declared effort vocabulary, 200K window) — which those models may
reject; the remediation is the model definition's capabilities JSON or
a current model, release-noted under Unreleased → Removed.

This also retires the transport-collapse review's thrice-reported
"stream-rejecting o1-era models are stranded" finding by removing its
subject: no row in the table describes a non-streaming model anymore.

Tests migrate to 5.4-era equivalents that pin the same behaviors:
always-reasoning temperature suppression and off-list effort snap
(gpt-5.4-pro for gpt-5-pro/o3), explicit-none forwarding (gpt-5.4 for
gpt-5.1), empty-effort-vocabulary knob drop (gpt-5-search-api for
o1-mini), and the longest-prefix shadow hazard (gpt-5.4-pro vs gpt-5.4
for codex-max vs gpt-5.1).
This commit is contained in:
Patrick Buckley
2026-07-13 18:53:55 -07:00
parent 3a28dc2f16
commit 91c46051d9
7 changed files with 79 additions and 219 deletions
+19 -4
View File
@@ -45,10 +45,10 @@ Earlier stable lines (`stable/1.6`, `stable/1.5`) are frozen.
accepts its own terminal marker; the chat lane treats a clean
end-of-stream with content as completion) — only streams that die
mid-generation, deliver nothing, or drop the connection fail, and
those retry before surfacing. Legacy models
that reject streaming requests outright (o1-era) likewise need a
model alias pointing at a current model — the unread
`supports_streaming` capability flag (and its admin tile) is gone.
those retry before surfacing. The unread `supports_streaming`
capability flag (and its admin tile) is gone; the o-series models it
described are dropped from the capability table entirely (see
Removed).
- **One turn interface for every model call: `core/model_turn.py` (#827).**
Judges (intent + output guard), perception, title generation, compaction,
@@ -131,6 +131,21 @@ Earlier stable lines (`stable/1.6`, `stable/1.5`) are frozen.
inherit semantics the next time you change the model or a sampling
knob in that workstream. New workstreams inherit from the start.
### Removed
- **O-series and pre-5.4 GPT-5 rows dropped from the OpenAI capability
table.** `o1`, `o1-mini`, `o3`, `o3-mini`, `o3-pro`, `o4-mini`,
`gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-pro`, `gpt-5.1`,
`gpt-5.1-codex-max`, `gpt-5.2`, `gpt-5.2-pro`, and `gpt-5.3` no longer
have built-in capability rows — these models are effectively unused in
the field. The table floor is now `gpt-5.4`; the search-api and
audio/STT/TTS rows are unchanged. An alias still pinning one of the
removed ids resolves to the generic commercial defaults (temperature
sent, no declared reasoning-effort vocabulary, 200K window), which
those models may reject — declare the contract on the model
definition's capabilities JSON if you must stay on one, or move to a
current model.
### Fixed
- **OpenAI Responses streaming: truncated and refused responses no longer
+15 -12
View File
@@ -200,22 +200,25 @@ class TestFlatParamLanes:
assert eff["xhigh"] == "xhigh"
assert eff["max"] == "xhigh"
def test_openai_o3_registry_row(self) -> None:
"""o-series (except o1-mini) accept low/medium/high; no declared
"none" level, so the knob's off position omits the param."""
eff = _as_map(effort_ladder_for_model("openai", "o3", None))
def test_openai_always_reasoning_row_snaps_without_none(self) -> None:
"""Always-reasoning rows (gpt-5.4-pro: medium/high/xhigh) declare no
"none" level, so the knob's off position omits the param and low
positions snap UP onto the declared floor."""
eff = _as_map(effort_ladder_for_model("openai", "gpt-5.4-pro", None))
assert eff["none"] == "default"
assert eff["minimal"] == "low"
assert eff["minimal"] == "medium"
assert eff["medium"] == "medium"
assert eff["xhigh"] == eff["max"] == "high"
def test_openai_codex_max_has_xhigh(self) -> None:
"""gpt-5.1-codex-max must not prefix-fall onto the gpt-5.1 row
(which lacks xhigh) — xhigh reaches the wire verbatim."""
eff = _as_map(effort_ladder_for_model("openai", "gpt-5.1-codex-max", None))
assert eff["xhigh"] == "xhigh"
assert eff["max"] == "xhigh"
def test_openai_pro_row_wins_longest_prefix(self) -> None:
"""gpt-5.4-pro must not prefix-fall onto the gpt-5.4 row (which
declares "none") — the pro ladder has no off position, so the
longest-prefix row must win or the knob would wrongly omit."""
eff = _as_map(effort_ladder_for_model("openai", "gpt-5.4-pro", None))
assert eff["none"] == "default"
base = _as_map(effort_ladder_for_model("openai", "gpt-5.4", None))
assert base["none"] == "none"
def test_anthropic_effort_applies_even_with_thinking_mode_none(self) -> None:
"""output_config gates on supports_effort alone at request time."""
caps = ModelCapabilities(
+15 -14
View File
@@ -60,11 +60,11 @@ class TestProbeModelEndpoint:
@patch("turnstone.core.providers.create_client")
def test_target_found(self, mock_cc: MagicMock) -> None:
m1 = _mock_model("gpt-5")
m1 = _mock_model("gpt-5.4")
mock_cc.return_value = _mock_client(m1)
result = probe_model_endpoint(
"openai", "http://localhost:8000/v1", "key", target_model="gpt-5"
"openai", "http://localhost:8000/v1", "key", target_model="gpt-5.4"
)
assert result["model_found"] is True
@@ -74,7 +74,7 @@ class TestProbeModelEndpoint:
mock_cc.return_value = _mock_client(m1)
result = probe_model_endpoint(
"openai", "http://localhost:8000/v1", "key", target_model="gpt-5"
"openai", "http://localhost:8000/v1", "key", target_model="gpt-5.4"
)
assert result["model_found"] is False
assert result["available_models"] == ["model-a"]
@@ -98,7 +98,7 @@ class TestProbeModelEndpoint:
@patch("turnstone.core.providers.create_client")
def test_server_type_openai(self, mock_cc: MagicMock) -> None:
m = _mock_model("gpt-5")
m = _mock_model("gpt-5.4")
mock_cc.return_value = _mock_client(m)
result = probe_model_endpoint("openai", "https://api.openai.com/v1", "sk-test")
@@ -203,13 +203,13 @@ class TestProbeModelEndpoint:
@patch("turnstone.core.providers.create_client")
def test_context_window_openai_static_table(self, mock_cc: MagicMock) -> None:
"""When base_url is api.openai.com and model is known, use static table."""
m = _mock_model("gpt-5")
m = _mock_model("gpt-5.4")
mock_cc.return_value = _mock_client(m)
result = probe_model_endpoint(
"openai", "https://api.openai.com/v1", "sk-test", target_model="gpt-5"
"openai", "https://api.openai.com/v1", "sk-test", target_model="gpt-5.4"
)
assert result["context_window"] == 400000
assert result["context_window"] == 1050000
# ---------------------------------------------------------------------------
@@ -219,10 +219,11 @@ class TestProbeModelEndpoint:
class TestLookupModelCapabilities:
def test_known_openai_model(self) -> None:
caps = lookup_model_capabilities("openai", "gpt-5")
caps = lookup_model_capabilities("openai", "gpt-5.4")
assert caps is not None
assert caps["context_window"] == 400000
assert caps["supports_temperature"] is False
assert caps["context_window"] == 1050000
# 5.4 accepts temperature (applied only when effort="none")
assert caps["supports_temperature"] is True
def test_known_anthropic_model(self) -> None:
caps = lookup_model_capabilities("anthropic", "claude-opus-4-6")
@@ -235,13 +236,13 @@ class TestLookupModelCapabilities:
assert caps is None
def test_tuples_converted_to_lists(self) -> None:
caps = lookup_model_capabilities("openai", "gpt-5")
caps = lookup_model_capabilities("openai", "gpt-5.4")
assert caps is not None
for val in caps.values():
assert not isinstance(val, tuple), f"Found tuple: {val}"
def test_reasoning_effort_values_are_list(self) -> None:
caps = lookup_model_capabilities("openai", "gpt-5")
caps = lookup_model_capabilities("openai", "gpt-5.4")
assert caps is not None
assert isinstance(caps["reasoning_effort_values"], list)
assert "medium" in caps["reasoning_effort_values"]
@@ -252,7 +253,7 @@ class TestLookupModelCapabilities:
def test_invalid_provider_raises(self) -> None:
with pytest.raises(ValueError, match="Unknown provider"):
lookup_model_capabilities("bad-provider", "gpt-5")
lookup_model_capabilities("bad-provider", "gpt-5.4")
# ---------------------------------------------------------------------------
@@ -263,7 +264,7 @@ class TestLookupModelCapabilities:
class TestListKnownModels:
def test_openai_models(self) -> None:
models = list_known_models("openai")
assert "gpt-5" in models
assert "gpt-5.4" in models
assert isinstance(models, list)
assert models == sorted(models)
+23 -30
View File
@@ -2484,58 +2484,51 @@ class TestOpenAIParameterGating:
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none")
assert "reasoning_effort" not in kwargs
def test_gpt5_no_temperature_has_reasoning_effort(self) -> None:
"""GPT-5 base: no temperature, reasoning_effort sent."""
caps = lookup_openai_capabilities("gpt-5")
def test_always_reasoning_row_no_temperature_effort_sent(self) -> None:
"""Always-reasoning rows (gpt-5.4-pro): no temperature ever, the
knob's effort value reaches the wire."""
caps = lookup_openai_capabilities("gpt-5.4-pro")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "high"
def test_gpt51_temperature_when_effort_none(self) -> None:
"""GPT-5.1: temperature only when reasoning_effort='none'; the
def test_gpt54_temperature_when_effort_none(self) -> None:
"""GPT-5.4: temperature only when reasoning_effort='none'; the
declared "none" level is forwarded explicitly (knob = off)."""
caps = lookup_openai_capabilities("gpt-5.1")
caps = lookup_openai_capabilities("gpt-5.4")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="none")
assert kwargs["temperature"] == 0.7
assert kwargs["reasoning_effort"] == "none"
def test_gpt51_no_temperature_when_reasoning_active(self) -> None:
"""GPT-5.1: no temperature when reasoning is active."""
caps = lookup_openai_capabilities("gpt-5.1")
def test_gpt54_no_temperature_when_reasoning_active(self) -> None:
"""GPT-5.4: no temperature when reasoning is active."""
caps = lookup_openai_capabilities("gpt-5.4")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "high"
def test_o_series_no_temperature_but_effort_forwarded(self) -> None:
"""O-series: no temperature; low/medium/high ARE valid effort
values (all o-series except o1-mini) and the knob reaches them."""
caps = lookup_openai_capabilities("o3")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "medium"
def test_o1_mini_has_no_effort_control(self) -> None:
"""o1-mini is the one o-series model without reasoning_effort."""
caps = lookup_openai_capabilities("o1-mini")
def test_no_effort_vocabulary_row_drops_the_knob(self) -> None:
"""A commercial row with an EMPTY effort vocabulary (the search-api
model) drops the session knob — nothing valid to send."""
caps = lookup_openai_capabilities("gpt-5-search-api")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium")
assert "reasoning_effort" not in kwargs
def test_gpt5_pro_unsupported_effort_falls_back(self) -> None:
"""GPT-5 pro only supports 'high'; unsupported values fall back to default."""
caps = lookup_openai_capabilities("gpt-5-pro")
def test_pro_row_off_list_effort_snaps_onto_floor(self) -> None:
"""gpt-5.4-pro declares medium/high/xhigh; an off-list low value
rounds UP onto the declared floor."""
caps = lookup_openai_capabilities("gpt-5.4-pro")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="medium")
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="low")
assert "temperature" not in kwargs
assert kwargs["reasoning_effort"] == "high" # fell back to default
assert kwargs["reasoning_effort"] == "medium"
def test_gpt5_pro_supported_effort_passes_through(self) -> None:
"""GPT-5 pro accepts 'high' directly."""
caps = lookup_openai_capabilities("gpt-5-pro")
def test_pro_row_supported_effort_passes_through(self) -> None:
caps = lookup_openai_capabilities("gpt-5.4-pro")
kwargs: dict[str, Any] = {}
apply_temperature_and_effort(kwargs, caps, temperature=0.7, reasoning_effort="high")
assert kwargs["reasoning_effort"] == "high"
@@ -3999,7 +3992,7 @@ class TestVisionCapabilities:
assert caps.supports_vision is False
def test_openai_commercial_supports_vision(self) -> None:
for model in ("gpt-5", "gpt-5-mini", "gpt-5.4", "o3", "o4-mini"):
for model in ("gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-5.6-luna"):
caps = lookup_openai_capabilities(model)
assert caps.supports_vision is True, f"{model} should support vision"
+1 -1
View File
@@ -518,7 +518,7 @@ class TestProviderPdfCapabilities:
lookup_openai_capabilities,
)
assert lookup_openai_capabilities("gpt-5").supports_pdf is True
assert lookup_openai_capabilities("gpt-5.4").supports_pdf is True
# Unknown / local models stay False (PDF → client-side fallback).
assert OPENAI_DEFAULT.supports_pdf is False
+5 -157
View File
@@ -28,102 +28,12 @@ log = structlog.get_logger(__name__)
# Model capability table
# ---------------------------------------------------------------------------
# Table floor: gpt-5.4 (2026-07 pruning) — the o-series and pre-5.4 GPT-5
# rows were dropped as unused in the field. A legacy commercial id now
# resolves to OPENAI_DEFAULT (generic caps: temperature sent, no declared
# effort vocabulary); anyone still pinning one declares the contract on
# the model definition's capabilities JSON or moves to a current model.
OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = {
# GPT-5 base — NO temperature support
"gpt-5": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
supports_temperature=False,
reasoning_effort_values=("minimal", "low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
"gpt-5-mini": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
supports_temperature=False,
reasoning_effort_values=("minimal", "low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
"gpt-5-nano": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
supports_temperature=False,
reasoning_effort_values=("minimal", "low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# GPT-5 pro — high reasoning only, extended output
"gpt-5-pro": ModelCapabilities(
context_window=400000,
max_output_tokens=272000,
supports_temperature=False,
reasoning_effort_values=("high",),
default_reasoning_effort="high",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# GPT-5.1 — temperature OK when reasoning_effort=none (default)
"gpt-5.1": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
reasoning_effort_values=("none", "low", "medium", "high"),
default_reasoning_effort="none",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# GPT-5.1 codex-max — the model xhigh was introduced on; without this
# row it would prefix-match "gpt-5.1" (no xhigh) and the knob's xhigh
# would wrongly cap at high. Responses-only, supports explicit none.
"gpt-5.1-codex-max": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
reasoning_effort_values=("none", "low", "medium", "high", "xhigh"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# GPT-5.2 — adds xhigh
"gpt-5.2": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
reasoning_effort_values=("none", "low", "medium", "high", "xhigh"),
default_reasoning_effort="none",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# GPT-5.2 pro — always-reasoning variant
"gpt-5.2-pro": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
supports_temperature=False,
reasoning_effort_values=("medium", "high", "xhigh"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# GPT-5.3 — same capabilities as 5.2 (matches gpt-5.3-chat-latest, codex)
"gpt-5.3": ModelCapabilities(
context_window=400000,
max_output_tokens=128000,
reasoning_effort_values=("none", "low", "medium", "high", "xhigh"),
default_reasoning_effort="none",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# GPT-5.4 — 1M context window, native tool search
"gpt-5.4": ModelCapabilities(
context_window=1050000,
@@ -216,68 +126,6 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = {
supports_verbosity=True,
supports_pro_mode=True,
),
# O-series reasoning models
"o1": ModelCapabilities(
context_window=200000,
max_output_tokens=100000,
supports_temperature=False,
reasoning_effort_values=("low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
"o1-mini": ModelCapabilities(
context_window=128000,
max_output_tokens=65536,
supports_temperature=False,
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# low/medium/high on every o-series model EXCEPT o1-mini (Azure
# reasoning guide, 2026-06 revision) — without declared values the
# session knob was silently dropped for these models.
"o3": ModelCapabilities(
context_window=200000,
max_output_tokens=100000,
supports_temperature=False,
reasoning_effort_values=("low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
"o3-mini": ModelCapabilities(
context_window=200000,
max_output_tokens=100000,
supports_temperature=False,
reasoning_effort_values=("low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
"o3-pro": ModelCapabilities(
context_window=200000,
max_output_tokens=100000,
supports_temperature=False,
reasoning_effort_values=("low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
"o4-mini": ModelCapabilities(
context_window=200000,
max_output_tokens=100000,
supports_temperature=False,
reasoning_effort_values=("low", "medium", "high"),
default_reasoning_effort="medium",
supports_vision=True,
supports_pdf=True,
supports_reasoning_replay=True,
),
# Search models — always search on every request, no reasoning_effort
"gpt-5-search-api": ModelCapabilities(
context_window=400000,
+1 -1
View File
@@ -269,7 +269,7 @@ class ModelCapabilities:
# user's effort setting always reaches the wire, and the serving
# box is the authority on what it means. Commercial rows leave
# this False: there, an empty values list means the model has no
# effort control at all (o1-mini) and the param must be omitted.
# effort control at all (legacy o1-mini-era models) and the param must be omitted.
effort_passthrough: bool = False
supports_web_search: bool = False
supports_tool_search: bool = False