diff --git a/docs/architecture.md b/docs/architecture.md index 4314329b..bad7ac6b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -852,6 +852,16 @@ both local-server lanes, so `thinking_mode`/`thinking_param`/ `effort_param` mean the same thing whichever endpoint serves the model. Only the Responses API surface (native reasoning) ignores it. +The console surfaces this projection as an *effective effort ladder*: +the admin model form's per-model effort select and the skill +launch-config effort select annotate knob positions that alias to the +same wire behavior (e.g. "Max (= high)"), computed server-side by +`providers/effort_ladder.py` from the same mapping functions the +providers use at request time and shipped on `/v1/api/models` rows and +`POST /v1/api/admin/models/effort-ladder`. The ladder describes what +Turnstone sends — a server-side template may alias further (DeepSeek-V4 +folds `low`/`medium` into its default `high` tier). + The `anthropic-compatible` lane never sends Anthropic's native `thinking`/`output_config` params — they are not in vLLM's request schema. The real `anthropic` provider is unaffected: official Claude diff --git a/tests/test_effort_ladder.py b/tests/test_effort_ladder.py new file mode 100644 index 00000000..c0ab753c --- /dev/null +++ b/tests/test_effort_ladder.py @@ -0,0 +1,115 @@ +"""Tests for the effective effort-ladder projection. + +The ladder must mirror the request-time mapping functions exactly — +equal ``effective`` tokens promise byte-identical effort behavior on +the wire, which is what the UI annotations lean on. +""" + +from __future__ import annotations + +from turnstone.core.providers._protocol import ModelCapabilities +from turnstone.core.providers.effort_ladder import ( + KNOB_VALUES, + effort_ladder, + effort_ladder_for_model, +) + + +def _as_map(ladder: list[dict[str, str]]) -> dict[str, str]: + assert [r["value"] for r in ladder] == list(KNOB_VALUES) + return {r["value"]: r["effective"] for r in ladder} + + +class TestLocalLanes: + def test_qwen_style_toggle_only_two_groups(self) -> None: + """Toggle-only model: none=off, everything else one 'on' group.""" + caps = ModelCapabilities(thinking_mode="manual", thinking_param="enable_thinking") + eff = _as_map(effort_ladder("anthropic-compatible", caps)) + assert eff["none"] == "off" + assert {eff[k] for k in KNOB_VALUES if k != "none"} == {"on"} + + def test_freeform_effort_param_forwards_each_value(self) -> None: + """deepseek-style config: toggle + verbatim effort per position.""" + caps = ModelCapabilities( + thinking_mode="manual", + thinking_param="thinking", + effort_param="reasoning_effort", + ) + eff = _as_map(effort_ladder("anthropic-compatible", caps)) + assert eff["none"] == "off" + assert eff["low"] == "on+low" + assert eff["max"] == "on+max" + + def test_validated_effort_param_shows_snapping(self) -> None: + """Declared values collapse off-list positions onto the default.""" + caps = ModelCapabilities( + thinking_mode="manual", + thinking_param="enable_thinking", + effort_param="reasoning_effort", + reasoning_effort_values=("low", "medium", "high"), + default_reasoning_effort="medium", + ) + eff = _as_map(effort_ladder("openai-compatible", caps)) + assert eff["xhigh"] == "on+medium" + assert eff["max"] == "on+medium" + assert eff["high"] == "on+high" + + def test_openai_compatible_flat_param_without_effort_param(self) -> None: + caps = ModelCapabilities( + reasoning_effort_values=("low", "medium", "high"), + default_reasoning_effort="medium", + ) + eff = _as_map(effort_ladder("openai-compatible", caps)) + assert eff["none"] == "default" + assert eff["high"] == "high" + assert eff["xhigh"] == "medium" + + def test_adaptive_local_never_off(self) -> None: + caps = ModelCapabilities(thinking_mode="adaptive", thinking_param="enable_thinking") + eff = _as_map(effort_ladder("openai-compatible", caps)) + assert eff["none"] == "on" + assert eff["max"] == "on" + + +class TestNativeAnthropicLane: + def test_adaptive_with_effort_levels(self) -> None: + caps = ModelCapabilities( + thinking_mode="adaptive", + supports_effort=True, + effort_levels=("low", "medium", "high", "xhigh", "max"), + ) + eff = _as_map(effort_ladder("anthropic", caps)) + assert eff["none"] == "adaptive" # thinking on, model decides + assert eff["minimal"] == "adaptive" # unmapped knob level + assert eff["low"] == "low" + assert eff["max"] == "max" + + def test_manual_budget_ladder(self) -> None: + caps = ModelCapabilities(thinking_mode="manual") + eff = _as_map(effort_ladder("anthropic", caps)) + assert eff["none"] == "off" + assert eff["low"] == "budget:1024" + assert eff["medium"] == "budget:4096" + assert eff["high"] == "budget:16384" + # Off-map knob levels share the default budget — aliased group. + assert eff["minimal"] == eff["xhigh"] == eff["max"] == "budget:4096" + + +class TestFlatParamLanes: + def test_google_default_caps(self) -> None: + eff = _as_map(effort_ladder_for_model("google", "gemini-3-flash", None)) + assert eff["none"] == "default" + assert eff["minimal"] == "minimal" + assert eff["high"] == "high" + assert eff["xhigh"] == eff["max"] == "high" + + def test_overrides_merge_and_unknown_keys_ignored(self) -> None: + eff = _as_map( + effort_ladder_for_model( + "google", + "gemini-3-flash", + {"reasoning_effort_values": [], "not_a_field": True}, + ) + ) + # Operator cleared the values → nothing effort-related is sent. + assert set(eff.values()) == {"default"} diff --git a/turnstone/console/server.py b/turnstone/console/server.py index 290b1b4a..d39b6842 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -1908,8 +1908,26 @@ async def list_available_models(request: Request) -> JSONResponse: return err rows = storage.list_model_definitions(enabled_only=True) - # Only expose alias/model/provider — rows also contain api_key, base_url, etc. - models = [{"alias": r["alias"], "model": r["model"], "provider": r["provider"]} for r in rows] + # Only expose alias/model/provider (+ the derived effort ladder) — + # rows also contain api_key, base_url, etc. + from turnstone.core.providers.effort_ladder import effort_ladder_for_model + + models = [] + for r in rows: + entry: dict[str, Any] = { + "alias": r["alias"], + "model": r["model"], + "provider": r["provider"], + } + try: + entry["effort_ladder"] = effort_ladder_for_model( + r["provider"], r["model"], r.get("capabilities") or {} + ) + except Exception: + # Unknown provider string / malformed capabilities row must + # not take down the picker — the ladder is an annotation. + log.debug("models.effort_ladder_failed alias=%s", r.get("alias"), exc_info=True) + models.append(entry) # Include effective defaults for clients (web UI, channel gateway). default_alias = "" @@ -11592,6 +11610,47 @@ async def admin_model_capabilities(request: Request) -> JSONResponse: ) +async def admin_effort_ladder(request: Request) -> JSONResponse: + """POST /v1/api/admin/models/effort-ladder — knob→wire projection. + + Body: ``{"provider": ..., "model": ..., "capabilities": {...}}`` — + capabilities are the (possibly unsaved) overrides from the model + form, so the modal can annotate its effort select live while the + operator edits thinking mode / effort param. Pure computation; no + stored state is read or written. + """ + from turnstone.core.auth import require_permission + from turnstone.core.providers.effort_ladder import effort_ladder_for_model + + err = require_permission(request, "admin.models") + if err: + return err + + try: + body = await request.json() + except Exception: + return JSONResponse({"error": "invalid JSON body"}, status_code=400) + provider = str(body.get("provider") or "").strip() + model = str(body.get("model") or "").strip() + capabilities = body.get("capabilities") + + if provider not in _MODEL_PROVIDERS: + return JSONResponse({"error": f"Unknown provider: {provider!r}"}, status_code=400) + if not model: + return JSONResponse({"error": "model is required"}, status_code=400) + if capabilities is not None and not isinstance(capabilities, dict): + return JSONResponse({"error": "capabilities must be an object"}, status_code=400) + + try: + ladder = effort_ladder_for_model(provider, model, capabilities or {}) + except Exception: + # Garbage override values (wrong types for capability fields) + # surface as a clean 400 rather than a 500 — the form's raw + # JSON is operator-typed. + return JSONResponse({"error": "could not resolve capabilities"}, status_code=400) + return JSONResponse({"provider": provider, "model": model, "ladder": ladder}) + + async def admin_known_models(request: Request) -> JSONResponse: """GET /v1/api/admin/model-capabilities/known — list known model name prefixes.""" from turnstone.core.auth import require_permission @@ -13988,6 +14047,11 @@ def create_app( "/api/admin/model-capabilities/known", admin_known_models, ), + Route( + "/api/admin/models/effort-ladder", + admin_effort_ladder, + methods=["POST"], + ), # Governance: Prompt Policies Route("/api/admin/prompt-policies", admin_list_prompt_policies), Route( diff --git a/turnstone/console/static/admin.js b/turnstone/console/static/admin.js index 9c1a09cf..bba01d89 100644 --- a/turnstone/console/static/admin.js +++ b/turnstone/console/static/admin.js @@ -6905,6 +6905,7 @@ function showCreateModelModal() { document.getElementById("model-thinking-param").value = ""; document.getElementById("model-thinking-param-row").hidden = true; document.getElementById("model-effort-param").value = ""; + _annotateEffortSelect(document.getElementById("model-reasoning-effort"), null); document.getElementById("model-extra-body").value = ""; document.getElementById("model-capabilities").value = ""; // Clear validation error styling from prior submit attempts @@ -7033,6 +7034,7 @@ function showEditModelModal(definitionId) { }); _modelRenderTiles(); _modelCapsRefreshBaseline(); + _scheduleEffortLadder(); // Remove structured fields from capabilities display — only delete // thinking_mode/thinking_param when the UI successfully captured them. delete capsObj.server_compat; @@ -7625,6 +7627,99 @@ let _modelCapsSeq = 0; function _onModelFieldChange() { clearTimeout(_capsTimer); _capsTimer = setTimeout(_modelCapsRefreshBaseline, 500); + _scheduleEffortLadder(); +} + +/* Effort-ladder annotation: label knob positions that alias to the same + wire behavior for a given model, from the server-computed projection + (providers/effort_ladder.py — equal "effective" tokens ⇒ identical + requests). Defined here and shared as a page global with + governance.js (skill launch config), which loads after this file. */ +function _annotateEffortSelect(sel, ladder) { + if (!sel) return; + const byVal = {}; + const firstOf = {}; + (ladder || []).forEach(function (row) { + byVal[row.value] = row.effective; + if (!(row.effective in firstOf)) firstOf[row.effective] = row.value; + }); + Array.from(sel.options).forEach(function (opt) { + if (!opt.value) return; // "" = inherit-the-default option + if (!opt.dataset.baseLabel) opt.dataset.baseLabel = opt.textContent; + let label = opt.dataset.baseLabel; + let title = ""; + const eff = byVal[opt.value]; + if (eff !== undefined) { + const first = firstOf[eff]; + if (first !== opt.value) { + label += " (= " + first + ")"; + } else if (eff === "default") { + label += " (model default)"; + } + title = "sends: " + eff; + } + opt.textContent = label; + opt.title = title; + }); +} + +let _effortLadderTimer = null; +let _effortLadderSeq = 0; +function _scheduleEffortLadder() { + clearTimeout(_effortLadderTimer); + _effortLadderTimer = setTimeout(_refreshModelEffortLadder, 500); +} +function _refreshModelEffortLadder() { + const shelf = document.getElementById("model-shelf"); + const sel = document.getElementById("model-reasoning-effort"); + if (!shelf || !shelf.open || !sel) return; + const provider = document.getElementById("model-provider").value; + const modelName = document.getElementById("model-name").value.trim(); + if (!modelName) { + _annotateEffortSelect(sel, null); + return; + } + // Assemble the same capabilities the save path would persist: raw + // JSON base, structured thinking/effort fields overlaid. Mid-edit + // invalid JSON annotates from the structured fields alone. + let caps = {}; + const rawText = document.getElementById("model-capabilities").value.trim(); + if (rawText) { + try { + const parsed = JSON.parse(rawText); + if (_isPlainObject(parsed)) caps = parsed; + } catch (e) { + /* fall through */ + } + } + const tm = document.getElementById("model-thinking-mode").value; + if (tm) { + caps.thinking_mode = tm; + const tp = document.getElementById("model-thinking-param").value; + if (tp) caps.thinking_param = tp; + } + const ep = document.getElementById("model-effort-param").value.trim(); + if (ep) caps.effort_param = ep; + const seq = ++_effortLadderSeq; + authFetch("/v1/api/admin/models/effort-ladder", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + provider: provider, + model: modelName, + capabilities: caps, + }), + }) + .then(function (r) { + return r.ok ? r.json() : null; + }) + .then(function (d) { + if (seq !== _effortLadderSeq) return; // superseded by a newer edit + _annotateEffortSelect(sel, d && d.ladder); + }) + .catch(function () { + /* silent — annotation only */ + }); } /* Known-model lookup feeding the tile matrix: the table becomes the display @@ -7771,6 +7866,13 @@ function _refreshModelSuggestions() { const provEl = document.getElementById("model-provider"); const tmEl = document.getElementById("model-thinking-mode"); if (tmEl) tmEl.addEventListener("change", _toggleThinkingParam); + if (tmEl) tmEl.addEventListener("change", _scheduleEffortLadder); + ["model-thinking-param", "model-effort-param", "model-capabilities"].forEach( + function (id) { + const el = document.getElementById(id); + if (el) el.addEventListener("input", _scheduleEffortLadder); + }, + ); const grid = document.getElementById("model-capgrid"); if (grid) { grid.addEventListener("change", function (e) { diff --git a/turnstone/console/static/governance.js b/turnstone/console/static/governance.js index 88134d8c..6f62ffa2 100644 --- a/turnstone/console/static/governance.js +++ b/turnstone/console/static/governance.js @@ -1965,6 +1965,7 @@ function showEditTemplateModal(tmplId) { document.getElementById("skl-default").checked = tmpl.is_default; // Session config fields document.getElementById("sklc-model").value = tmpl.model || ""; + _sklcScheduleEffortLadder(); document.getElementById("sklc-temperature").value = tmpl.temperature != null ? tmpl.temperature : ""; document.getElementById("sklc-reasoning-effort").value = @@ -5044,3 +5045,43 @@ function _submitOGPShelf() { errEl.classList.add("is-visible"); }); } + +/* Effort-ladder annotation for the skill launch-config effort select. + Resolves the typed alias against /v1/api/models (each row carries a + server-computed effort_ladder) and reuses the page-global + _annotateEffortSelect from admin.js, which loads before this file. */ +let _sklcModelsPromise = null; +let _sklcLadderTimer = null; +function _sklcRefreshEffortLadder() { + const sel = document.getElementById("sklc-reasoning-effort"); + const aliasEl = document.getElementById("sklc-model"); + if (!sel || !aliasEl || typeof _annotateEffortSelect !== "function") return; + const alias = aliasEl.value.trim(); + if (!alias) { + _annotateEffortSelect(sel, null); + return; + } + if (!_sklcModelsPromise) { + _sklcModelsPromise = authFetch("/v1/api/models").then(function (r) { + return r.json(); + }); + } + _sklcModelsPromise + .then(function (d) { + const row = (d.models || []).find(function (m) { + return m.alias === alias; + }); + _annotateEffortSelect(sel, row ? row.effort_ladder : null); + }) + .catch(function () { + /* silent — annotation only */ + }); +} +function _sklcScheduleEffortLadder() { + clearTimeout(_sklcLadderTimer); + _sklcLadderTimer = setTimeout(_sklcRefreshEffortLadder, 400); +} +(function () { + const el = document.getElementById("sklc-model"); + if (el) el.addEventListener("input", _sklcScheduleEffortLadder); +})(); diff --git a/turnstone/core/providers/_anthropic.py b/turnstone/core/providers/_anthropic.py index d3432597..4975d550 100644 --- a/turnstone/core/providers/_anthropic.py +++ b/turnstone/core/providers/_anthropic.py @@ -78,6 +78,12 @@ _TOOL_SEARCH_TOOL_TYPE = "tool_search_tool_bm25" # bodies byte-identical when a caller threads thinking overrides. _INTERNAL_EXTRA_PARAMS = frozenset({"thinking_budget_tokens"}) +# Manual-mode thinking budgets per effort knob level; knob values outside +# the map (minimal, xhigh, max) fall back to the default budget. Shared +# with ``effort_ladder`` so UI projections can't drift from the wire. +_EFFORT_BUDGET_MAP = {"low": 1024, "medium": 4096, "high": 16384} +_DEFAULT_THINKING_BUDGET = 4096 + # -- model capabilities ------------------------------------------------------- _ANTHROPIC_DEFAULT = ModelCapabilities( @@ -778,8 +784,7 @@ class AnthropicProvider: """ if not reasoning_effort or reasoning_effort in ("none", ""): return {} - budget_map = {"low": 1024, "medium": 4096, "high": 16384} - budget = budget_map.get(reasoning_effort, 4096) + budget = _EFFORT_BUDGET_MAP.get(reasoning_effort, _DEFAULT_THINKING_BUDGET) if extra_params and "thinking_budget_tokens" in extra_params: budget = extra_params["thinking_budget_tokens"] if budget > 0: diff --git a/turnstone/core/providers/effort_ladder.py b/turnstone/core/providers/effort_ladder.py new file mode 100644 index 00000000..5a8fde36 --- /dev/null +++ b/turnstone/core/providers/effort_ladder.py @@ -0,0 +1,118 @@ +"""Effective effort-ladder projection — what each knob position actually does. + +The session reasoning-effort knob has seven positions; how many are +DISTINCT depends on the provider lane and the model's capabilities +(qwen3.6: two — off/on; a freeform ``effort_param`` model: one per +forwarded value; Claude with effort levels: one per level plus the +adaptive fallback). This module projects the knob domain through the +same mapping functions the providers use at request time, so UIs can +annotate knob positions that alias to identical wire behavior instead +of presenting seven positions as seven behaviors. + +Pure computation — no network, no provider clients. Labels are short +comparable tokens: two knob positions with the same ``effective`` string +produce the same request. ``"default"`` means nothing effort-related is +sent (the server/model default applies); ``"off"``/``"on"`` describe the +local-lane template toggle. The ladder describes what Turnstone SENDS — +a server-side template may alias further (e.g. DeepSeek-V4 maps +``low``/``medium`` to its default ``high`` tier). +""" + +from __future__ import annotations + +import dataclasses +from typing import Any + +from turnstone.core.providers._anthropic import ( + _DEFAULT_THINKING_BUDGET, + _EFFORT_BUDGET_MAP, + _map_reasoning_to_effort, +) +from turnstone.core.providers._protocol import ( + ModelCapabilities, + reasoning_template_kwargs, + resolve_reasoning_effort, +) + +# The full session-knob domain, in ladder order (mirrors the console +# selects; "" rides as the caller's alias for its default and is not a +# ladder row). +KNOB_VALUES: tuple[str, ...] = ("none", "minimal", "low", "medium", "high", "xhigh", "max") + +# Lanes whose reasoning control rides chat_template_kwargs +# (merge_reasoning_template_kwargs) rather than native params. +_LOCAL_LANES = frozenset({"openai-compatible", "anthropic-compatible"}) + + +def effort_ladder(provider_name: str, caps: ModelCapabilities) -> list[dict[str, str]]: + """Project every knob position to its effective wire behavior. + + Returns ``[{"value": , "effective": }, ...]`` in knob + order. Equal ``effective`` tokens ⇒ identical requests. + """ + return [ + {"value": knob, "effective": _effective(provider_name, caps, knob)} for knob in KNOB_VALUES + ] + + +def effort_ladder_for_model( + provider_name: str, + model: str, + capability_overrides: dict[str, Any] | None, +) -> list[dict[str, str]]: + """Ladder for a stored model row: provider defaults + operator overrides. + + Mirrors ``ChatSession._resolve_capabilities`` — overrides are + field-filtered and applied over the provider's per-model defaults. + """ + from turnstone.core.providers import create_provider + + caps = create_provider(provider_name).get_capabilities(model) + if capability_overrides: + fields = {f.name for f in dataclasses.fields(type(caps))} + overrides = {k: v for k, v in capability_overrides.items() if k in fields} + if overrides: + caps = dataclasses.replace(caps, **overrides) + return effort_ladder(provider_name, caps) + + +def _effective(provider_name: str, caps: ModelCapabilities, knob: str) -> str: + if provider_name == "anthropic": + return _effective_anthropic(caps, knob) + if provider_name in _LOCAL_LANES: + return _effective_local(provider_name, caps, knob) + # Flat-param lanes: openai (chat + responses), google, xai. + return resolve_reasoning_effort(caps, knob) or "default" + + +def _effective_anthropic(caps: ModelCapabilities, knob: str) -> str: + """Native lane — mirrors ``_build_thinking_and_kwargs``.""" + effort = _map_reasoning_to_effort(knob, caps.effort_levels) if caps.supports_effort else None + if caps.thinking_mode == "adaptive": + # Thinking always on, model self-regulates; effort rides + # output_config when the knob maps onto a supported level. + return effort or "adaptive" + if caps.thinking_mode == "manual": + if not knob or knob == "none": + return "off" + budget = _EFFORT_BUDGET_MAP.get(knob, _DEFAULT_THINKING_BUDGET) + return f"{effort}·budget:{budget}" if effort else f"budget:{budget}" + return "default" + + +def _effective_local(provider_name: str, caps: ModelCapabilities, knob: str) -> str: + """Local lanes — mirrors ``merge_reasoning_template_kwargs`` (+ flat param).""" + updates = reasoning_template_kwargs(caps, knob) + parts: list[str] = [] + if caps.thinking_param in updates: + parts.append("on" if updates[caps.thinking_param] else "off") + if caps.effort_param and caps.effort_param in updates: + parts.append(str(updates[caps.effort_param])) + # openai-compatible additionally sends the flat param when no + # effort_param declares the template channel (suppression rule in + # apply_temperature_and_effort). + if provider_name == "openai-compatible" and not caps.effort_param: + flat = resolve_reasoning_effort(caps, knob) + if flat: + parts.append(flat) + return "+".join(parts) if parts else "default"