feat(console): surface each model's effective effort ladder

Seven knob positions render as seven behaviors in the UI, but the real
ladder depends on the lane and the model: qwen3.6 has two (off/on),
DeepSeek-V4 three, Claude 4.6 five. Operators had no way to see which
positions alias — the confusion class behind silently-equal effort
levels.

providers/effort_ladder.py projects the knob domain through the same
mapping functions the providers use at request time (resolve_reasoning_
effort, reasoning_template_kwargs, the manual budget map — hoisted to a
shared constant so the projection can't drift), yielding
{value, effective} rows where equal tokens promise identical requests.
/v1/api/models rows now carry the ladder (guarded per row), and
POST /v1/api/admin/models/effort-ladder computes it for the admin
modal's unsaved edits.

The admin per-model effort select and the skill launch-config effort
select annotate aliased positions ("Max (= high)", "None (model
default)") with a sends-tooltip; annotations refresh as thinking-mode /
effort-param / capabilities fields change. The ladder describes what
Turnstone sends — server-side templates may alias further (DeepSeek-V4
folds low/medium into its default high tier).
This commit is contained in:
Patrick Buckley
2026-07-04 01:32:33 -07:00
parent d7941c88be
commit 59a527f2f2
7 changed files with 459 additions and 4 deletions
+10
View File
@@ -852,6 +852,16 @@ both local-server lanes, so `thinking_mode`/`thinking_param`/
`effort_param` mean the same thing whichever endpoint serves the model.
Only the Responses API surface (native reasoning) ignores it.
The console surfaces this projection as an *effective effort ladder*:
the admin model form's per-model effort select and the skill
launch-config effort select annotate knob positions that alias to the
same wire behavior (e.g. "Max (= high)"), computed server-side by
`providers/effort_ladder.py` from the same mapping functions the
providers use at request time and shipped on `/v1/api/models` rows and
`POST /v1/api/admin/models/effort-ladder`. The ladder describes what
Turnstone sends — a server-side template may alias further (DeepSeek-V4
folds `low`/`medium` into its default `high` tier).
The `anthropic-compatible` lane never sends Anthropic's native
`thinking`/`output_config` params — they are not in vLLM's request
schema. The real `anthropic` provider is unaffected: official Claude
+115
View File
@@ -0,0 +1,115 @@
"""Tests for the effective effort-ladder projection.
The ladder must mirror the request-time mapping functions exactly —
equal ``effective`` tokens promise byte-identical effort behavior on
the wire, which is what the UI annotations lean on.
"""
from __future__ import annotations
from turnstone.core.providers._protocol import ModelCapabilities
from turnstone.core.providers.effort_ladder import (
KNOB_VALUES,
effort_ladder,
effort_ladder_for_model,
)
def _as_map(ladder: list[dict[str, str]]) -> dict[str, str]:
assert [r["value"] for r in ladder] == list(KNOB_VALUES)
return {r["value"]: r["effective"] for r in ladder}
class TestLocalLanes:
def test_qwen_style_toggle_only_two_groups(self) -> None:
"""Toggle-only model: none=off, everything else one 'on' group."""
caps = ModelCapabilities(thinking_mode="manual", thinking_param="enable_thinking")
eff = _as_map(effort_ladder("anthropic-compatible", caps))
assert eff["none"] == "off"
assert {eff[k] for k in KNOB_VALUES if k != "none"} == {"on"}
def test_freeform_effort_param_forwards_each_value(self) -> None:
"""deepseek-style config: toggle + verbatim effort per position."""
caps = ModelCapabilities(
thinking_mode="manual",
thinking_param="thinking",
effort_param="reasoning_effort",
)
eff = _as_map(effort_ladder("anthropic-compatible", caps))
assert eff["none"] == "off"
assert eff["low"] == "on+low"
assert eff["max"] == "on+max"
def test_validated_effort_param_shows_snapping(self) -> None:
"""Declared values collapse off-list positions onto the default."""
caps = ModelCapabilities(
thinking_mode="manual",
thinking_param="enable_thinking",
effort_param="reasoning_effort",
reasoning_effort_values=("low", "medium", "high"),
default_reasoning_effort="medium",
)
eff = _as_map(effort_ladder("openai-compatible", caps))
assert eff["xhigh"] == "on+medium"
assert eff["max"] == "on+medium"
assert eff["high"] == "on+high"
def test_openai_compatible_flat_param_without_effort_param(self) -> None:
caps = ModelCapabilities(
reasoning_effort_values=("low", "medium", "high"),
default_reasoning_effort="medium",
)
eff = _as_map(effort_ladder("openai-compatible", caps))
assert eff["none"] == "default"
assert eff["high"] == "high"
assert eff["xhigh"] == "medium"
def test_adaptive_local_never_off(self) -> None:
caps = ModelCapabilities(thinking_mode="adaptive", thinking_param="enable_thinking")
eff = _as_map(effort_ladder("openai-compatible", caps))
assert eff["none"] == "on"
assert eff["max"] == "on"
class TestNativeAnthropicLane:
def test_adaptive_with_effort_levels(self) -> None:
caps = ModelCapabilities(
thinking_mode="adaptive",
supports_effort=True,
effort_levels=("low", "medium", "high", "xhigh", "max"),
)
eff = _as_map(effort_ladder("anthropic", caps))
assert eff["none"] == "adaptive" # thinking on, model decides
assert eff["minimal"] == "adaptive" # unmapped knob level
assert eff["low"] == "low"
assert eff["max"] == "max"
def test_manual_budget_ladder(self) -> None:
caps = ModelCapabilities(thinking_mode="manual")
eff = _as_map(effort_ladder("anthropic", caps))
assert eff["none"] == "off"
assert eff["low"] == "budget:1024"
assert eff["medium"] == "budget:4096"
assert eff["high"] == "budget:16384"
# Off-map knob levels share the default budget — aliased group.
assert eff["minimal"] == eff["xhigh"] == eff["max"] == "budget:4096"
class TestFlatParamLanes:
def test_google_default_caps(self) -> None:
eff = _as_map(effort_ladder_for_model("google", "gemini-3-flash", None))
assert eff["none"] == "default"
assert eff["minimal"] == "minimal"
assert eff["high"] == "high"
assert eff["xhigh"] == eff["max"] == "high"
def test_overrides_merge_and_unknown_keys_ignored(self) -> None:
eff = _as_map(
effort_ladder_for_model(
"google",
"gemini-3-flash",
{"reasoning_effort_values": [], "not_a_field": True},
)
)
# Operator cleared the values → nothing effort-related is sent.
assert set(eff.values()) == {"default"}
+66 -2
View File
@@ -1908,8 +1908,26 @@ async def list_available_models(request: Request) -> JSONResponse:
return err
rows = storage.list_model_definitions(enabled_only=True)
# Only expose alias/model/provider — rows also contain api_key, base_url, etc.
models = [{"alias": r["alias"], "model": r["model"], "provider": r["provider"]} for r in rows]
# Only expose alias/model/provider (+ the derived effort ladder) —
# rows also contain api_key, base_url, etc.
from turnstone.core.providers.effort_ladder import effort_ladder_for_model
models = []
for r in rows:
entry: dict[str, Any] = {
"alias": r["alias"],
"model": r["model"],
"provider": r["provider"],
}
try:
entry["effort_ladder"] = effort_ladder_for_model(
r["provider"], r["model"], r.get("capabilities") or {}
)
except Exception:
# Unknown provider string / malformed capabilities row must
# not take down the picker — the ladder is an annotation.
log.debug("models.effort_ladder_failed alias=%s", r.get("alias"), exc_info=True)
models.append(entry)
# Include effective defaults for clients (web UI, channel gateway).
default_alias = ""
@@ -11592,6 +11610,47 @@ async def admin_model_capabilities(request: Request) -> JSONResponse:
)
async def admin_effort_ladder(request: Request) -> JSONResponse:
"""POST /v1/api/admin/models/effort-ladder — knob→wire projection.
Body: ``{"provider": ..., "model": ..., "capabilities": {...}}``
capabilities are the (possibly unsaved) overrides from the model
form, so the modal can annotate its effort select live while the
operator edits thinking mode / effort param. Pure computation; no
stored state is read or written.
"""
from turnstone.core.auth import require_permission
from turnstone.core.providers.effort_ladder import effort_ladder_for_model
err = require_permission(request, "admin.models")
if err:
return err
try:
body = await request.json()
except Exception:
return JSONResponse({"error": "invalid JSON body"}, status_code=400)
provider = str(body.get("provider") or "").strip()
model = str(body.get("model") or "").strip()
capabilities = body.get("capabilities")
if provider not in _MODEL_PROVIDERS:
return JSONResponse({"error": f"Unknown provider: {provider!r}"}, status_code=400)
if not model:
return JSONResponse({"error": "model is required"}, status_code=400)
if capabilities is not None and not isinstance(capabilities, dict):
return JSONResponse({"error": "capabilities must be an object"}, status_code=400)
try:
ladder = effort_ladder_for_model(provider, model, capabilities or {})
except Exception:
# Garbage override values (wrong types for capability fields)
# surface as a clean 400 rather than a 500 — the form's raw
# JSON is operator-typed.
return JSONResponse({"error": "could not resolve capabilities"}, status_code=400)
return JSONResponse({"provider": provider, "model": model, "ladder": ladder})
async def admin_known_models(request: Request) -> JSONResponse:
"""GET /v1/api/admin/model-capabilities/known — list known model name prefixes."""
from turnstone.core.auth import require_permission
@@ -13988,6 +14047,11 @@ def create_app(
"/api/admin/model-capabilities/known",
admin_known_models,
),
Route(
"/api/admin/models/effort-ladder",
admin_effort_ladder,
methods=["POST"],
),
# Governance: Prompt Policies
Route("/api/admin/prompt-policies", admin_list_prompt_policies),
Route(
+102
View File
@@ -6905,6 +6905,7 @@ function showCreateModelModal() {
document.getElementById("model-thinking-param").value = "";
document.getElementById("model-thinking-param-row").hidden = true;
document.getElementById("model-effort-param").value = "";
_annotateEffortSelect(document.getElementById("model-reasoning-effort"), null);
document.getElementById("model-extra-body").value = "";
document.getElementById("model-capabilities").value = "";
// Clear validation error styling from prior submit attempts
@@ -7033,6 +7034,7 @@ function showEditModelModal(definitionId) {
});
_modelRenderTiles();
_modelCapsRefreshBaseline();
_scheduleEffortLadder();
// Remove structured fields from capabilities display — only delete
// thinking_mode/thinking_param when the UI successfully captured them.
delete capsObj.server_compat;
@@ -7625,6 +7627,99 @@ let _modelCapsSeq = 0;
function _onModelFieldChange() {
clearTimeout(_capsTimer);
_capsTimer = setTimeout(_modelCapsRefreshBaseline, 500);
_scheduleEffortLadder();
}
/* Effort-ladder annotation: label knob positions that alias to the same
wire behavior for a given model, from the server-computed projection
(providers/effort_ladder.py equal "effective" tokens identical
requests). Defined here and shared as a page global with
governance.js (skill launch config), which loads after this file. */
function _annotateEffortSelect(sel, ladder) {
if (!sel) return;
const byVal = {};
const firstOf = {};
(ladder || []).forEach(function (row) {
byVal[row.value] = row.effective;
if (!(row.effective in firstOf)) firstOf[row.effective] = row.value;
});
Array.from(sel.options).forEach(function (opt) {
if (!opt.value) return; // "" = inherit-the-default option
if (!opt.dataset.baseLabel) opt.dataset.baseLabel = opt.textContent;
let label = opt.dataset.baseLabel;
let title = "";
const eff = byVal[opt.value];
if (eff !== undefined) {
const first = firstOf[eff];
if (first !== opt.value) {
label += " (= " + first + ")";
} else if (eff === "default") {
label += " (model default)";
}
title = "sends: " + eff;
}
opt.textContent = label;
opt.title = title;
});
}
let _effortLadderTimer = null;
let _effortLadderSeq = 0;
function _scheduleEffortLadder() {
clearTimeout(_effortLadderTimer);
_effortLadderTimer = setTimeout(_refreshModelEffortLadder, 500);
}
function _refreshModelEffortLadder() {
const shelf = document.getElementById("model-shelf");
const sel = document.getElementById("model-reasoning-effort");
if (!shelf || !shelf.open || !sel) return;
const provider = document.getElementById("model-provider").value;
const modelName = document.getElementById("model-name").value.trim();
if (!modelName) {
_annotateEffortSelect(sel, null);
return;
}
// Assemble the same capabilities the save path would persist: raw
// JSON base, structured thinking/effort fields overlaid. Mid-edit
// invalid JSON annotates from the structured fields alone.
let caps = {};
const rawText = document.getElementById("model-capabilities").value.trim();
if (rawText) {
try {
const parsed = JSON.parse(rawText);
if (_isPlainObject(parsed)) caps = parsed;
} catch (e) {
/* fall through */
}
}
const tm = document.getElementById("model-thinking-mode").value;
if (tm) {
caps.thinking_mode = tm;
const tp = document.getElementById("model-thinking-param").value;
if (tp) caps.thinking_param = tp;
}
const ep = document.getElementById("model-effort-param").value.trim();
if (ep) caps.effort_param = ep;
const seq = ++_effortLadderSeq;
authFetch("/v1/api/admin/models/effort-ladder", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
provider: provider,
model: modelName,
capabilities: caps,
}),
})
.then(function (r) {
return r.ok ? r.json() : null;
})
.then(function (d) {
if (seq !== _effortLadderSeq) return; // superseded by a newer edit
_annotateEffortSelect(sel, d && d.ladder);
})
.catch(function () {
/* silent — annotation only */
});
}
/* Known-model lookup feeding the tile matrix: the table becomes the display
@@ -7771,6 +7866,13 @@ function _refreshModelSuggestions() {
const provEl = document.getElementById("model-provider");
const tmEl = document.getElementById("model-thinking-mode");
if (tmEl) tmEl.addEventListener("change", _toggleThinkingParam);
if (tmEl) tmEl.addEventListener("change", _scheduleEffortLadder);
["model-thinking-param", "model-effort-param", "model-capabilities"].forEach(
function (id) {
const el = document.getElementById(id);
if (el) el.addEventListener("input", _scheduleEffortLadder);
},
);
const grid = document.getElementById("model-capgrid");
if (grid) {
grid.addEventListener("change", function (e) {
+41
View File
@@ -1965,6 +1965,7 @@ function showEditTemplateModal(tmplId) {
document.getElementById("skl-default").checked = tmpl.is_default;
// Session config fields
document.getElementById("sklc-model").value = tmpl.model || "";
_sklcScheduleEffortLadder();
document.getElementById("sklc-temperature").value =
tmpl.temperature != null ? tmpl.temperature : "";
document.getElementById("sklc-reasoning-effort").value =
@@ -5044,3 +5045,43 @@ function _submitOGPShelf() {
errEl.classList.add("is-visible");
});
}
/* Effort-ladder annotation for the skill launch-config effort select.
Resolves the typed alias against /v1/api/models (each row carries a
server-computed effort_ladder) and reuses the page-global
_annotateEffortSelect from admin.js, which loads before this file. */
let _sklcModelsPromise = null;
let _sklcLadderTimer = null;
function _sklcRefreshEffortLadder() {
const sel = document.getElementById("sklc-reasoning-effort");
const aliasEl = document.getElementById("sklc-model");
if (!sel || !aliasEl || typeof _annotateEffortSelect !== "function") return;
const alias = aliasEl.value.trim();
if (!alias) {
_annotateEffortSelect(sel, null);
return;
}
if (!_sklcModelsPromise) {
_sklcModelsPromise = authFetch("/v1/api/models").then(function (r) {
return r.json();
});
}
_sklcModelsPromise
.then(function (d) {
const row = (d.models || []).find(function (m) {
return m.alias === alias;
});
_annotateEffortSelect(sel, row ? row.effort_ladder : null);
})
.catch(function () {
/* silent — annotation only */
});
}
function _sklcScheduleEffortLadder() {
clearTimeout(_sklcLadderTimer);
_sklcLadderTimer = setTimeout(_sklcRefreshEffortLadder, 400);
}
(function () {
const el = document.getElementById("sklc-model");
if (el) el.addEventListener("input", _sklcScheduleEffortLadder);
})();
+7 -2
View File
@@ -78,6 +78,12 @@ _TOOL_SEARCH_TOOL_TYPE = "tool_search_tool_bm25"
# bodies byte-identical when a caller threads thinking overrides.
_INTERNAL_EXTRA_PARAMS = frozenset({"thinking_budget_tokens"})
# Manual-mode thinking budgets per effort knob level; knob values outside
# the map (minimal, xhigh, max) fall back to the default budget. Shared
# with ``effort_ladder`` so UI projections can't drift from the wire.
_EFFORT_BUDGET_MAP = {"low": 1024, "medium": 4096, "high": 16384}
_DEFAULT_THINKING_BUDGET = 4096
# -- model capabilities -------------------------------------------------------
_ANTHROPIC_DEFAULT = ModelCapabilities(
@@ -778,8 +784,7 @@ class AnthropicProvider:
"""
if not reasoning_effort or reasoning_effort in ("none", ""):
return {}
budget_map = {"low": 1024, "medium": 4096, "high": 16384}
budget = budget_map.get(reasoning_effort, 4096)
budget = _EFFORT_BUDGET_MAP.get(reasoning_effort, _DEFAULT_THINKING_BUDGET)
if extra_params and "thinking_budget_tokens" in extra_params:
budget = extra_params["thinking_budget_tokens"]
if budget > 0:
+118
View File
@@ -0,0 +1,118 @@
"""Effective effort-ladder projection — what each knob position actually does.
The session reasoning-effort knob has seven positions; how many are
DISTINCT depends on the provider lane and the model's capabilities
(qwen3.6: two — off/on; a freeform ``effort_param`` model: one per
forwarded value; Claude with effort levels: one per level plus the
adaptive fallback). This module projects the knob domain through the
same mapping functions the providers use at request time, so UIs can
annotate knob positions that alias to identical wire behavior instead
of presenting seven positions as seven behaviors.
Pure computation — no network, no provider clients. Labels are short
comparable tokens: two knob positions with the same ``effective`` string
produce the same request. ``"default"`` means nothing effort-related is
sent (the server/model default applies); ``"off"``/``"on"`` describe the
local-lane template toggle. The ladder describes what Turnstone SENDS —
a server-side template may alias further (e.g. DeepSeek-V4 maps
``low``/``medium`` to its default ``high`` tier).
"""
from __future__ import annotations
import dataclasses
from typing import Any
from turnstone.core.providers._anthropic import (
_DEFAULT_THINKING_BUDGET,
_EFFORT_BUDGET_MAP,
_map_reasoning_to_effort,
)
from turnstone.core.providers._protocol import (
ModelCapabilities,
reasoning_template_kwargs,
resolve_reasoning_effort,
)
# The full session-knob domain, in ladder order (mirrors the console
# selects; "" rides as the caller's alias for its default and is not a
# ladder row).
KNOB_VALUES: tuple[str, ...] = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
# Lanes whose reasoning control rides chat_template_kwargs
# (merge_reasoning_template_kwargs) rather than native params.
_LOCAL_LANES = frozenset({"openai-compatible", "anthropic-compatible"})
def effort_ladder(provider_name: str, caps: ModelCapabilities) -> list[dict[str, str]]:
"""Project every knob position to its effective wire behavior.
Returns ``[{"value": <knob>, "effective": <token>}, ...]`` in knob
order. Equal ``effective`` tokens ⇒ identical requests.
"""
return [
{"value": knob, "effective": _effective(provider_name, caps, knob)} for knob in KNOB_VALUES
]
def effort_ladder_for_model(
provider_name: str,
model: str,
capability_overrides: dict[str, Any] | None,
) -> list[dict[str, str]]:
"""Ladder for a stored model row: provider defaults + operator overrides.
Mirrors ``ChatSession._resolve_capabilities`` — overrides are
field-filtered and applied over the provider's per-model defaults.
"""
from turnstone.core.providers import create_provider
caps = create_provider(provider_name).get_capabilities(model)
if capability_overrides:
fields = {f.name for f in dataclasses.fields(type(caps))}
overrides = {k: v for k, v in capability_overrides.items() if k in fields}
if overrides:
caps = dataclasses.replace(caps, **overrides)
return effort_ladder(provider_name, caps)
def _effective(provider_name: str, caps: ModelCapabilities, knob: str) -> str:
if provider_name == "anthropic":
return _effective_anthropic(caps, knob)
if provider_name in _LOCAL_LANES:
return _effective_local(provider_name, caps, knob)
# Flat-param lanes: openai (chat + responses), google, xai.
return resolve_reasoning_effort(caps, knob) or "default"
def _effective_anthropic(caps: ModelCapabilities, knob: str) -> str:
"""Native lane — mirrors ``_build_thinking_and_kwargs``."""
effort = _map_reasoning_to_effort(knob, caps.effort_levels) if caps.supports_effort else None
if caps.thinking_mode == "adaptive":
# Thinking always on, model self-regulates; effort rides
# output_config when the knob maps onto a supported level.
return effort or "adaptive"
if caps.thinking_mode == "manual":
if not knob or knob == "none":
return "off"
budget = _EFFORT_BUDGET_MAP.get(knob, _DEFAULT_THINKING_BUDGET)
return f"{effort}·budget:{budget}" if effort else f"budget:{budget}"
return "default"
def _effective_local(provider_name: str, caps: ModelCapabilities, knob: str) -> str:
"""Local lanes — mirrors ``merge_reasoning_template_kwargs`` (+ flat param)."""
updates = reasoning_template_kwargs(caps, knob)
parts: list[str] = []
if caps.thinking_param in updates:
parts.append("on" if updates[caps.thinking_param] else "off")
if caps.effort_param and caps.effort_param in updates:
parts.append(str(updates[caps.effort_param]))
# openai-compatible additionally sends the flat param when no
# effort_param declares the template channel (suppression rule in
# apply_temperature_and_effort).
if provider_name == "openai-compatible" and not caps.effort_param:
flat = resolve_reasoning_effort(caps, knob)
if flat:
parts.append(flat)
return "+".join(parts) if parts else "default"