mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-14 07:52:25 -06:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f290eb4880 |
@@ -48,7 +48,7 @@ jobs:
|
||||
id: detect
|
||||
run: |
|
||||
updates=()
|
||||
for lib in katex hljs mermaid hls; do
|
||||
for lib in katex hljs mermaid; do
|
||||
version=$(grep -oE "${lib}-[0-9.]+" pyproject.toml | head -1 | sed "s/${lib}-//")
|
||||
[[ -z "$version" ]] && continue
|
||||
[[ -d "turnstone/shared_static/${lib}-${version}" ]] && continue
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "1.4.0a2"
|
||||
version = "1.3.0"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
@@ -80,7 +80,7 @@ include = [
|
||||
"turnstone/shared_static/katex-0.16.45/**/*",
|
||||
"turnstone/shared_static/hljs-11.11.1/**/*",
|
||||
"turnstone/shared_static/mermaid-11.14.0/**/*",
|
||||
"turnstone/shared_static/hls-1.6.16/**/*",
|
||||
"turnstone/shared_static/hls-1.6.15/**/*",
|
||||
"turnstone/sdk/py.typed",
|
||||
"turnstone/deploy/*.yaml",
|
||||
]
|
||||
|
||||
@@ -152,52 +152,6 @@ class TestOpenAIProvider:
|
||||
def test_provider_name(self) -> None:
|
||||
assert self.provider.provider_name == "openai-compatible"
|
||||
|
||||
# -- _apply_thinking_mode -------------------------------------------------
|
||||
|
||||
def test_thinking_mode_none_does_nothing(self) -> None:
|
||||
"""No thinking params injected when thinking_mode is 'none'."""
|
||||
caps = ModelCapabilities(thinking_mode="none")
|
||||
extra_body: dict[str, Any] = {"chat_template_kwargs": {"reasoning_effort": "medium"}}
|
||||
OpenAIProvider._apply_thinking_mode(extra_body, caps)
|
||||
assert "enable_thinking" not in extra_body["chat_template_kwargs"]
|
||||
|
||||
def test_thinking_mode_manual_injects_param(self) -> None:
|
||||
"""Manual thinking mode injects enable_thinking into chat_template_kwargs."""
|
||||
caps = ModelCapabilities(thinking_mode="manual")
|
||||
extra_body: dict[str, Any] = {"chat_template_kwargs": {"reasoning_effort": "medium"}}
|
||||
OpenAIProvider._apply_thinking_mode(extra_body, caps)
|
||||
assert extra_body["chat_template_kwargs"]["enable_thinking"] is True
|
||||
assert extra_body["chat_template_kwargs"]["reasoning_effort"] == "medium"
|
||||
|
||||
def test_thinking_mode_custom_param(self) -> None:
|
||||
"""Custom thinking_param (e.g. Granite's 'thinking') is used."""
|
||||
caps = ModelCapabilities(thinking_mode="manual", thinking_param="thinking")
|
||||
extra_body: dict[str, Any] = {"chat_template_kwargs": {}}
|
||||
OpenAIProvider._apply_thinking_mode(extra_body, caps)
|
||||
assert extra_body["chat_template_kwargs"]["thinking"] is True
|
||||
assert "enable_thinking" not in extra_body["chat_template_kwargs"]
|
||||
|
||||
def test_thinking_mode_does_not_override_explicit(self) -> None:
|
||||
"""If operator explicitly set the param to False, provider respects it."""
|
||||
caps = ModelCapabilities(thinking_mode="manual")
|
||||
extra_body: dict[str, Any] = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
OpenAIProvider._apply_thinking_mode(extra_body, caps)
|
||||
assert extra_body["chat_template_kwargs"]["enable_thinking"] is False
|
||||
|
||||
def test_thinking_mode_creates_ctk_if_missing(self) -> None:
|
||||
"""Creates chat_template_kwargs dict if not present in extra_body."""
|
||||
caps = ModelCapabilities(thinking_mode="manual")
|
||||
extra_body: dict[str, Any] = {}
|
||||
OpenAIProvider._apply_thinking_mode(extra_body, caps)
|
||||
assert extra_body["chat_template_kwargs"]["enable_thinking"] is True
|
||||
|
||||
def test_thinking_mode_adaptive(self) -> None:
|
||||
"""Adaptive thinking mode also injects the param."""
|
||||
caps = ModelCapabilities(thinking_mode="adaptive")
|
||||
extra_body: dict[str, Any] = {"chat_template_kwargs": {}}
|
||||
OpenAIProvider._apply_thinking_mode(extra_body, caps)
|
||||
assert extra_body["chat_template_kwargs"]["enable_thinking"] is True
|
||||
|
||||
# -- _sanitize_messages ---------------------------------------------------
|
||||
|
||||
def test_sanitize_messages_none_content_no_tool_calls(self) -> None:
|
||||
|
||||
@@ -1,273 +0,0 @@
|
||||
"""Tests for turnstone.core.server_compat — profile suggestion and merging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from turnstone.core.providers._openai_chat import OpenAIChatCompletionsProvider
|
||||
from turnstone.core.providers._protocol import ModelCapabilities
|
||||
from turnstone.core.server_compat import merge_server_compat, suggest_profile
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# suggest_profile
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSuggestProfile:
|
||||
def test_vllm_gemma4(self) -> None:
|
||||
p = suggest_profile("vllm", "google/gemma-4-31B-it")
|
||||
assert p["capabilities"]["thinking_mode"] == "manual"
|
||||
assert p["capabilities"]["thinking_param"] == "enable_thinking"
|
||||
assert p["server_compat"]["extra_body"]["skip_special_tokens"] is False
|
||||
|
||||
def test_vllm_gemma3(self) -> None:
|
||||
p = suggest_profile("vllm", "google/gemma-3-27b-it")
|
||||
assert p["capabilities"]["thinking_mode"] == "manual"
|
||||
|
||||
def test_vllm_qwen3(self) -> None:
|
||||
p = suggest_profile("vllm", "Qwen/Qwen3-8B")
|
||||
assert p["capabilities"]["thinking_mode"] == "manual"
|
||||
assert p["capabilities"]["thinking_param"] == "enable_thinking"
|
||||
# Qwen doesn't need skip_special_tokens workaround
|
||||
assert "extra_body" not in p.get("server_compat", {})
|
||||
|
||||
def test_vllm_qwq(self) -> None:
|
||||
p = suggest_profile("vllm", "Qwen/QwQ-32B")
|
||||
assert p["capabilities"]["thinking_mode"] == "manual"
|
||||
|
||||
def test_vllm_granite(self) -> None:
|
||||
p = suggest_profile("vllm", "ibm-granite/granite-3.2-2b-instruct")
|
||||
assert p["capabilities"]["thinking_param"] == "thinking"
|
||||
|
||||
def test_vllm_deepseek_r1(self) -> None:
|
||||
p = suggest_profile("vllm", "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
|
||||
assert p["capabilities"]["thinking_param"] == "thinking"
|
||||
|
||||
def test_vllm_deepseek_v3_no_thinking(self) -> None:
|
||||
"""DeepSeek-V3 is a chat model, not a reasoning model — no thinking profile."""
|
||||
p = suggest_profile("vllm", "deepseek-ai/DeepSeek-V3-0324")
|
||||
assert "capabilities" not in p
|
||||
assert p["server_compat"]["server_type"] == "vllm"
|
||||
|
||||
def test_vllm_non_thinking_model(self) -> None:
|
||||
p = suggest_profile("vllm", "meta-llama/Llama-3-70B-Instruct")
|
||||
assert "capabilities" not in p
|
||||
assert p["server_compat"]["server_type"] == "vllm"
|
||||
|
||||
def test_llama_cpp_non_thinking(self) -> None:
|
||||
p = suggest_profile("llama.cpp", "some-model")
|
||||
assert p["server_compat"]["server_type"] == "llama.cpp"
|
||||
assert "capabilities" not in p
|
||||
|
||||
def test_llama_cpp_gemma_thinking(self) -> None:
|
||||
"""llama.cpp with Gemma model gets thinking profile with reasoning_format."""
|
||||
p = suggest_profile("llama.cpp", "gemma-4-E4B-it.gguf")
|
||||
assert p["capabilities"]["thinking_mode"] == "manual"
|
||||
assert p["server_compat"]["extra_body"]["reasoning_format"] == "auto"
|
||||
|
||||
def test_llama_cpp_qwen_thinking(self) -> None:
|
||||
p = suggest_profile("llama.cpp", "Qwen3-8B-Q4_K_M.gguf")
|
||||
assert p["capabilities"]["thinking_mode"] == "manual"
|
||||
|
||||
def test_sglang(self) -> None:
|
||||
p = suggest_profile("sglang", "some-model")
|
||||
assert p["server_compat"]["server_type"] == "sglang"
|
||||
|
||||
def test_unknown_server(self) -> None:
|
||||
assert suggest_profile("unknown", "foo") == {}
|
||||
|
||||
def test_empty_inputs(self) -> None:
|
||||
assert suggest_profile("", "") == {}
|
||||
|
||||
def test_openai_compatible_fallback(self) -> None:
|
||||
"""Generic openai-compatible without a specific profile."""
|
||||
assert suggest_profile("openai-compatible", "some-local-model") == {}
|
||||
|
||||
def test_case_insensitive_model_match(self) -> None:
|
||||
"""Model matching should be case-insensitive."""
|
||||
p = suggest_profile("vllm", "Google/GEMMA-4-31B-IT")
|
||||
assert p["capabilities"]["thinking_mode"] == "manual"
|
||||
|
||||
def test_holo_requires_holo2(self) -> None:
|
||||
"""Short 'holo' prefix shouldn't false-match; 'holo2' should match."""
|
||||
p_short = suggest_profile("vllm", "some-org/hologram-7b")
|
||||
assert "capabilities" not in p_short
|
||||
p_long = suggest_profile("vllm", "some-org/Holo2-14B")
|
||||
assert p_long["capabilities"]["thinking_mode"] == "manual"
|
||||
|
||||
def test_suggest_returns_deep_copy(self) -> None:
|
||||
"""Mutating the returned profile should not affect future calls."""
|
||||
p1 = suggest_profile("vllm", "google/gemma-4-31B-it")
|
||||
p1["capabilities"]["thinking_mode"] = "none"
|
||||
p2 = suggest_profile("vllm", "google/gemma-4-31B-it")
|
||||
assert p2["capabilities"]["thinking_mode"] == "manual"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# merge_server_compat
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergeServerCompat:
|
||||
def test_empty_compat_returns_base_only(self) -> None:
|
||||
base = {"reasoning_effort": "medium"}
|
||||
result = merge_server_compat(base, {})
|
||||
assert result == {"chat_template_kwargs": {"reasoning_effort": "medium"}}
|
||||
|
||||
def test_extra_body_merged_top_level(self) -> None:
|
||||
base = {"reasoning_effort": "medium"}
|
||||
compat = {"extra_body": {"skip_special_tokens": False}}
|
||||
result = merge_server_compat(base, compat)
|
||||
assert result["skip_special_tokens"] is False
|
||||
assert "chat_template_kwargs" in result
|
||||
|
||||
def test_full_vllm_gemma_compat(self) -> None:
|
||||
base = {"reasoning_effort": "medium"}
|
||||
compat = {
|
||||
"server_type": "vllm",
|
||||
"extra_body": {"skip_special_tokens": False},
|
||||
}
|
||||
result = merge_server_compat(base, compat)
|
||||
assert result == {
|
||||
"chat_template_kwargs": {"reasoning_effort": "medium"},
|
||||
"skip_special_tokens": False,
|
||||
}
|
||||
|
||||
def test_extra_body_chat_template_kwargs_deep_merged(self) -> None:
|
||||
"""chat_template_kwargs in extra_body is deep-merged, operator wins."""
|
||||
base = {"reasoning_effort": "medium"}
|
||||
compat = {
|
||||
"extra_body": {
|
||||
"chat_template_kwargs": {"custom_flag": True, "reasoning_effort": "high"},
|
||||
"skip_special_tokens": False,
|
||||
},
|
||||
}
|
||||
result = merge_server_compat(base, compat)
|
||||
# Operator values win over base
|
||||
assert result["chat_template_kwargs"]["custom_flag"] is True
|
||||
assert result["chat_template_kwargs"]["reasoning_effort"] == "high"
|
||||
assert result["skip_special_tokens"] is False
|
||||
|
||||
def test_extra_body_chat_template_kwargs_non_dict_ignored(self) -> None:
|
||||
"""Non-dict chat_template_kwargs in extra_body is safely ignored."""
|
||||
base = {"reasoning_effort": "medium"}
|
||||
compat = {"extra_body": {"chat_template_kwargs": "bad"}}
|
||||
result = merge_server_compat(base, compat)
|
||||
assert result["chat_template_kwargs"] == {"reasoning_effort": "medium"}
|
||||
|
||||
def test_base_not_mutated(self) -> None:
|
||||
base = {"reasoning_effort": "medium"}
|
||||
compat = {"extra_body": {"skip_special_tokens": False}}
|
||||
merge_server_compat(base, compat)
|
||||
assert "skip_special_tokens" not in base
|
||||
|
||||
def test_non_dict_extra_body_ignored(self) -> None:
|
||||
"""Gracefully handle malformed server_compat."""
|
||||
base = {"reasoning_effort": "medium"}
|
||||
result = merge_server_compat(base, {"extra_body": 42})
|
||||
assert result == {"chat_template_kwargs": {"reasoning_effort": "medium"}}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end: session merge + provider thinking mode
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEndToEndRequestShaping:
|
||||
"""Compose both layers — session builds extra_params, provider applies thinking."""
|
||||
|
||||
def test_vllm_gemma_full_flow(self) -> None:
|
||||
"""Session merges server workarounds, provider adds thinking param."""
|
||||
caps = ModelCapabilities(thinking_mode="manual", thinking_param="enable_thinking")
|
||||
base_ctk = {"reasoning_effort": "medium"}
|
||||
server_compat = {
|
||||
"server_type": "vllm",
|
||||
"extra_body": {"skip_special_tokens": False},
|
||||
}
|
||||
# Step 1: session merges
|
||||
extra_params = merge_server_compat(base_ctk, server_compat)
|
||||
# Step 2: provider finalises
|
||||
extra_body = dict(extra_params)
|
||||
OpenAIChatCompletionsProvider._apply_thinking_mode(extra_body, caps)
|
||||
|
||||
assert extra_body == {
|
||||
"chat_template_kwargs": {
|
||||
"reasoning_effort": "medium",
|
||||
"enable_thinking": True,
|
||||
},
|
||||
"skip_special_tokens": False,
|
||||
}
|
||||
|
||||
def test_granite_thinking_key(self) -> None:
|
||||
"""Granite uses 'thinking' instead of 'enable_thinking'."""
|
||||
caps = ModelCapabilities(thinking_mode="manual", thinking_param="thinking")
|
||||
extra_params = merge_server_compat({"reasoning_effort": "low"}, {})
|
||||
extra_body = dict(extra_params)
|
||||
OpenAIChatCompletionsProvider._apply_thinking_mode(extra_body, caps)
|
||||
|
||||
assert extra_body["chat_template_kwargs"]["thinking"] is True
|
||||
assert "enable_thinking" not in extra_body["chat_template_kwargs"]
|
||||
|
||||
def test_non_thinking_model_no_injection(self) -> None:
|
||||
"""Non-thinking model gets no thinking params."""
|
||||
caps = ModelCapabilities() # thinking_mode="none"
|
||||
extra_params = merge_server_compat({"reasoning_effort": "medium"}, {})
|
||||
extra_body = dict(extra_params)
|
||||
OpenAIChatCompletionsProvider._apply_thinking_mode(extra_body, caps)
|
||||
|
||||
assert extra_body == {"chat_template_kwargs": {"reasoning_effort": "medium"}}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probe integration: suggest_profile called from _detect_openai_compat
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProbeIntegration:
|
||||
def test_detect_vllm_gemma_suggests_profile(self) -> None:
|
||||
"""_detect_openai_compat returns suggested_capabilities and suggested_server_compat."""
|
||||
from turnstone.core.model_registry import _detect_openai_compat
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"reachable": True,
|
||||
"model_found": True,
|
||||
"available_models": ["google/gemma-4-31B-it"],
|
||||
"context_window": None,
|
||||
"server_type": None,
|
||||
"error": None,
|
||||
}
|
||||
model_obj = MagicMock()
|
||||
model_obj.model_dump.return_value = {"owned_by": "vllm"}
|
||||
|
||||
_detect_openai_compat(
|
||||
result, model_obj, "google/gemma-4-31B-it", "http://localhost:8000/v1"
|
||||
)
|
||||
|
||||
assert result["server_type"] == "vllm"
|
||||
assert result["suggested_capabilities"]["thinking_mode"] == "manual"
|
||||
assert result["suggested_capabilities"]["thinking_param"] == "enable_thinking"
|
||||
assert result["suggested_server_compat"]["extra_body"]["skip_special_tokens"] is False
|
||||
|
||||
def test_detect_non_thinking_no_suggested_capabilities(self) -> None:
|
||||
"""Non-thinking vLLM model gets server_compat but no capabilities suggestion."""
|
||||
from turnstone.core.model_registry import _detect_openai_compat
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"reachable": True,
|
||||
"model_found": True,
|
||||
"available_models": ["meta-llama/Llama-3-70B"],
|
||||
"context_window": None,
|
||||
"server_type": None,
|
||||
"error": None,
|
||||
}
|
||||
model_obj = MagicMock()
|
||||
model_obj.model_dump.return_value = {"owned_by": "vllm"}
|
||||
|
||||
_detect_openai_compat(
|
||||
result, model_obj, "meta-llama/Llama-3-70B", "http://localhost:8000/v1"
|
||||
)
|
||||
|
||||
assert result["server_type"] == "vllm"
|
||||
assert "suggested_capabilities" not in result
|
||||
assert result["suggested_server_compat"]["server_type"] == "vllm"
|
||||
@@ -1080,85 +1080,3 @@ class TestProviderExtraParams:
|
||||
openai_prov = create_provider("openai")
|
||||
result = session._provider_extra_params(provider=openai_prov)
|
||||
assert result is None
|
||||
|
||||
def test_server_compat_extra_body_merged(self, tmp_db):
|
||||
"""server_compat.extra_body workarounds are merged into extra_params."""
|
||||
from turnstone.core.model_registry import ModelConfig, ModelRegistry
|
||||
|
||||
session = self._session_with_provider("openai-compatible", tmp_db)
|
||||
cfg = ModelConfig(
|
||||
alias="test",
|
||||
base_url="http://localhost:8000/v1",
|
||||
api_key="none",
|
||||
model="google/gemma-4-31B-it",
|
||||
server_compat={
|
||||
"extra_body": {"skip_special_tokens": False},
|
||||
},
|
||||
)
|
||||
session._registry = ModelRegistry(models={"test": cfg}, default="test")
|
||||
session._model_alias = "test"
|
||||
result = session._provider_extra_params()
|
||||
assert result is not None
|
||||
assert result["chat_template_kwargs"]["reasoning_effort"] == "medium"
|
||||
assert result["skip_special_tokens"] is False
|
||||
|
||||
def test_empty_server_compat_backwards_compatible(self, tmp_db):
|
||||
"""Empty server_compat produces same output as before."""
|
||||
session = self._session_with_provider("openai-compatible", tmp_db)
|
||||
result = session._provider_extra_params()
|
||||
assert result == {"chat_template_kwargs": {"reasoning_effort": "medium"}}
|
||||
|
||||
def test_server_compat_with_reasoning_effort_override(self, tmp_db):
|
||||
"""reasoning_effort override works alongside server_compat."""
|
||||
from turnstone.core.model_registry import ModelConfig, ModelRegistry
|
||||
|
||||
session = self._session_with_provider("openai-compatible", tmp_db)
|
||||
cfg = ModelConfig(
|
||||
alias="test",
|
||||
base_url="http://localhost:8000/v1",
|
||||
api_key="none",
|
||||
model="google/gemma-4-31B-it",
|
||||
server_compat={"extra_body": {"skip_special_tokens": False}},
|
||||
)
|
||||
session._registry = ModelRegistry(models={"test": cfg}, default="test")
|
||||
session._model_alias = "test"
|
||||
result = session._provider_extra_params(reasoning_effort="high")
|
||||
assert result is not None
|
||||
assert result["chat_template_kwargs"]["reasoning_effort"] == "high"
|
||||
assert result["skip_special_tokens"] is False
|
||||
|
||||
def test_model_alias_resolves_target_compat(self, tmp_db):
|
||||
"""model_alias parameter selects compat from the target, not the primary."""
|
||||
from turnstone.core.model_registry import ModelConfig, ModelRegistry
|
||||
|
||||
session = self._session_with_provider("openai-compatible", tmp_db)
|
||||
primary = ModelConfig(
|
||||
alias="primary",
|
||||
base_url="http://localhost:8000/v1",
|
||||
api_key="none",
|
||||
model="google/gemma-4-31B-it",
|
||||
server_compat={"extra_body": {"skip_special_tokens": False}},
|
||||
)
|
||||
fallback = ModelConfig(
|
||||
alias="fallback",
|
||||
base_url="http://localhost:9000/v1",
|
||||
api_key="none",
|
||||
model="meta-llama/Llama-3-70B",
|
||||
)
|
||||
reg = ModelRegistry(
|
||||
models={"primary": primary, "fallback": fallback},
|
||||
default="primary",
|
||||
fallback=["fallback"],
|
||||
)
|
||||
session._registry = reg
|
||||
session._model_alias = "primary"
|
||||
|
||||
# Primary alias → gets Gemma workaround
|
||||
result_primary = session._provider_extra_params()
|
||||
assert result_primary is not None
|
||||
assert result_primary["skip_special_tokens"] is False
|
||||
|
||||
# Fallback alias → no compat, just base kwargs
|
||||
result_fallback = session._provider_extra_params(model_alias="fallback")
|
||||
assert result_fallback == {"chat_template_kwargs": {"reasoning_effort": "medium"}}
|
||||
assert "skip_special_tokens" not in result_fallback
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
|
||||
|
||||
__version__ = "1.4.0a2"
|
||||
__version__ = "1.3.0"
|
||||
|
||||
@@ -4596,19 +4596,6 @@ function _renderModels(items) {
|
||||
});
|
||||
}
|
||||
|
||||
function _isPlainObject(v) {
|
||||
return v !== null && typeof v === "object" && !Array.isArray(v);
|
||||
}
|
||||
|
||||
function _toggleThinkingParam() {
|
||||
var mode = document.getElementById("model-thinking-mode").value;
|
||||
var row = document.getElementById("model-thinking-param-row");
|
||||
row.style.display = mode ? "" : "none";
|
||||
// Set default when first enabling
|
||||
var paramEl = document.getElementById("model-thinking-param");
|
||||
if (mode && !paramEl.value) paramEl.value = "enable_thinking";
|
||||
}
|
||||
|
||||
function showCreateModelModal() {
|
||||
_modelCreateTrigger = document.activeElement;
|
||||
var ov = document.getElementById("model-create-overlay");
|
||||
@@ -4627,18 +4614,7 @@ function showCreateModelModal() {
|
||||
document.getElementById("model-temperature").value = "";
|
||||
document.getElementById("model-max-tokens").value = "";
|
||||
document.getElementById("model-reasoning-effort").value = "";
|
||||
document.getElementById("model-server-type").value = "";
|
||||
document.getElementById("model-thinking-mode").value = "";
|
||||
document.getElementById("model-thinking-param").value = "";
|
||||
document.getElementById("model-thinking-param-row").style.display = "none";
|
||||
document.getElementById("model-extra-body").value = "";
|
||||
document.getElementById("model-capabilities").value = "";
|
||||
// Clear validation error styling from prior submit attempts
|
||||
["model-extra-body", "model-capabilities"].forEach(function (id) {
|
||||
var el = document.getElementById(id);
|
||||
el.removeAttribute("aria-invalid");
|
||||
el.style.borderColor = "";
|
||||
});
|
||||
document.getElementById("model-enabled").checked = true;
|
||||
document.getElementById("model-detect-result").style.display = "none";
|
||||
document.getElementById("model-detect-btn").disabled = false;
|
||||
@@ -4677,49 +4653,15 @@ function showEditModelModal(definitionId) {
|
||||
m.max_tokens != null ? m.max_tokens : "";
|
||||
document.getElementById("model-reasoning-effort").value =
|
||||
m.reasoning_effort != null ? m.reasoning_effort : "";
|
||||
// Parse capabilities JSON and extract server_compat for structured fields
|
||||
var capsObj = {};
|
||||
// Parse capabilities JSON for display
|
||||
var caps = m.capabilities || "{}";
|
||||
try {
|
||||
capsObj = JSON.parse(m.capabilities || "{}");
|
||||
caps = JSON.stringify(JSON.parse(caps), null, 2);
|
||||
} catch (e) {
|
||||
/* keep empty */
|
||||
/* keep raw */
|
||||
}
|
||||
// Defend against null/array/primitive values in the DB
|
||||
if (!_isPlainObject(capsObj)) capsObj = {};
|
||||
var sc = _isPlainObject(capsObj.server_compat)
|
||||
? capsObj.server_compat
|
||||
: {};
|
||||
// Only extract thinking_mode into the dropdown when the UI can
|
||||
// represent it ("manual" or ""). Values like "adaptive" (Anthropic-
|
||||
// only) stay in the raw capabilities JSON so they aren't silently
|
||||
// lost on save.
|
||||
var tmVal = capsObj.thinking_mode || "";
|
||||
var tmRepresentable = tmVal === "" || tmVal === "manual";
|
||||
if (tmRepresentable) {
|
||||
document.getElementById("model-thinking-mode").value = tmVal;
|
||||
document.getElementById("model-thinking-param").value =
|
||||
capsObj.thinking_param || "";
|
||||
} else {
|
||||
document.getElementById("model-thinking-mode").value = "";
|
||||
document.getElementById("model-thinking-param").value = "";
|
||||
}
|
||||
_toggleThinkingParam();
|
||||
// Server compat: server_type and extra_body workarounds
|
||||
document.getElementById("model-server-type").value = sc.server_type || "";
|
||||
var eb = sc.extra_body || {};
|
||||
var ebText = JSON.stringify(eb, null, 2);
|
||||
document.getElementById("model-extra-body").value =
|
||||
ebText === "{}" ? "" : ebText;
|
||||
// Remove structured fields from capabilities display — only delete
|
||||
// thinking_mode/thinking_param when the UI successfully captured them.
|
||||
delete capsObj.server_compat;
|
||||
if (tmRepresentable) {
|
||||
delete capsObj.thinking_mode;
|
||||
delete capsObj.thinking_param;
|
||||
}
|
||||
var capsText = JSON.stringify(capsObj, null, 2);
|
||||
document.getElementById("model-capabilities").value =
|
||||
capsText === "{}" ? "" : capsText;
|
||||
if (caps === "{}") caps = "";
|
||||
document.getElementById("model-capabilities").value = caps;
|
||||
document.getElementById("model-enabled").checked = m.enabled !== false;
|
||||
_applyProviderDefaults();
|
||||
})
|
||||
@@ -4752,65 +4694,15 @@ function submitCreateModel() {
|
||||
return;
|
||||
}
|
||||
|
||||
var capsEl = document.getElementById("model-capabilities");
|
||||
var capsText = capsEl.value.trim();
|
||||
var capsText = document.getElementById("model-capabilities").value.trim();
|
||||
var caps = {};
|
||||
capsEl.removeAttribute("aria-invalid");
|
||||
capsEl.style.borderColor = "";
|
||||
if (capsText) {
|
||||
try {
|
||||
caps = JSON.parse(capsText);
|
||||
} catch (e) {
|
||||
capsEl.setAttribute("aria-invalid", "true");
|
||||
capsEl.style.borderColor = "var(--red)";
|
||||
_showModelError("Invalid JSON in capabilities");
|
||||
return;
|
||||
}
|
||||
if (!_isPlainObject(caps)) {
|
||||
capsEl.setAttribute("aria-invalid", "true");
|
||||
capsEl.style.borderColor = "var(--red)";
|
||||
_showModelError(
|
||||
"Capabilities must be a JSON object (not array or primitive)",
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Thinking mode → capabilities (provider uses this to inject
|
||||
// the correct chat_template_kwargs param automatically).
|
||||
var thinkingMode = document.getElementById("model-thinking-mode").value;
|
||||
if (thinkingMode) {
|
||||
caps.thinking_mode = thinkingMode;
|
||||
// Preserve thinking_param so Granite/DeepSeek "thinking" key
|
||||
// isn't silently reverted to the default "enable_thinking".
|
||||
var savedParam = document.getElementById("model-thinking-param").value;
|
||||
if (savedParam) caps.thinking_param = savedParam;
|
||||
}
|
||||
|
||||
// Build server_compat from structured fields
|
||||
var serverCompat = {};
|
||||
var serverType = document.getElementById("model-server-type").value;
|
||||
if (serverType) serverCompat.server_type = serverType;
|
||||
var ebEl = document.getElementById("model-extra-body");
|
||||
var ebText = ebEl.value.trim();
|
||||
ebEl.removeAttribute("aria-invalid");
|
||||
ebEl.style.borderColor = "";
|
||||
if (ebText) {
|
||||
try {
|
||||
var ebParsed = JSON.parse(ebText);
|
||||
if (!_isPlainObject(ebParsed)) {
|
||||
throw new Error("not an object");
|
||||
}
|
||||
serverCompat.extra_body = ebParsed;
|
||||
} catch (e) {
|
||||
ebEl.setAttribute("aria-invalid", "true");
|
||||
ebEl.style.borderColor = "var(--red)";
|
||||
_showModelError("Extra body params must be a JSON object");
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (Object.keys(serverCompat).length > 0) {
|
||||
caps.server_compat = serverCompat;
|
||||
}
|
||||
|
||||
var form = {
|
||||
@@ -5003,52 +4895,6 @@ function detectModel() {
|
||||
resultDiv.appendChild(
|
||||
_detectResultLine("Server type: " + d.server_type),
|
||||
);
|
||||
// Auto-fill server type if not already set and value is a known option
|
||||
var stEl = document.getElementById("model-server-type");
|
||||
var stOpts = Array.from(stEl.options).map(function (o) {
|
||||
return o.value;
|
||||
});
|
||||
if (!stEl.value && stOpts.indexOf(d.server_type) !== -1)
|
||||
stEl.value = d.server_type;
|
||||
}
|
||||
// Auto-fill capabilities from suggested profile
|
||||
if (d.suggested_capabilities) {
|
||||
var sc2 = d.suggested_capabilities;
|
||||
var tmEl = document.getElementById("model-thinking-mode");
|
||||
if (!tmEl.value && sc2.thinking_mode) {
|
||||
tmEl.value = sc2.thinking_mode;
|
||||
}
|
||||
if (sc2.thinking_param) {
|
||||
var tpEl = document.getElementById("model-thinking-param");
|
||||
if (!tpEl.value) tpEl.value = sc2.thinking_param;
|
||||
}
|
||||
_toggleThinkingParam();
|
||||
}
|
||||
// Auto-fill server compat from suggested profile
|
||||
if (d.suggested_server_compat) {
|
||||
var ssc = d.suggested_server_compat;
|
||||
var stEl2 = document.getElementById("model-server-type");
|
||||
var stOpts2 = Array.from(stEl2.options).map(function (o) {
|
||||
return o.value;
|
||||
});
|
||||
if (
|
||||
!stEl2.value &&
|
||||
ssc.server_type &&
|
||||
stOpts2.indexOf(ssc.server_type) !== -1
|
||||
)
|
||||
stEl2.value = ssc.server_type;
|
||||
if (ssc.extra_body) {
|
||||
var ebEl2 = document.getElementById("model-extra-body");
|
||||
if (!ebEl2.value.trim()) {
|
||||
var ebJson = JSON.stringify(ssc.extra_body, null, 2);
|
||||
if (ebJson !== "{}") ebEl2.value = ebJson;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (d.suggested_capabilities || d.suggested_server_compat) {
|
||||
resultDiv.appendChild(
|
||||
_detectResultLine("\u2713 Compatibility profile suggested", "green"),
|
||||
);
|
||||
}
|
||||
resultDiv.style.borderColor = "var(--green)";
|
||||
})
|
||||
@@ -5142,11 +4988,6 @@ function _applyProviderDefaults() {
|
||||
if (!def) return;
|
||||
document.getElementById("model-base-url").placeholder = def.urlPlaceholder;
|
||||
document.getElementById("model-name").placeholder = def.modelPlaceholder;
|
||||
// Server compat section only applies to local model servers
|
||||
var scSection = document.getElementById("model-server-compat-section");
|
||||
if (scSection) {
|
||||
scSection.style.display = provider === "openai-compatible" ? "" : "none";
|
||||
}
|
||||
}
|
||||
|
||||
/* Populate the model name datalist with known model prefixes for the
|
||||
|
||||
@@ -1567,26 +1567,6 @@ window.TURNSTONE_KB_SHORTCUTS = [
|
||||
<option value="xhigh">xhigh</option>
|
||||
<option value="max">max</option>
|
||||
</select>
|
||||
<div id="model-server-compat-section" style="display:none">
|
||||
<div class="modal-section-divider" role="separator">Server Compatibility</div>
|
||||
<label for="model-server-type">Server Type <span style="font-weight:400;text-transform:none">(auto-detected or manual)</span></label>
|
||||
<select id="model-server-type">
|
||||
<option value="">Auto / Unknown</option>
|
||||
<option value="vllm">vLLM</option>
|
||||
<option value="llama.cpp">llama.cpp</option>
|
||||
<option value="openai-compatible">Other OpenAI-compatible</option>
|
||||
</select>
|
||||
<label for="model-thinking-mode">Thinking Mode <span style="font-weight:400;text-transform:none">(reasoning / chain-of-thought)</span></label>
|
||||
<select id="model-thinking-mode" onchange="_toggleThinkingParam()">
|
||||
<option value="">None</option>
|
||||
<option value="manual">Enabled</option>
|
||||
</select>
|
||||
<div id="model-thinking-param-row" style="display:none">
|
||||
<label for="model-thinking-param" style="font-size:11px">Template param name <span style="font-weight:400;text-transform:none">(Granite/DeepSeek use "thinking")</span></label>
|
||||
<input type="text" id="model-thinking-param" value="enable_thinking" placeholder="enable_thinking" style="font-family:var(--font-mono);font-size:11px"></div>
|
||||
<label for="model-extra-body">Extra body params <span style="font-weight:400;text-transform:none">(JSON, merged into every request)</span></label>
|
||||
<textarea id="model-extra-body" rows="2" placeholder='{"skip_special_tokens": false}' style="font-family:var(--font-mono);font-size:11px"></textarea>
|
||||
</div>
|
||||
<label for="model-capabilities">Capabilities <span style="font-weight:400;text-transform:none">(JSON)</span></label>
|
||||
<textarea id="model-capabilities" rows="3" placeholder='{"supports_vision": true}' style="font-family:var(--font-mono);font-size:11px"></textarea>
|
||||
<div style="display:flex;gap:20px;margin-top:14px">
|
||||
|
||||
@@ -39,9 +39,6 @@ class ModelConfig:
|
||||
temperature: float | None = None
|
||||
max_tokens: int | None = None
|
||||
reasoning_effort: str | None = None
|
||||
# Server compatibility settings for openai-compatible backends.
|
||||
# Populated from capabilities["server_compat"] during load.
|
||||
server_compat: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -269,10 +266,6 @@ def load_model_registry(
|
||||
caps = parsed
|
||||
except (_json.JSONDecodeError, TypeError):
|
||||
pass # falls back to empty capabilities
|
||||
# Extract server_compat from capabilities (namespaced key)
|
||||
row_server_compat = caps.pop("server_compat", {})
|
||||
if not isinstance(row_server_compat, dict):
|
||||
row_server_compat = {}
|
||||
row_base_url = _resolve_env_vars(row.get("base_url", ""))
|
||||
row_provider = _resolve_openai_provider(row.get("provider", "openai"), row_base_url)
|
||||
row_model = row["model"]
|
||||
@@ -297,7 +290,6 @@ def load_model_registry(
|
||||
reasoning_effort=row_reasoning_effort
|
||||
if row_reasoning_effort is not None
|
||||
else None,
|
||||
server_compat=row_server_compat,
|
||||
)
|
||||
except Exception:
|
||||
log.warning("Failed to load model definitions from storage", exc_info=True)
|
||||
@@ -341,14 +333,6 @@ def load_model_registry(
|
||||
raw_effort = entry.get("reasoning_effort")
|
||||
if raw_effort is not None:
|
||||
entry_effort = str(raw_effort)
|
||||
entry_caps = (
|
||||
dict(entry.get("capabilities", {}))
|
||||
if isinstance(entry.get("capabilities"), dict)
|
||||
else {}
|
||||
)
|
||||
entry_server_compat = entry_caps.pop("server_compat", {})
|
||||
if not isinstance(entry_server_compat, dict):
|
||||
entry_server_compat = {}
|
||||
configs[alias] = ModelConfig(
|
||||
alias=alias,
|
||||
base_url=entry_base_url,
|
||||
@@ -356,12 +340,13 @@ def load_model_registry(
|
||||
model=model_name,
|
||||
context_window=entry.get("context_window", context_window),
|
||||
provider=_resolve_openai_provider(entry.get("provider", "openai"), entry_base_url),
|
||||
capabilities=entry_caps,
|
||||
capabilities=entry.get("capabilities", {})
|
||||
if isinstance(entry.get("capabilities"), dict)
|
||||
else {},
|
||||
source="config",
|
||||
temperature=entry_temp,
|
||||
max_tokens=entry_max_tokens,
|
||||
reasoning_effort=entry_effort,
|
||||
server_compat=entry_server_compat,
|
||||
)
|
||||
|
||||
# 3. Ensure a "default" entry from CLI args (only if not already defined
|
||||
@@ -657,12 +642,3 @@ def _detect_openai_compat(
|
||||
result["server_type"] = "vllm"
|
||||
else:
|
||||
result["server_type"] = "openai-compatible"
|
||||
|
||||
# Suggest capabilities and server compat based on detected server_type
|
||||
from turnstone.core.server_compat import suggest_profile
|
||||
|
||||
suggested = suggest_profile(result.get("server_type", ""), model_id)
|
||||
if suggested.get("capabilities"):
|
||||
result["suggested_capabilities"] = suggested["capabilities"]
|
||||
if suggested.get("server_compat"):
|
||||
result["suggested_server_compat"] = suggested["server_compat"]
|
||||
|
||||
@@ -568,10 +568,9 @@ class AnthropicProvider:
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
cancel_ref: list[Any] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> Iterator[StreamChunk]:
|
||||
_ensure_anthropic()
|
||||
caps = capabilities or self.get_capabilities(model)
|
||||
caps = self.get_capabilities(model)
|
||||
system_prompt, converted_msgs = self._convert_messages(messages)
|
||||
kwargs = self._build_thinking_and_kwargs(
|
||||
caps,
|
||||
@@ -772,10 +771,9 @@ class AnthropicProvider:
|
||||
reasoning_effort: str = "medium",
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> CompletionResult:
|
||||
_ensure_anthropic()
|
||||
caps = capabilities or self.get_capabilities(model)
|
||||
caps = self.get_capabilities(model)
|
||||
system_prompt, converted_msgs = self._convert_messages(messages)
|
||||
kwargs = self._build_thinking_and_kwargs(
|
||||
caps,
|
||||
|
||||
@@ -108,52 +108,6 @@ class OpenAIChatCompletionsProvider:
|
||||
kwargs["web_search_options"] = {}
|
||||
return tools
|
||||
|
||||
# -- thinking mode -------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _apply_thinking_mode(
|
||||
extra_body: dict[str, Any],
|
||||
caps: ModelCapabilities,
|
||||
) -> None:
|
||||
"""Inject thinking-mode params into *extra_body* based on capabilities.
|
||||
|
||||
When ``caps.thinking_mode`` is ``"manual"`` or ``"adaptive"``, sets
|
||||
the model-family-specific key (``caps.thinking_param``, e.g.
|
||||
``"enable_thinking"`` or ``"thinking"``) to ``True`` inside
|
||||
``extra_body["chat_template_kwargs"]``.
|
||||
|
||||
Does nothing when thinking mode is ``"none"`` or the key is already
|
||||
present (operator override via ``extra_body`` takes precedence).
|
||||
"""
|
||||
if caps.thinking_mode == "none":
|
||||
return
|
||||
ctk = extra_body.get("chat_template_kwargs")
|
||||
if not isinstance(ctk, dict):
|
||||
ctk = {}
|
||||
extra_body["chat_template_kwargs"] = ctk
|
||||
if caps.thinking_param not in ctk:
|
||||
ctk[caps.thinking_param] = True
|
||||
|
||||
def _finalize_extra_body(
|
||||
self,
|
||||
extra_params: dict[str, Any] | None,
|
||||
caps: ModelCapabilities,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Build the final ``extra_body``, injecting thinking params if needed.
|
||||
|
||||
Returns ``None`` when the result would be empty (no extra_body needed).
|
||||
Shallow-copies *extra_params* and its ``chat_template_kwargs`` so the
|
||||
caller's dict is never mutated.
|
||||
"""
|
||||
eb: dict[str, Any] = {}
|
||||
if extra_params:
|
||||
eb = dict(extra_params)
|
||||
ctk = eb.get("chat_template_kwargs")
|
||||
if isinstance(ctk, dict):
|
||||
eb["chat_template_kwargs"] = dict(ctk)
|
||||
self._apply_thinking_mode(eb, caps)
|
||||
return eb or None
|
||||
|
||||
# -- streaming -----------------------------------------------------------
|
||||
|
||||
def create_streaming(
|
||||
@@ -169,9 +123,8 @@ class OpenAIChatCompletionsProvider:
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
cancel_ref: list[Any] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> Iterator[StreamChunk]:
|
||||
caps = capabilities or self.get_capabilities(model)
|
||||
caps = self.get_capabilities(model)
|
||||
messages = self._prepare_messages(messages)
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": model,
|
||||
@@ -186,9 +139,8 @@ class OpenAIChatCompletionsProvider:
|
||||
tools = apply_tool_search(caps, tools, deferred_names)
|
||||
if tools:
|
||||
kwargs["tools"] = tools
|
||||
extra_body = self._finalize_extra_body(extra_params, caps)
|
||||
if extra_body:
|
||||
kwargs["extra_body"] = extra_body
|
||||
if extra_params:
|
||||
kwargs["extra_body"] = extra_params
|
||||
|
||||
log.debug(
|
||||
"openai.chat.request",
|
||||
@@ -298,9 +250,8 @@ class OpenAIChatCompletionsProvider:
|
||||
reasoning_effort: str = "medium",
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> CompletionResult:
|
||||
caps = capabilities or self.get_capabilities(model)
|
||||
caps = self.get_capabilities(model)
|
||||
messages = self._prepare_messages(messages)
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": model,
|
||||
@@ -314,9 +265,8 @@ class OpenAIChatCompletionsProvider:
|
||||
tools = apply_tool_search(caps, tools, deferred_names)
|
||||
if tools:
|
||||
kwargs["tools"] = tools
|
||||
extra_body = self._finalize_extra_body(extra_params, caps)
|
||||
if extra_body:
|
||||
kwargs["extra_body"] = extra_body
|
||||
if extra_params:
|
||||
kwargs["extra_body"] = extra_params
|
||||
|
||||
log.debug(
|
||||
"openai.chat.request",
|
||||
|
||||
@@ -223,10 +223,9 @@ class OpenAIResponsesProvider:
|
||||
temperature: float,
|
||||
reasoning_effort: str,
|
||||
deferred_names: frozenset[str] | None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the kwargs dict for ``client.responses.create/stream``."""
|
||||
caps = capabilities or self.get_capabilities(model)
|
||||
caps = self.get_capabilities(model)
|
||||
|
||||
instructions, input_items = self._convert_messages(messages)
|
||||
tools = apply_tool_search(caps, tools, deferred_names)
|
||||
@@ -277,7 +276,6 @@ class OpenAIResponsesProvider:
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
cancel_ref: list[Any] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> Iterator[StreamChunk]:
|
||||
if extra_params:
|
||||
log.debug("openai.responses: extra_params ignored (not supported by Responses API)")
|
||||
@@ -289,7 +287,6 @@ class OpenAIResponsesProvider:
|
||||
temperature,
|
||||
reasoning_effort,
|
||||
deferred_names,
|
||||
capabilities=capabilities,
|
||||
)
|
||||
kwargs["stream"] = True
|
||||
|
||||
@@ -458,7 +455,6 @@ class OpenAIResponsesProvider:
|
||||
reasoning_effort: str = "medium",
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> CompletionResult:
|
||||
if extra_params:
|
||||
log.debug("openai.responses: extra_params ignored (not supported by Responses API)")
|
||||
@@ -470,7 +466,6 @@ class OpenAIResponsesProvider:
|
||||
temperature,
|
||||
reasoning_effort,
|
||||
deferred_names,
|
||||
capabilities=capabilities,
|
||||
)
|
||||
|
||||
log.debug(
|
||||
|
||||
@@ -74,11 +74,6 @@ class ModelCapabilities:
|
||||
supports_tools: bool = True
|
||||
token_param: str = "max_completion_tokens"
|
||||
thinking_mode: str = "none" # "none" | "manual" | "adaptive"
|
||||
# For openai-compatible servers: the chat_template_kwargs key that
|
||||
# toggles thinking (e.g. "enable_thinking" for Gemma/Qwen,
|
||||
# "thinking" for Granite/DeepSeek). Ignored when thinking_mode is
|
||||
# "none" or by providers that handle thinking natively (Anthropic).
|
||||
thinking_param: str = "enable_thinking"
|
||||
supports_effort: bool = False
|
||||
effort_levels: tuple[str, ...] = ()
|
||||
reasoning_effort_values: tuple[str, ...] = ()
|
||||
@@ -132,16 +127,9 @@ class LLMProvider(Protocol):
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
cancel_ref: list[Any] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> Iterator[StreamChunk]:
|
||||
"""Create a streaming request, yielding normalized StreamChunks.
|
||||
|
||||
If *capabilities* is provided the provider uses it instead of
|
||||
calling ``get_capabilities(model)`` internally. This lets the
|
||||
session pass config-merged capabilities so that overrides from
|
||||
the model registry (e.g. ``thinking_mode``, ``token_param``)
|
||||
are respected.
|
||||
|
||||
If *cancel_ref* is provided the provider appends the underlying SDK
|
||||
stream object (which has a ``.close()`` method) before yielding the
|
||||
first chunk. The caller can then close it from another thread to
|
||||
@@ -161,7 +149,6 @@ class LLMProvider(Protocol):
|
||||
reasoning_effort: str = "medium",
|
||||
extra_params: dict[str, Any] | None = None,
|
||||
deferred_names: frozenset[str] | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> CompletionResult:
|
||||
"""Create a non-streaming request, returning a normalized result."""
|
||||
...
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
"""Server compatibility profiles for OpenAI-compatible backends.
|
||||
|
||||
Different local model servers (vLLM, llama.cpp, SGLang) need different
|
||||
request shaping. This module separates two concerns:
|
||||
|
||||
1. **Model capabilities** — ``thinking_mode`` and ``thinking_param`` are
|
||||
properties of the *model* (Gemma thinks, Llama doesn't). These go
|
||||
into the ``capabilities`` dict and flow through ``ModelCapabilities``
|
||||
so the provider can act on them (just like Anthropic's thinking mode).
|
||||
|
||||
2. **Server workarounds** — ``extra_body`` overrides like
|
||||
``skip_special_tokens=false`` are properties of the *server* (vLLM
|
||||
bug workaround). These stay in ``server_compat`` and get merged
|
||||
into the request's ``extra_body`` at call time.
|
||||
|
||||
Profiles are *suggestions* only. The admin UI auto-fills them on
|
||||
Detect; the operator has final say, and the stored DB config is what
|
||||
actually gets used at request time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Profile suggestions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Each profile has two optional parts:
|
||||
# "capabilities" — merged into the model's capabilities dict (thinking_mode etc.)
|
||||
# "server_compat" — stored as server_compat (extra_body workarounds)
|
||||
|
||||
_PROFILES: dict[str, dict[str, Any]] = {
|
||||
"vllm-gemma-thinking": {
|
||||
"capabilities": {
|
||||
"thinking_mode": "manual",
|
||||
"thinking_param": "enable_thinking",
|
||||
},
|
||||
"server_compat": {
|
||||
"server_type": "vllm",
|
||||
# Workaround: vLLM strips special tokens before the Gemma4
|
||||
# reasoning parser sees them. skip_special_tokens=false
|
||||
# preserves <|channel> / <channel|> markers so reasoning
|
||||
# content is extracted correctly.
|
||||
"extra_body": {"skip_special_tokens": False},
|
||||
},
|
||||
},
|
||||
"vllm-qwen-thinking": {
|
||||
"capabilities": {
|
||||
"thinking_mode": "manual",
|
||||
"thinking_param": "enable_thinking",
|
||||
},
|
||||
"server_compat": {
|
||||
"server_type": "vllm",
|
||||
},
|
||||
},
|
||||
"vllm-granite-thinking": {
|
||||
"capabilities": {
|
||||
"thinking_mode": "manual",
|
||||
"thinking_param": "thinking",
|
||||
},
|
||||
"server_compat": {
|
||||
"server_type": "vllm",
|
||||
},
|
||||
},
|
||||
"vllm-deepseek-thinking": {
|
||||
"capabilities": {
|
||||
"thinking_mode": "manual",
|
||||
"thinking_param": "thinking",
|
||||
},
|
||||
"server_compat": {
|
||||
"server_type": "vllm",
|
||||
},
|
||||
},
|
||||
"vllm-holo-thinking": {
|
||||
"capabilities": {
|
||||
"thinking_mode": "manual",
|
||||
"thinking_param": "enable_thinking",
|
||||
},
|
||||
"server_compat": {
|
||||
"server_type": "vllm",
|
||||
},
|
||||
},
|
||||
"vllm": {
|
||||
"server_compat": {
|
||||
"server_type": "vllm",
|
||||
},
|
||||
},
|
||||
"llama.cpp": {
|
||||
"server_compat": {
|
||||
"server_type": "llama.cpp",
|
||||
},
|
||||
},
|
||||
"llama.cpp-thinking": {
|
||||
"capabilities": {
|
||||
"thinking_mode": "manual",
|
||||
"thinking_param": "enable_thinking",
|
||||
},
|
||||
"server_compat": {
|
||||
"server_type": "llama.cpp",
|
||||
# llama.cpp uses reasoning_format (top-level request param) to
|
||||
# extract thinking into the reasoning_content response field.
|
||||
# "auto" lets the server decide based on the model's template;
|
||||
# "deepseek" forces extraction for all thinking models.
|
||||
"extra_body": {"reasoning_format": "auto"},
|
||||
},
|
||||
},
|
||||
"sglang": {
|
||||
"server_compat": {
|
||||
"server_type": "sglang",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# Model-family → profile key mapping. Checked in order; first match wins.
|
||||
_VLLM_MODEL_PROFILES: list[tuple[str, str]] = [
|
||||
("gemma-4", "vllm-gemma-thinking"),
|
||||
("gemma-3", "vllm-gemma-thinking"),
|
||||
("gemma4", "vllm-gemma-thinking"),
|
||||
("gemma3", "vllm-gemma-thinking"),
|
||||
("qwen3", "vllm-qwen-thinking"),
|
||||
("qwq", "vllm-qwen-thinking"),
|
||||
("granite-3", "vllm-granite-thinking"),
|
||||
("granite3", "vllm-granite-thinking"),
|
||||
("deepseek-r1", "vllm-deepseek-thinking"),
|
||||
("holo2", "vllm-holo-thinking"),
|
||||
]
|
||||
|
||||
# llama.cpp model-family → profile key mapping.
|
||||
_LLAMA_CPP_MODEL_PROFILES: list[tuple[str, str]] = [
|
||||
("gemma-4", "llama.cpp-thinking"),
|
||||
("gemma-3", "llama.cpp-thinking"),
|
||||
("gemma4", "llama.cpp-thinking"),
|
||||
("gemma3", "llama.cpp-thinking"),
|
||||
("qwen3", "llama.cpp-thinking"),
|
||||
("qwq", "llama.cpp-thinking"),
|
||||
("deepseek-r1", "llama.cpp-thinking"),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def suggest_profile(server_type: str, model_id: str) -> dict[str, Any]:
|
||||
"""Suggest capabilities and server compat based on server type and model.
|
||||
|
||||
Returns a dict with optional ``"capabilities"`` and ``"server_compat"``
|
||||
keys. Empty dict when no special settings are needed.
|
||||
"""
|
||||
profile_key: str | None = None
|
||||
model_lower = (model_id or "").lower()
|
||||
if server_type == "vllm":
|
||||
for substring, key in _VLLM_MODEL_PROFILES:
|
||||
if substring in model_lower:
|
||||
profile_key = key
|
||||
break
|
||||
if profile_key is None:
|
||||
profile_key = "vllm"
|
||||
elif server_type == "llama.cpp":
|
||||
for substring, key in _LLAMA_CPP_MODEL_PROFILES:
|
||||
if substring in model_lower:
|
||||
profile_key = key
|
||||
break
|
||||
if profile_key is None:
|
||||
profile_key = "llama.cpp"
|
||||
elif server_type in _PROFILES:
|
||||
profile_key = server_type
|
||||
|
||||
if profile_key is None:
|
||||
return {}
|
||||
return copy.deepcopy(_PROFILES[profile_key])
|
||||
|
||||
|
||||
def merge_server_compat(
|
||||
base_chat_template_kwargs: dict[str, Any],
|
||||
server_compat: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Build the ``extra_body`` dict by merging server compat into base kwargs.
|
||||
|
||||
*base_chat_template_kwargs* always contains at least ``reasoning_effort``.
|
||||
*server_compat* comes from ``ModelConfig.server_compat``.
|
||||
|
||||
Note: thinking-mode params (``enable_thinking``, ``thinking``) are **not**
|
||||
merged here — the provider handles those via ``ModelCapabilities``.
|
||||
This function only merges server workarounds from ``extra_body``.
|
||||
|
||||
Returns the complete dict to pass as ``extra_body`` to the OpenAI client.
|
||||
"""
|
||||
extra: dict[str, Any] = {"chat_template_kwargs": dict(base_chat_template_kwargs)}
|
||||
|
||||
# Merge top-level extra_body overrides (skip_special_tokens, etc.)
|
||||
compat_eb = server_compat.get("extra_body")
|
||||
if isinstance(compat_eb, dict):
|
||||
for key, value in compat_eb.items():
|
||||
if key == "chat_template_kwargs":
|
||||
# Deep-merge: operator values in extra_body win over the
|
||||
# base dict (which has reasoning_effort). This lets
|
||||
# operators intentionally extend chat_template_kwargs.
|
||||
if isinstance(value, dict):
|
||||
extra["chat_template_kwargs"].update(value)
|
||||
continue
|
||||
extra[key] = value
|
||||
|
||||
return extra
|
||||
@@ -1448,50 +1448,21 @@ class ChatSession:
|
||||
self,
|
||||
reasoning_effort: str | None = None,
|
||||
provider: LLMProvider | None = None,
|
||||
model_alias: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Build provider-specific extra parameters.
|
||||
|
||||
``chat_template_kwargs`` is only meaningful for local model servers
|
||||
(``openai-compatible``). Commercial OpenAI rejects it as an unknown
|
||||
parameter, and handles ``reasoning_effort`` natively.
|
||||
|
||||
Merges server workarounds (``skip_special_tokens``, etc.) from
|
||||
``ModelConfig.server_compat`` into the request's ``extra_body``.
|
||||
Thinking-mode params (``enable_thinking``) are handled separately
|
||||
by the provider based on ``ModelCapabilities.thinking_mode``.
|
||||
|
||||
*model_alias* controls which model config supplies server compat
|
||||
settings. When ``None``, defaults to the session's primary alias.
|
||||
"""
|
||||
from turnstone.core.server_compat import merge_server_compat
|
||||
|
||||
prov = provider or self._provider
|
||||
if prov.provider_name == "openai-compatible":
|
||||
ctk_base = dict(self._chat_template_kwargs_base)
|
||||
kwargs = dict(self._chat_template_kwargs_base)
|
||||
if reasoning_effort:
|
||||
ctk_base["reasoning_effort"] = reasoning_effort
|
||||
return merge_server_compat(
|
||||
ctk_base,
|
||||
self._get_server_compat(model_alias),
|
||||
)
|
||||
kwargs["reasoning_effort"] = reasoning_effort
|
||||
return {"chat_template_kwargs": kwargs}
|
||||
return None
|
||||
|
||||
def _get_server_compat(self, model_alias: str | None = None) -> dict[str, Any]:
|
||||
"""Get server compatibility settings from a model config.
|
||||
|
||||
*model_alias* selects the config to read. Falls back to the
|
||||
session's primary alias when ``None``.
|
||||
"""
|
||||
alias = model_alias or self._model_alias
|
||||
if self._registry and alias:
|
||||
try:
|
||||
cfg = self._registry.get_config(alias)
|
||||
return dict(cfg.server_compat)
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _utility_completion(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
@@ -1517,7 +1488,6 @@ class ChatSession:
|
||||
temperature=temperature,
|
||||
reasoning_effort=reasoning_effort,
|
||||
extra_params=self._provider_extra_params(reasoning_effort=reasoning_effort),
|
||||
capabilities=caps,
|
||||
)
|
||||
|
||||
# -- tool search helpers --------------------------------------------------
|
||||
@@ -1652,16 +1622,8 @@ class ChatSession:
|
||||
try:
|
||||
fb_client, fb_model, _ = self._registry.resolve(alias)
|
||||
fb_provider = self._registry.get_provider(alias)
|
||||
fb_caps = self._resolve_capabilities(fb_provider, fb_model, alias)
|
||||
self.ui.on_info(f"[Primary model failed, falling back to {alias}]")
|
||||
result = self._try_stream(
|
||||
fb_client,
|
||||
fb_model,
|
||||
msgs,
|
||||
provider=fb_provider,
|
||||
capabilities=fb_caps,
|
||||
model_alias=alias,
|
||||
)
|
||||
result = self._try_stream(fb_client, fb_model, msgs, provider=fb_provider)
|
||||
if fb_tracker:
|
||||
fb_tracker.record_success()
|
||||
return result
|
||||
@@ -1677,8 +1639,6 @@ class ChatSession:
|
||||
model: str,
|
||||
msgs: list[dict[str, Any]],
|
||||
provider: LLMProvider | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
model_alias: str | None = None,
|
||||
) -> Iterator[StreamChunk]:
|
||||
"""Attempt a streaming API call with retries on transient errors."""
|
||||
prov = provider or self._provider
|
||||
@@ -1710,12 +1670,9 @@ class ChatSession:
|
||||
max_tokens=self.max_tokens,
|
||||
temperature=self.temperature,
|
||||
reasoning_effort=self.reasoning_effort,
|
||||
extra_params=self._provider_extra_params(
|
||||
provider=prov, model_alias=model_alias
|
||||
),
|
||||
extra_params=self._provider_extra_params(provider=prov),
|
||||
deferred_names=self._get_deferred_names(),
|
||||
cancel_ref=self._cancel_ref,
|
||||
capabilities=capabilities or self._get_capabilities(prov, model),
|
||||
)
|
||||
except Exception as e:
|
||||
ename = type(e).__name__
|
||||
@@ -5424,12 +5381,10 @@ class ChatSession:
|
||||
if not agent_caps.supports_web_search and not self._resolve_search_client():
|
||||
tools = _without_tool(tools, "web_search")
|
||||
|
||||
# Build extra params for agent calls — resolve server compat from the
|
||||
# agent's own model alias, not the session's primary model.
|
||||
# Build extra params for agent calls
|
||||
agent_extra = self._provider_extra_params(
|
||||
reasoning_effort=reasoning_effort,
|
||||
provider=agent_provider,
|
||||
model_alias=agent_alias,
|
||||
)
|
||||
|
||||
def _api_call(
|
||||
@@ -5448,7 +5403,6 @@ class ChatSession:
|
||||
temperature=self.temperature,
|
||||
reasoning_effort=reasoning_effort or self.reasoning_effort,
|
||||
extra_params=agent_extra,
|
||||
capabilities=agent_caps,
|
||||
)
|
||||
except Exception as e:
|
||||
ename = type(e).__name__
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -4504,7 +4504,7 @@ function _loadHls(callback) {
|
||||
if (_hlsState === "loading") return;
|
||||
_hlsState = "loading";
|
||||
var script = document.createElement("script");
|
||||
script.src = "/shared/hls-1.6.16/hls.min.js";
|
||||
script.src = "/shared/hls-1.6.15/hls.min.js";
|
||||
script.onload = function () {
|
||||
_hlsState = "ready";
|
||||
var q = _hlsQueue;
|
||||
|
||||
Reference in New Issue
Block a user