diff --git a/tests/test_audio.py b/tests/test_audio.py new file mode 100644 index 00000000..d2756a59 --- /dev/null +++ b/tests/test_audio.py @@ -0,0 +1,231 @@ +"""Unit tests for the STT/TTS audio helper (model-role resolution + backends). + +``transcribe`` / ``synthesize`` are exercised through the registry boundary +with a mocked OpenAI-SDK client (mocking ``client.audio.*``), so the real +helper code runs end-to-end without a network call. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from turnstone.core import audio + + +class _Cfg: + """Stand-in for ModelConfig — only the fields audio.py reads.""" + + def __init__(self, model: str, capabilities: dict | None = None) -> None: + self.model = model + self.capabilities = capabilities or {} + + +class _FakeConfigStore: + def __init__(self, **values: str) -> None: + self._values = values + + def get(self, key: str, default: str = "") -> str: + return self._values.get(key, default) + + +class _FakeRegistry: + """Minimal registry exposing the surface audio.py uses.""" + + def __init__(self, alias: str, cfg: _Cfg, client: object) -> None: + self._alias = alias + self._cfg = cfg + self._client = client + + def has_alias(self, alias: str) -> bool: + return alias == self._alias + + def get_config(self, alias: str) -> _Cfg: + if alias != self._alias: + raise ValueError(alias) + return self._cfg + + def resolve(self, alias: str | None = None): + if alias not in (None, self._alias): + raise ValueError(alias) + return self._client, self._cfg.model, self._cfg + + +# --------------------------------------------------------------------------- +# Capability gating +# --------------------------------------------------------------------------- + + +class TestModelSupportsRole: + def test_explicit_flag_wins(self): + assert audio.model_supports_role(_Cfg("anything", {"supports_transcription": True}), "stt") + # Explicit False overrides the would-be inference from the model name. + assert not audio.model_supports_role( + _Cfg("gpt-4o-mini-tts", {"supports_speech_synthesis": False}), "tts" + ) + + def test_infers_known_openai_audio_models(self): + assert audio.model_supports_role(_Cfg("gpt-4o-mini-transcribe"), "stt") + assert audio.model_supports_role(_Cfg("whisper-1"), "stt") + assert audio.model_supports_role(_Cfg("gpt-4o-mini-tts"), "tts") + assert audio.model_supports_role(_Cfg("tts-1"), "tts") + + def test_chat_model_not_eligible(self): + assert not audio.model_supports_role(_Cfg("gpt-5"), "stt") + # Anthropic has no audio API — gated out of every audio role. + assert not audio.model_supports_role(_Cfg("claude-opus-4-8"), "tts") + assert not audio.model_supports_role(_Cfg("claude-opus-4-8"), "stt") + + def test_unknown_role(self): + assert not audio.model_supports_role(_Cfg("whisper-1"), "vision_eval") + + def test_hint_seed_lists_are_pinned(self): + # Mirrored verbatim in admin.js AUDIO_MODEL_HINTS — if these change, + # update the JS dropdown gate too (this pin makes the change deliberate). + assert audio._AUDIO_MODEL_HINTS == { + "stt": ("transcribe", "whisper", "-asr"), + "tts": ("tts-", "-tts"), + } + + +# --------------------------------------------------------------------------- +# Role resolution +# --------------------------------------------------------------------------- + + +class TestResolveRoleAlias: + def test_resolves_configured_capable_alias(self): + reg = _FakeRegistry("voice", _Cfg("gpt-4o-mini-transcribe"), MagicMock()) + cs = _FakeConfigStore(**{"audio.stt_model_alias": "voice"}) + assert audio.resolve_role_alias(config_store=cs, registry=reg, role="stt") == "voice" + + def test_none_when_unset(self): + reg = _FakeRegistry("voice", _Cfg("gpt-4o-mini-transcribe"), MagicMock()) + assert ( + audio.resolve_role_alias(config_store=_FakeConfigStore(), registry=reg, role="stt") + is None + ) + + def test_none_when_alias_missing_from_registry(self): + reg = _FakeRegistry("voice", _Cfg("gpt-4o-mini-transcribe"), MagicMock()) + cs = _FakeConfigStore(**{"audio.stt_model_alias": "ghost"}) + assert audio.resolve_role_alias(config_store=cs, registry=reg, role="stt") is None + + def test_none_when_alias_not_capability_eligible(self): + # Alias exists but its model can't do TTS -> gated out (Anthropic case). + reg = _FakeRegistry("brain", _Cfg("claude-opus-4-8"), MagicMock()) + cs = _FakeConfigStore(**{"audio.tts_model_alias": "brain"}) + assert audio.resolve_role_alias(config_store=cs, registry=reg, role="tts") is None + + def test_none_when_no_registry_or_store(self): + assert audio.resolve_role_alias(config_store=None, registry=None, role="stt") is None + + +# --------------------------------------------------------------------------- +# transcribe / synthesize — boundary: mocked OpenAI-SDK client +# --------------------------------------------------------------------------- + + +class TestTranscribe: + def test_calls_audio_transcriptions_and_returns_text(self): + client = MagicMock() + client.audio.transcriptions.create.return_value = MagicMock(text=" hello world ") + reg = _FakeRegistry("voice", _Cfg("gpt-4o-mini-transcribe"), client) + res = audio.transcribe( + registry=reg, alias="voice", data=b"RIFFfake", filename="speech.webm" + ) + assert res.transcript == "hello world" + assert res.model_alias == "voice" + assert res.model == "gpt-4o-mini-transcribe" + kwargs = client.audio.transcriptions.create.call_args.kwargs + assert kwargs["model"] == "gpt-4o-mini-transcribe" + assert kwargs["file"] == ("speech.webm", b"RIFFfake") + + def test_prompt_forwarded_when_set(self): + client = MagicMock() + client.audio.transcriptions.create.return_value = MagicMock(text="ok") + reg = _FakeRegistry("voice", _Cfg("whisper-1"), client) + audio.transcribe( + registry=reg, alias="voice", data=b"x", filename="a.wav", prompt="ACME jargon" + ) + assert client.audio.transcriptions.create.call_args.kwargs["prompt"] == "ACME jargon" + + def test_prompt_omitted_when_blank(self): + client = MagicMock() + client.audio.transcriptions.create.return_value = MagicMock(text="ok") + reg = _FakeRegistry("voice", _Cfg("whisper-1"), client) + audio.transcribe(registry=reg, alias="voice", data=b"x", filename="a.wav") + assert "prompt" not in client.audio.transcriptions.create.call_args.kwargs + + def test_backend_failure_raises_backend_error(self): + client = MagicMock() + client.audio.transcriptions.create.side_effect = RuntimeError("boom") + reg = _FakeRegistry("voice", _Cfg("whisper-1"), client) + with pytest.raises(audio.AudioBackendError): + audio.transcribe(registry=reg, alias="voice", data=b"x", filename="a.wav") + + +class TestSynthesize: + def test_calls_audio_speech_and_returns_bytes(self): + client = MagicMock() + speech = MagicMock() + speech.read.return_value = b"RIFF...wavbytes" + client.audio.speech.create.return_value = speech + reg = _FakeRegistry("voice", _Cfg("gpt-4o-mini-tts"), client) + res = audio.synthesize(registry=reg, alias="voice", text="hi", voice="nova") + assert res.audio_bytes == b"RIFF...wavbytes" + assert res.media_type == "audio/mpeg" + assert res.model_alias == "voice" + kwargs = client.audio.speech.create.call_args.kwargs + assert kwargs["voice"] == "nova" + assert kwargs["input"] == "hi" + + def test_default_voice_when_empty(self): + client = MagicMock() + client.audio.speech.create.return_value = MagicMock(read=lambda: b"a") + reg = _FakeRegistry("voice", _Cfg("gpt-4o-mini-tts"), client) + audio.synthesize(registry=reg, alias="voice", text="hi", voice="") + assert client.audio.speech.create.call_args.kwargs["voice"] == "alloy" + + def test_backend_failure_raises_backend_error(self): + client = MagicMock() + client.audio.speech.create.side_effect = RuntimeError("down") + reg = _FakeRegistry("voice", _Cfg("gpt-4o-mini-tts"), client) + with pytest.raises(audio.AudioBackendError): + audio.synthesize(registry=reg, alias="voice", text="hi", voice="nova") + + +class TestOpenAIAudioModelsKnown: + """The current OpenAI STT/TTS lineup is registered in the static capability + table, so the admin 'suggested capabilities' recognizes them and they show + in the known-models list. (Role gating also works via name inference for + openai-compatible/local backends that aren't in the static table.)""" + + def test_stt_models_flagged(self): + from turnstone.core.providers import lookup_model_capabilities + + for m in ( + "whisper-1", + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + "gpt-4o-transcribe-diarize", # prefix variant + ): + caps = lookup_model_capabilities("openai", m) or {} + assert caps.get("supports_transcription") is True, m + assert caps.get("supports_speech_synthesis") is False, m + + def test_tts_models_flagged(self): + from turnstone.core.providers import lookup_model_capabilities + + for m in ("tts-1", "tts-1-hd", "gpt-4o-mini-tts"): # tts-1-hd is a prefix variant + caps = lookup_model_capabilities("openai", m) or {} + assert caps.get("supports_speech_synthesis") is True, m + assert caps.get("supports_transcription") is False, m + + def test_chat_model_has_no_audio_flags(self): + from turnstone.core.providers import lookup_model_capabilities + + caps = lookup_model_capabilities("openai", "gpt-5") or {} + assert not caps.get("supports_transcription") + assert not caps.get("supports_speech_synthesis") diff --git a/tests/test_openapi.py b/tests/test_openapi.py index 03ff500e..3004a660 100644 --- a/tests/test_openapi.py +++ b/tests/test_openapi.py @@ -41,12 +41,29 @@ class TestServerSpec: "/v1/api/command", "/v1/api/events/global", "/v1/api/workstreams/new", + "/v1/api/workstreams/{ws_id}/speech-to-text", + "/v1/api/tts", "/v1/api/auth/login", "/v1/api/auth/logout", "/health", } assert expected.issubset(paths), f"Missing: {expected - paths}" + def test_voice_endpoints_documented(self): + from turnstone.api.server_spec import build_server_spec + + spec = build_server_spec() + stt = spec["paths"]["/v1/api/workstreams/{ws_id}/speech-to-text"]["post"] + tts = spec["paths"]["/v1/api/tts"]["post"] + assert "responses" in stt + assert "requestBody" in tts + assert "application/json" in tts["requestBody"]["content"] + schemas = spec["components"]["schemas"] + assert "capabilities" in schemas["AvailableModelInfo"]["properties"] + models_props = schemas["ListAvailableModelsResponse"]["properties"] + assert "stt_default_alias" in models_props + assert "tts_default_alias" in models_props + def test_workstream_history_has_limit_query_param(self): """Mirror of the coord-side history limit param test — server now exposes the same endpoint via the lifted factory.""" diff --git a/tests/test_server_attachments_endpoints.py b/tests/test_server_attachments_endpoints.py index ab587f2b..34aacf73 100644 --- a/tests/test_server_attachments_endpoints.py +++ b/tests/test_server_attachments_endpoints.py @@ -1082,3 +1082,210 @@ class TestServiceScopedActorFlow: atts = captured["attachments"] assert atts is not None and len(atts) == 1 assert atts[0].attachment_id == aid + + +# --------------------------------------------------------------------------- +# Voice I/O (STT / TTS) endpoints +# --------------------------------------------------------------------------- + + +class _VoiceConfigStore: + def __init__(self, **values: str) -> None: + self._values = dict(values) + + def get(self, key: str, default: str = "") -> str: + return self._values.get(key, default) + + +@pytest.fixture +def voice_app_client(tmp_path): + """App wired with an audio-capable registry alias + a mocked OpenAI client. + + The mock is injected into ``registry._clients`` so the real endpoint → + resolve_role_alias → transcribe/synthesize path runs end-to-end with only + the SDK network call stubbed. + """ + import sqlalchemy as sa + + import turnstone.server as srv_mod + from turnstone.core.memory import register_workstream + from turnstone.core.metrics import MetricsCollector + from turnstone.core.model_registry import ModelConfig, ModelRegistry + from turnstone.core.storage import init_storage, reset_storage + from turnstone.core.storage._registry import get_storage + from turnstone.core.storage._schema import workstreams as ws_tbl + + db_path = tmp_path / "voice.db" + reset_storage() + init_storage("sqlite", path=str(db_path), run_migrations=False) + + srv_mod._metrics = MetricsCollector() + srv_mod._metrics.model = "test-model" + + register_workstream("ws-A", name="A") + with get_storage()._conn() as conn: + conn.execute(sa.update(ws_tbl).where(ws_tbl.c.ws_id == "ws-A").values(user_id="userA")) + conn.commit() + + registry = ModelRegistry( + models={ + "voice": ModelConfig( + "voice", + "http://localhost:9/v1", + "none", + "gpt-4o-mini-tts", + capabilities={ + "supports_transcription": True, + "supports_speech_synthesis": True, + }, + ), + }, + default="voice", + ) + mock_client = MagicMock() + mock_client.audio.transcriptions.create.return_value = MagicMock(text="hello from speech") + speech = MagicMock() + speech.read.return_value = b"RIFF\x00\x00fakeaudio" + mock_client.audio.speech.create.return_value = speech + registry._clients["voice"] = mock_client # bypass real SDK client construction + + config_store = _VoiceConfigStore( + **{ + "audio.stt_model_alias": "voice", + "audio.tts_model_alias": "voice", + "audio.tts_voice": "alloy", + } + ) + + mock_mgr = MagicMock() + mock_mgr.get.return_value = None + mock_mgr.list_all.return_value = [] + mock_mgr.max_active = 10 + + app = srv_mod.create_app( + workstreams=mock_mgr, + global_queue=queue.Queue(), + global_listeners=[], + global_listeners_lock=threading.Lock(), + skip_permissions=False, + jwt_secret=_TEST_JWT_SECRET, + registry=registry, + config_store=config_store, + ) + client = TestClient(app, raise_server_exceptions=False) + try: + yield client, mock_client + finally: + client.close() + reset_storage() + + +class TestSpeechToText: + def test_unconfigured_returns_503(self, app_client): + client, _ = app_client + resp = client.post( + "/v1/api/workstreams/ws-A/speech-to-text", + files={"audio": ("speech.webm", b"RIFFfake", "audio/webm")}, + headers=_auth("userA"), + ) + assert resp.status_code == 503 + assert "not configured" in resp.json()["error"] + + def test_happy_path_returns_transcript(self, voice_app_client): + client, mock_client = voice_app_client + resp = client.post( + "/v1/api/workstreams/ws-A/speech-to-text", + files={"audio": ("speech.webm", b"RIFFfake", "audio/webm")}, + headers=_auth("userA"), + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["transcript"] == "hello from speech" + assert body["model_alias"] == "voice" + assert mock_client.audio.transcriptions.create.called + + def test_empty_upload_returns_400(self, voice_app_client): + client, _ = voice_app_client + resp = client.post( + "/v1/api/workstreams/ws-A/speech-to-text", + files={"audio": ("speech.webm", b"", "audio/webm")}, + headers=_auth("userA"), + ) + assert resp.status_code == 400 + + def test_silence_returns_422(self, voice_app_client): + # A successful transcription with no speech is not a backend failure. + client, mock_client = voice_app_client + mock_client.audio.transcriptions.create.return_value = MagicMock(text=" ") + resp = client.post( + "/v1/api/workstreams/ws-A/speech-to-text", + files={"audio": ("speech.webm", b"RIFFfake", "audio/webm")}, + headers=_auth("userA"), + ) + assert resp.status_code == 422 + assert "No speech detected" in resp.json()["error"] + + def test_backend_failure_returns_masked_502(self, voice_app_client): + # Backend SDK error detail must not leak into the client-facing body. + client, mock_client = voice_app_client + mock_client.audio.transcriptions.create.side_effect = RuntimeError( + "Error code: 401 - internal-host:9 invalid_api_key" + ) + resp = client.post( + "/v1/api/workstreams/ws-A/speech-to-text", + files={"audio": ("speech.webm", b"RIFFfake", "audio/webm")}, + headers=_auth("userA"), + ) + assert resp.status_code == 502 + body = resp.json() + assert body["error"] == "Speech transcription backend failed" + assert "internal-host" not in body["error"] + + def test_unknown_workstream_404(self, voice_app_client): + # Trusted-team semantics: ownership isn't row-enforced, but a + # nonexistent workstream is masked as 404 (no enumeration). + client, _ = voice_app_client + resp = client.post( + "/v1/api/workstreams/ws-DOES-NOT-EXIST/speech-to-text", + files={"audio": ("speech.webm", b"RIFFfake", "audio/webm")}, + headers=_auth("userA"), + ) + assert resp.status_code == 404 + + +class TestTextToSpeech: + def test_unconfigured_returns_503(self, app_client): + client, _ = app_client + resp = client.post("/v1/api/tts", json={"text": "hello"}, headers=_auth("userA")) + assert resp.status_code == 503 + + def test_happy_path_returns_audio(self, voice_app_client): + client, mock_client = voice_app_client + resp = client.post("/v1/api/tts", json={"text": "hello"}, headers=_auth("userA")) + assert resp.status_code == 200, resp.text + assert resp.headers["content-type"].startswith("audio/") + assert resp.content == b"RIFF\x00\x00fakeaudio" + assert resp.headers.get("x-model-alias") == "voice" + # audio.tts_voice setting supplies the voice when the body omits one. + assert mock_client.audio.speech.create.call_args.kwargs["voice"] == "alloy" + + def test_empty_text_returns_400(self, voice_app_client): + client, _ = voice_app_client + resp = client.post("/v1/api/tts", json={"text": " "}, headers=_auth("userA")) + assert resp.status_code == 400 + + def test_too_long_text_returns_400(self, voice_app_client): + client, _ = voice_app_client + resp = client.post("/v1/api/tts", json={"text": "x" * 9000}, headers=_auth("userA")) + assert resp.status_code == 400 + + def test_backend_failure_returns_masked_502(self, voice_app_client): + client, mock_client = voice_app_client + mock_client.audio.speech.create.side_effect = RuntimeError( + "Error code: 500 - internal-host:9 boom" + ) + resp = client.post("/v1/api/tts", json={"text": "hello"}, headers=_auth("userA")) + assert resp.status_code == 502 + body = resp.json() + assert body["error"] == "Speech synthesis backend failed" + assert "internal-host" not in body["error"] diff --git a/tests/test_server_available_models.py b/tests/test_server_available_models.py index 958adca3..81368521 100644 --- a/tests/test_server_available_models.py +++ b/tests/test_server_available_models.py @@ -52,6 +52,7 @@ class _StubRegistry: alias=alias, model=self._aliases[alias], provider="openai-compatible", + capabilities={}, ) diff --git a/tests/test_settings_registry.py b/tests/test_settings_registry.py index db581cfe..db44149e 100644 --- a/tests/test_settings_registry.py +++ b/tests/test_settings_registry.py @@ -28,6 +28,24 @@ class TestValidateKey: with pytest.raises(ValueError, match="Unknown setting"): validate_key("nonexistent.key") + def test_audio_role_settings_registered(self): + for key in ( + "audio.stt_model_alias", + "audio.stt_prompt", + "audio.tts_model_alias", + "audio.tts_voice", + ): + defn = validate_key(key) + assert defn.key == key + assert defn.type == "str" + assert defn.section == "audio" + assert key in SETTINGS + # Voice has a concrete default; the role aliases + prompt default to empty. + assert validate_key("audio.stt_model_alias").default == "" + assert validate_key("audio.tts_model_alias").default == "" + assert validate_key("audio.stt_prompt").default == "" + assert validate_key("audio.tts_voice").default == "alloy" + # --------------------------------------------------------------------------- # validate_value — type coercion diff --git a/turnstone/api/server_schemas.py b/turnstone/api/server_schemas.py index 52c9a6d3..018e6cff 100644 --- a/turnstone/api/server_schemas.py +++ b/turnstone/api/server_schemas.py @@ -87,6 +87,21 @@ class ListAttachmentsResponse(BaseModel): ) +class SpeechToTextResponse(BaseModel): + """Transcript returned for the browser to place into the composer.""" + + status: str = Field(default="ok", description="Request outcome") + transcript: str = Field(description="Transcribed text") + model_alias: str = Field(default="", description="STT role alias used") + + +class TextToSpeechRequest(BaseModel): + text: str = Field(description="Text to synthesize") + voice: str = Field( + default="", description="Optional voice override (else audio.tts_voice setting)" + ) + + class ApproveRequest(BaseModel): approved: bool = Field(description="True to approve, false to deny") feedback: str | None = Field(default=None, description="Optional denial reason") @@ -628,6 +643,10 @@ class AvailableModelInfo(BaseModel): alias: str model: str provider: str + capabilities: dict[str, Any] = Field( + default_factory=dict, + description="Operator-set capability flags for this alias (e.g. supports_transcription)", + ) class ListAvailableModelsResponse(BaseModel): @@ -635,3 +654,11 @@ class ListAvailableModelsResponse(BaseModel): default_alias: str = "" channel_default_alias: str = "" judge_default_alias: str = "" + stt_default_alias: str = Field( + default="", + description="Effective speech-to-text role alias (blank = voice input disabled)", + ) + tts_default_alias: str = Field( + default="", + description="Effective text-to-speech role alias (blank = voice output disabled)", + ) diff --git a/turnstone/api/server_spec.py b/turnstone/api/server_spec.py index d7107c94..d3265ce2 100644 --- a/turnstone/api/server_spec.py +++ b/turnstone/api/server_spec.py @@ -43,6 +43,8 @@ from turnstone.api.server_schemas import ( SendRequest, SendResponse, SkillSummary, + SpeechToTextResponse, + TextToSpeechRequest, UploadAttachmentResponse, WorkstreamDetailResponse, WorkstreamHistoryResponse, @@ -313,6 +315,28 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [ error_codes=[403, 404], tags=["Attachments"], ), + # --- Voice I/O --- + EndpointSpec( + "/v1/api/workstreams/{ws_id}/speech-to-text", + "POST", + "Transcribe a short audio clip (multipart/form-data, field 'audio') " + "using the configured STT model role. Returns the transcript for the " + "client to place into the composer; this endpoint never sends on the " + "user's behalf. Returns 503 when no STT role is configured.", + response_model=SpeechToTextResponse, + error_codes=[400, 403, 404, 413, 502, 503], + tags=["Attachments"], + ), + EndpointSpec( + "/v1/api/tts", + "POST", + "Synthesize text to speech audio for browser playback using the " + "configured TTS model role. Returns audio bytes; 503 when no TTS " + "role is configured.", + request_model=TextToSpeechRequest, + error_codes=[400, 502, 503], + tags=["Chat"], + ), # --- Saved workstreams --- EndpointSpec( "/v1/api/workstreams/saved", @@ -495,6 +519,8 @@ _ALL_MODELS: list[type[BaseModel]] = [ ListSavedWorkstreamsResponse, UploadAttachmentResponse, ListAttachmentsResponse, + SpeechToTextResponse, + TextToSpeechRequest, HealthResponse, SaveMemoryRequest, MemoryInfo, diff --git a/turnstone/console/static/admin.js b/turnstone/console/static/admin.js index 66e28d31..b22a27e0 100644 --- a/turnstone/console/static/admin.js +++ b/turnstone/console/static/admin.js @@ -22,6 +22,8 @@ const ALIAS_SETTING_KEYS = [ "model.plan_alias", "model.task_alias", "channels.default_model_alias", + "audio.stt_model_alias", + "audio.tts_model_alias", ]; // Settings whose empty option means "inherit from a fallback chain", as @@ -2747,6 +2749,7 @@ function _settingsSectionLabel(section) { server: "Server", cluster: "Cluster", channels: "Channels", + audio: "Voice", mcp: "MCP", ratelimit: "Rate Limiting", health: "Health", @@ -5192,8 +5195,58 @@ const MODEL_ROLES = [ aliasKey: "channels.default_model_alias", fallbackKind: "default", }, + { + label: "Speech-to-text", + description: + "Transcribes microphone audio in the workstream composer (voice input). Empty disables the mic affordance — there is no audio-capable session fallback.", + aliasKey: "audio.stt_model_alias", + fallbackKind: "disabled", + mediaCapability: "supports_transcription", + mediaRole: "stt", + }, + { + label: "Text-to-speech", + description: + "Synthesizes assistant replies for playback (voice output). Empty disables the play affordance.", + aliasKey: "audio.tts_model_alias", + fallbackKind: "disabled", + mediaCapability: "supports_speech_synthesis", + mediaRole: "tts", + }, ]; +// Whether a model definition is eligible for an audio role. Mirrors +// turnstone/core/audio.py model_supports_role: an explicit capability flag +// wins; otherwise infer from well-known OpenAI audio model names so a stock +// gpt-4o-mini-transcribe / -tts / whisper alias shows up without hand-ticking. +// Known-model-name hints, mirrored VERBATIM from _AUDIO_MODEL_HINTS in +// turnstone/core/audio.py (the canonical source). A substring match marks +// eligibility. Keep these two lists in sync — the Python side is pinned by +// tests/test_audio.py so any change there is deliberate. +const AUDIO_MODEL_HINTS = { + stt: ["transcribe", "whisper", "-asr"], + tts: ["tts-", "-tts"], +}; + +function _audioModelEligible(md, capFlag, mediaRole) { + let caps = md && md.capabilities; + if (typeof caps === "string") { + try { + caps = JSON.parse(caps || "{}"); + } catch (e) { + caps = {}; + } + } + if (!caps || typeof caps !== "object") caps = {}; + if (Object.prototype.hasOwnProperty.call(caps, capFlag)) + return !!caps[capFlag]; + const name = ((md && md.model) || "").toLowerCase(); + const hints = AUDIO_MODEL_HINTS[mediaRole] || []; + return hints.some(function (h) { + return name.indexOf(h) !== -1; + }); +} + // Roles sub-tab reads/writes via ``/v1/api/admin/settings`` which // requires ``admin.settings`` — different from the ``admin.models`` // permission gating the Models tab itself. When the user has Models @@ -5387,7 +5440,9 @@ function _renderModelRoles(container, values, schema) { // "(default — )". const blank = document.createElement("option"); blank.value = ""; - if (role.fallbackKind === "inherit") { + if (role.fallbackKind === "disabled") { + blank.textContent = "(disabled — voice off)"; + } else if (role.fallbackKind === "inherit") { blank.textContent = "(inherit)"; } else { let defaultDef = null; @@ -5412,10 +5467,18 @@ function _renderModelRoles(container, values, schema) { } } aliasSel.appendChild(blank); + // Audio roles only list aliases capable of the role (capability flag or + // known-model inference); other roles list every enabled alias. + let roleAliases = enabledAliases; + if (role.mediaCapability) { + roleAliases = enabledAliases.filter(function (md) { + return _audioModelEligible(md, role.mediaCapability, role.mediaRole); + }); + } const currentAlias = aliasInfo.value || ""; let matched = false; - for (let m = 0; m < enabledAliases.length; m++) { - const md = enabledAliases[m]; + for (let m = 0; m < roleAliases.length; m++) { + const md = roleAliases[m]; const opt = document.createElement("option"); opt.value = md.alias; opt.textContent = diff --git a/turnstone/core/audio.py b/turnstone/core/audio.py new file mode 100644 index 00000000..a4436846 --- /dev/null +++ b/turnstone/core/audio.py @@ -0,0 +1,186 @@ +"""Speech-to-text and text-to-speech over the OpenAI audio wire protocol. + +STT/TTS are model *roles* (settings ``stt.model_alias`` / ``tts.model_alias``), +resolved the same way as ``judge.model``. A role resolves to a registry alias +whose client is an OpenAI-SDK-compatible client (provider ``openai`` / +``openai-compatible`` / ``google`` / ``xai``); the same ``client.audio.*`` calls +then work against OpenAI, a local vLLM / vLLM-Omni server, or any compatible +backend — selected purely by the alias's ``base_url``. Anthropic models have no +audio API, so they are capability-gated out of these roles (Claude stays valid +as the agent model). + +No local in-process models and no silent fallbacks: an unconfigured or failing +backend is surfaced as a typed error the endpoint maps to 503 / 502. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +# Setting key + capability flag per media role. Kept deliberately small; the +# perception/eval roles (vision_eval/av_eval/intent_eval) are a later slice. +_ROLE_SETTING: dict[str, str] = { + "stt": "audio.stt_model_alias", + "tts": "audio.tts_model_alias", +} +_ROLE_CAPABILITY: dict[str, str] = { + "stt": "supports_transcription", + "tts": "supports_speech_synthesis", +} + +# Known-model-name hints for capability inference. The explicit +# ``capabilities`` flag is ALWAYS canonical (see ``model_supports_role``); these +# only fill the gap so a stock OpenAI audio model alias works without an +# operator hand-ticking a box. A substring match against the lowercased model +# name marks eligibility. +# +# IMPORTANT: these lists are mirrored verbatim in the admin UI +# (``_audioModelEligible`` / ``AUDIO_*_MODEL_HINTS`` in +# ``turnstone/console/static/admin.js``) so the Models -> Roles dropdown offers +# exactly the aliases the endpoints will accept. Keep the two in sync; +# ``tests/test_audio.py`` pins these values so a change here is deliberate. +_AUDIO_MODEL_HINTS: dict[str, tuple[str, ...]] = { + "stt": ("transcribe", "whisper", "-asr"), + "tts": ("tts-", "-tts"), +} + +# response_format -> Content-Type for synthesized audio. +_MEDIA_TYPES: dict[str, str] = { + "mp3": "audio/mpeg", + "opus": "audio/ogg", + "aac": "audio/aac", + "flac": "audio/flac", + "wav": "audio/wav", + "pcm": "audio/pcm", +} + +_DEFAULT_VOICE = "alloy" + + +@dataclass(frozen=True) +class TranscriptionResult: + transcript: str + model_alias: str + model: str + + +@dataclass(frozen=True) +class SpeechResult: + audio_bytes: bytes + media_type: str + model_alias: str + model: str + + +class AudioUnavailableError(RuntimeError): + """No capable backend is configured for the requested role (maps to 503).""" + + +class AudioBackendError(RuntimeError): + """A configured audio backend failed during execution (maps to 502).""" + + +def _infer_audio_capability(model: str, role: str) -> bool: + """Best-effort capability default for well-known audio model names. + + The explicit ``capabilities`` flag always wins (see ``model_supports_role``); + this only fills the gap so a stock ``gpt-4o-mini-transcribe`` / + ``gpt-4o-mini-tts`` / ``whisper-1`` alias works without an operator + hand-ticking a capability box. Rules live in :data:`_AUDIO_MODEL_HINTS` + (mirrored in admin.js). + """ + name = (model or "").strip().lower() + if not name: + return False + return any(hint in name for hint in _AUDIO_MODEL_HINTS.get(role, ())) + + +def model_supports_role(cfg: Any, role: str) -> bool: + """Whether the alias's model is eligible for a media *role*. + + Explicit ``capabilities[]`` wins; otherwise fall back to a + known-model-name inference for OpenAI audio models. + """ + flag = _ROLE_CAPABILITY.get(role) + if not flag: + return False + caps = getattr(cfg, "capabilities", None) or {} + if flag in caps: + return bool(caps.get(flag)) + return _infer_audio_capability(getattr(cfg, "model", ""), role) + + +def resolve_role_alias(*, config_store: Any | None, registry: Any | None, role: str) -> str | None: + """Return the configured, capability-eligible alias for *role*, or ``None``. + + Resolution: ``.model_alias`` setting → must exist in the registry → + must be capability-eligible for the role. Any miss returns ``None`` so the + caller surfaces a 503 (or hides the affordance) rather than calling a + backend that can't serve audio. + """ + if registry is None or config_store is None: + return None + key = _ROLE_SETTING.get(role) + if not key: + return None + alias = (config_store.get(key) or "").strip() + if not alias or not registry.has_alias(alias): + return None + if not model_supports_role(registry.get_config(alias), role): + return None + return alias + + +def transcribe( + *, registry: Any, alias: str, data: bytes, filename: str, prompt: str = "" +) -> TranscriptionResult: + """Transcribe ``data`` using the STT role alias's audio backend. + + ``prompt`` (when non-empty) is forwarded as the transcription ``prompt`` + parameter to bias the model toward domain vocabulary / instructions; it is + omitted entirely when blank so backends that don't accept it aren't sent it. + """ + try: + client, model, _cfg = registry.resolve(alias) + except Exception as exc: # unknown/removed alias + raise AudioUnavailableError(f"STT model alias {alias!r} is not available") from exc + kwargs: dict[str, Any] = { + "model": model, + "file": (filename or "speech.webm", data), + "response_format": "json", + } + if prompt: + kwargs["prompt"] = prompt + try: + resp = client.audio.transcriptions.create(**kwargs) + transcript = (getattr(resp, "text", "") or "").strip() + except Exception as exc: + raise AudioBackendError(f"Transcription backend failed: {exc}") from exc + return TranscriptionResult(transcript=transcript, model_alias=alias, model=model) + + +def synthesize( + *, registry: Any, alias: str, text: str, voice: str, response_format: str = "mp3" +) -> SpeechResult: + """Synthesize ``text`` to speech using the TTS role alias's audio backend.""" + try: + client, model, _cfg = registry.resolve(alias) + except Exception as exc: + raise AudioUnavailableError(f"TTS model alias {alias!r} is not available") from exc + try: + resp = client.audio.speech.create( + model=model, + voice=voice or _DEFAULT_VOICE, + input=text, + response_format=response_format, + ) + audio_bytes = resp.read() if hasattr(resp, "read") else bytes(getattr(resp, "content", b"")) + except Exception as exc: + raise AudioBackendError(f"TTS backend failed: {exc}") from exc + return SpeechResult( + audio_bytes=audio_bytes, + media_type=_MEDIA_TYPES.get(response_format, "audio/mpeg"), + model_alias=alias, + model=model, + ) diff --git a/turnstone/core/auth.py b/turnstone/core/auth.py index ec19d6e1..b97a0f57 100644 --- a/turnstone/core/auth.py +++ b/turnstone/core/auth.py @@ -309,6 +309,7 @@ WRITE_PATHS: frozenset[str] = frozenset( "/api/workstreams/new", "/api/cluster/workstreams/new", "/api/memories", + "/api/tts", } ) @@ -694,6 +695,7 @@ def required_scope(method: str, path: str) -> str: "refresh-title", "title", "attachments", + "speech-to-text", "send", "cancel", "rewind", @@ -744,6 +746,7 @@ def required_scope(method: str, path: str) -> str: "refresh-title", "title", "attachments", + "speech-to-text", "send", "cancel", "rewind", diff --git a/turnstone/core/providers/_openai_common.py b/turnstone/core/providers/_openai_common.py index 86ad0ba9..44c6d600 100644 --- a/turnstone/core/providers/_openai_common.py +++ b/turnstone/core/providers/_openai_common.py @@ -197,6 +197,39 @@ OPENAI_CAPABILITIES: dict[str, ModelCapabilities] = { reasoning_effort_values=(), supports_vision=True, ), + # Audio models — not chat/session models; used only as STT/TTS roles via + # the /v1/audio/transcriptions and /v1/audio/speech endpoints. Prefixes + # cover variants: "gpt-4o-transcribe" → "-diarize", "tts-1" → "tts-1-hd". + "whisper-1": ModelCapabilities( + supports_temperature=False, + supports_streaming=False, + supports_tools=False, + supports_transcription=True, + ), + "gpt-4o-transcribe": ModelCapabilities( + supports_temperature=False, + supports_streaming=False, + supports_tools=False, + supports_transcription=True, + ), + "gpt-4o-mini-transcribe": ModelCapabilities( + supports_temperature=False, + supports_streaming=False, + supports_tools=False, + supports_transcription=True, + ), + "tts-1": ModelCapabilities( + supports_temperature=False, + supports_streaming=False, + supports_tools=False, + supports_speech_synthesis=True, + ), + "gpt-4o-mini-tts": ModelCapabilities( + supports_temperature=False, + supports_streaming=False, + supports_tools=False, + supports_speech_synthesis=True, + ), } # Default for unknown models (local servers: vLLM, llama.cpp, etc.) diff --git a/turnstone/core/providers/_protocol.py b/turnstone/core/providers/_protocol.py index ccd86d7d..58ea5094 100644 --- a/turnstone/core/providers/_protocol.py +++ b/turnstone/core/providers/_protocol.py @@ -86,6 +86,10 @@ class ModelCapabilities: supports_web_search: bool = False supports_tool_search: bool = False supports_vision: bool = False + # Audio I/O roles (STT / TTS) — not chat behavior; consumed by the audio + # endpoints and the Models -> Roles capability gate (turnstone/core/audio.py). + supports_transcription: bool = False + supports_speech_synthesis: bool = False # Server-side tool types to auto-inject into Responses-API ``tools[]`` # for this model (e.g. ``("web_search",)`` for OpenAI search models, # ``("web_search", "x_search")`` for Grok variants). The diff --git a/turnstone/core/settings_registry.py b/turnstone/core/settings_registry.py index fa8117e0..92c85683 100644 --- a/turnstone/core/settings_registry.py +++ b/turnstone/core/settings_registry.py @@ -404,6 +404,54 @@ def _build_registry() -> dict[str, SettingDef]: "Degraded backends are deprioritised in the fallback chain but requests are never " "blocked. The backend recovers automatically when a request succeeds.", ), + # -- audio / voice roles ------------------------------------------- + # Keys are section-prefixed (audio.*) with distinct leaves so the + # Settings tab (which labels by the key's last segment) doesn't render + # two identical "model_alias" rows. + SettingDef( + "audio.stt_model_alias", + "str", + "", + "Model alias for speech-to-text (empty = voice input disabled)", + "audio", + help="Which registered model alias transcribes microphone audio. Point it at an " + "audio-capable backend (e.g. an OpenAI gpt-4o-transcribe alias, or a local vLLM " + "whisper endpoint). Empty disables the microphone affordance — there is no " + "audio-capable session fallback, so this must name a transcription model. The " + "curated, capability-gated picker lives in Models -> Roles.", + ), + SettingDef( + "audio.stt_prompt", + "str", + "", + "Optional prompt sent with each transcription request", + "audio", + help="Optional text passed to the speech-to-text backend to bias the transcription " + "— useful for domain vocabulary, names, or acronyms, and required by some models " + "(e.g. Gemma-style ASR) that take an instruction prompt. Sent as the OpenAI " + "transcription `prompt` parameter; leave empty to omit it.", + ), + SettingDef( + "audio.tts_model_alias", + "str", + "", + "Model alias for text-to-speech (empty = voice output disabled)", + "audio", + help="Which registered model alias synthesizes assistant speech. Point it at an " + "audio-capable backend (e.g. an OpenAI gpt-4o-mini-tts alias, or a local " + "vLLM-Omni speech endpoint). Empty disables the playback affordance. The curated, " + "capability-gated picker lives in Models -> Roles.", + ), + SettingDef( + "audio.tts_voice", + "str", + "alloy", + "Voice identifier for text-to-speech", + "audio", + help="Voice passed to the TTS backend. OpenAI voices include alloy, echo, fable, " + "onyx, nova, shimmer; local backends (vLLM-Omni, Kokoro) define their own — set " + "this to a voice your configured TTS model backend supports.", + ), # -- judge ---------------------------------------------------------- SettingDef( "judge.enabled", diff --git a/turnstone/server.py b/turnstone/server.py index bd9b7adb..1937a91a 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -1176,6 +1176,7 @@ async def list_available_models(request: Request) -> JSONResponse: "alias": cfg.alias, "model": cfg.model, "provider": cfg.provider, + "capabilities": cfg.capabilities, } ) # Include effective defaults for clients (web UI, channel gateway). @@ -1210,16 +1211,165 @@ async def list_available_models(request: Request) -> JSONResponse: # an explicitly-configured, enabled alias surfaces a concrete default. if judge_default_alias and judge_default_alias not in enabled_aliases: judge_default_alias = "" + # STT/TTS are media roles: report a default only when the configured alias + # exists AND is capability-eligible (the same gate the endpoints apply), so + # the UI shows the mic / playback affordances only when they actually work. + from turnstone.core.audio import resolve_role_alias + + stt_default_alias = resolve_role_alias(config_store=cs, registry=registry, role="stt") or "" + tts_default_alias = resolve_role_alias(config_store=cs, registry=registry, role="tts") or "" return JSONResponse( { "models": models, "default_alias": default_alias, "channel_default_alias": channel_default_alias, "judge_default_alias": judge_default_alias, + "stt_default_alias": stt_default_alias, + "tts_default_alias": tts_default_alias, } ) +_STT_UPLOAD_CAP = 25 * 1024 * 1024 # 25 MiB — generous for short dictation clips +_TTS_TEXT_CAP = 8000 # characters per synthesis request + + +async def speech_to_text(request: Request) -> JSONResponse: + """POST /v1/api/workstreams/{ws_id}/speech-to-text — transcribe one audio clip. + + Multipart body with a single ``audio`` field. Returns the transcript for + the browser to place into the composer; this endpoint never sends on the + user's behalf (no auto-send, no request rewriting). + """ + from turnstone.core.audio import ( + AudioBackendError, + AudioUnavailableError, + resolve_role_alias, + transcribe, + ) + from turnstone.core.web_helpers import read_multipart_file_or_400 + + ws_id = request.path_params.get("ws_id", "") + if not ws_id: + return JSONResponse({"error": "ws_id is required"}, status_code=400) + _user_id, err = _require_ws_access(request, ws_id) + if err: + return err + + registry = getattr(request.app.state, "registry", None) + config_store = getattr(request.app.state, "config_store", None) + alias = resolve_role_alias(config_store=config_store, registry=registry, role="stt") + if not alias: + return JSONResponse( + { + "error": ( + "Speech-to-text is not configured. Assign an STT model role in Models → Roles." + ) + }, + status_code=503, + ) + + got = await read_multipart_file_or_400(request, field="audio", max_bytes=_STT_UPLOAD_CAP) + if isinstance(got, JSONResponse): + return got + filename, _claimed_mime, data = got + if not data: + return JSONResponse({"error": "Empty audio upload"}, status_code=400) + + stt_prompt = "" + if config_store is not None: + stt_prompt = (config_store.get("audio.stt_prompt") or "").strip() + try: + # Blocking SDK round-trip — offload so the shared event loop (and SSE + # streaming) stays responsive. + result = await asyncio.to_thread( + transcribe, + registry=registry, + alias=alias, + data=data, + filename=filename or "speech.webm", + prompt=stt_prompt, + ) + except AudioUnavailableError as exc: + return JSONResponse({"error": str(exc)}, status_code=503) + except AudioBackendError as exc: + # str(exc) wraps the backend SDK error, which can carry upstream + # response detail — log it, but return a static body to the caller. + log.warning("speech_to_text.backend_failed", error=str(exc), exc_info=True) + return JSONResponse({"error": "Speech transcription backend failed"}, status_code=502) + + if not result.transcript: + # Successful call that detected no speech (silence / non-speech audio) + # is not a backend failure — surface it as 422 so the UI can say so + # rather than treating a healthy backend as a bad gateway. + return JSONResponse({"error": "No speech detected"}, status_code=422) + + return JSONResponse( + { + "status": "ok", + "transcript": result.transcript, + "model_alias": result.model_alias, + } + ) + + +async def text_to_speech(request: Request) -> Response: + """POST /v1/api/tts — synthesize assistant text into playable audio.""" + from turnstone.core.audio import ( + AudioBackendError, + AudioUnavailableError, + resolve_role_alias, + synthesize, + ) + from turnstone.core.web_helpers import read_json_or_400 + + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + text = str(body.get("text") or "").strip() + if not text: + return JSONResponse({"error": "text is required"}, status_code=400) + if len(text) > _TTS_TEXT_CAP: + return JSONResponse( + {"error": f"text too long (cap {_TTS_TEXT_CAP} chars)"}, status_code=400 + ) + + registry = getattr(request.app.state, "registry", None) + config_store = getattr(request.app.state, "config_store", None) + alias = resolve_role_alias(config_store=config_store, registry=registry, role="tts") + if not alias: + return JSONResponse( + { + "error": ( + "Text-to-speech is not configured. Assign a TTS model role in Models → Roles." + ) + }, + status_code=503, + ) + + voice = str(body.get("voice") or "").strip() + if not voice and config_store is not None: + voice = (config_store.get("audio.tts_voice") or "").strip() + + try: + # Blocking SDK round-trip — offload off the event loop. + speech = await asyncio.to_thread( + synthesize, registry=registry, alias=alias, text=text, voice=voice + ) + except AudioUnavailableError as exc: + return JSONResponse({"error": str(exc)}, status_code=503) + except AudioBackendError as exc: + # Static body to the caller; backend SDK detail stays in the log. + log.warning("text_to_speech.backend_failed", error=str(exc), exc_info=True) + return JSONResponse({"error": "Speech synthesis backend failed"}, status_code=502) + + return Response( + speech.audio_bytes, + media_type=speech.media_type, + headers={"X-Model-Alias": speech.model_alias}, + ) + + def _count_ws_states(wss: list[Workstream]) -> dict[str, int]: """Count workstream states for health/metrics endpoints.""" counts = dict.fromkeys(("idle", "thinking", "running", "attention", "error"), 0) @@ -3830,6 +3980,14 @@ def create_app( ), ) v1_routes.append(Route("/api/dashboard", dashboard)) + v1_routes.append( + Route( + "/api/workstreams/{ws_id}/speech-to-text", + speech_to_text, + methods=["POST"], + ) + ) + v1_routes.append(Route("/api/tts", text_to_speech, methods=["POST"])) app = Starlette( routes=[ diff --git a/turnstone/shared_static/chat.css b/turnstone/shared_static/chat.css index 6dbec6bf..3d819fb4 100644 --- a/turnstone/shared_static/chat.css +++ b/turnstone/shared_static/chat.css @@ -364,8 +364,115 @@ border-color: var(--border-strong); } +/* Voice I/O — mic dictation button (composer row) mirrors .composer-attach, + plus a recording-pulse state and the per-message TTS playback states. */ +.composer-mic-btn { + display: inline-flex; + align-items: center; + justify-content: center; + background: transparent; + border: 1px solid var(--border-strong); + border-radius: var(--radius-sm); + color: var(--fg-dim); + cursor: pointer; + padding: 4px 8px; + line-height: 1; + height: 32px; + transition: + background 0.12s, + color 0.12s, + border-color 0.12s; +} +.composer-mic-btn:hover { + background: var(--bg-highlight); + color: var(--accent); + border-color: var(--accent-dim); +} +.composer-mic-btn:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 1px; +} +.composer-mic-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} +.composer-mic-btn.is-recording { + color: var(--red, #f87171); + border-color: var(--red, #f87171); + animation: ts-mic-pulse 1.2s ease-in-out infinite; +} +@keyframes ts-mic-pulse { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0.55; + } +} +.msg-tts-btn.is-playing { + color: var(--accent); +} +.msg-tts-btn.is-busy { + opacity: 0.6; + cursor: progress; +} +@media (prefers-reduced-motion: reduce) { + .composer-mic-btn.is-recording { + /* No pulse — keep an unmistakable static cue instead of the only signal + being a subtle icon swap. */ + animation: none; + background: var(--red-glow); + } +} +/* Coarse-pointer touch target (WCAG 2.5.5) — matches the .msg-action-btn rule. */ +@media (hover: none) and (pointer: coarse) { + .composer-mic-btn { + min-width: 36px; + min-height: 36px; + } +} + +/* Monochrome currentColor glyphs for the voice controls (CSS-mask technique), + so they theme and sit beside the line-art .icon-retry set rather than reading + as full-color, OS-dependent emoji. */ +.composer-mic-icon, +.icon-mic, +.icon-speaker { + display: inline-block; + width: 14px; + height: 14px; + background-color: currentColor; + -webkit-mask-repeat: no-repeat; + mask-repeat: no-repeat; + -webkit-mask-position: center; + mask-position: center; + -webkit-mask-size: contain; + mask-size: contain; +} +.icon-mic { + -webkit-mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23000' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M12 2a3 3 0 0 0-3 3v6a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3z'/%3E%3Cpath d='M19 10v1a7 7 0 0 1-14 0v-1'/%3E%3Cline x1='12' y1='19' x2='12' y2='22'/%3E%3C/svg%3E"); + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23000' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M12 2a3 3 0 0 0-3 3v6a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3z'/%3E%3Cpath d='M19 10v1a7 7 0 0 1-14 0v-1'/%3E%3Cline x1='12' y1='19' x2='12' y2='22'/%3E%3C/svg%3E"); +} +.icon-speaker { + -webkit-mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23000' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolygon points='11 5 6 9 2 9 2 15 6 15 11 19 11 5'/%3E%3Cpath d='M15.54 8.46a5 5 0 0 1 0 7.07'/%3E%3Cpath d='M19.07 4.93a10 10 0 0 1 0 14.14'/%3E%3C/svg%3E"); + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23000' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolygon points='11 5 6 9 2 9 2 15 6 15 11 19 11 5'/%3E%3Cpath d='M15.54 8.46a5 5 0 0 1 0 7.07'/%3E%3Cpath d='M19.07 4.93a10 10 0 0 1 0 14.14'/%3E%3C/svg%3E"); +} +/* Recording: a solid stop square (no mask) in place of the mic glyph. */ +.composer-mic-icon.icon-stop { + width: 11px; + height: 11px; + border-radius: 2px; + -webkit-mask-image: none; + mask-image: none; +} + .composer-input { flex: 1; + /* Allow the textarea to shrink below its content width so a long unbroken + token can't push the fixed-width row buttons (attach/mic/send) past the + edge of a narrow split pane. */ + min-width: 0; min-height: 32px; max-height: 160px; font-family: var(--font-mono); diff --git a/turnstone/shared_static/composer.js b/turnstone/shared_static/composer.js index 66658146..c4138d52 100644 --- a/turnstone/shared_static/composer.js +++ b/turnstone/shared_static/composer.js @@ -177,6 +177,10 @@ textRow = actionRow; } + // Supported surface for callers that need to add page-specific controls + // (e.g. a mic button) beside attach/send/stop. + this.actionsRowEl = actionRow; + this._buildAttachButton(actionRow, opts); this._buildInput(textRow, opts); this._buildOptionsToggle(actionRow, opts); diff --git a/turnstone/ui/static/app.js b/turnstone/ui/static/app.js index b076211c..b1d639a2 100644 --- a/turnstone/ui/static/app.js +++ b/turnstone/ui/static/app.js @@ -9,6 +9,43 @@ let _paneCounter = 0; +// Voice-role availability comes from /v1/api/models (stt_default_alias / +// tts_default_alias — present only when an audio-capable model role is +// configured). Memoized so all panes share a single fetch; affordances stay +// hidden until it resolves. +let _voiceRolesPromise = null; +function getVoiceRoles() { + if (!_voiceRolesPromise) { + _voiceRolesPromise = authFetch("/v1/api/models") + .then((r) => (r.ok ? r.json() : {})) + .then((d) => ({ + stt: !!(d && d.stt_default_alias), + tts: !!(d && d.tts_default_alias), + })) + .catch(() => ({ stt: false, tts: false })); + } + return _voiceRolesPromise; +} + +// Visually-hidden polite live region for voice status (recording / playback) +// so screen-reader users perceive state changes otherwise conveyed only by +// color/icon. Errors go through showToast (already a live region). Single +// shared node; clear-then-set so repeated identical messages re-announce. +let _voiceStatusEl = null; +function voiceAnnounce(msg) { + if (!_voiceStatusEl) { + _voiceStatusEl = document.createElement("div"); + _voiceStatusEl.className = "sr-only"; + _voiceStatusEl.setAttribute("role", "status"); + _voiceStatusEl.setAttribute("aria-live", "polite"); + document.body.appendChild(_voiceStatusEl); + } + _voiceStatusEl.textContent = ""; + window.setTimeout(() => { + if (_voiceStatusEl) _voiceStatusEl.textContent = msg; + }, 30); +} + class Pane { constructor(wsId) { this.id = "p" + ++_paneCounter; @@ -35,6 +72,21 @@ class Pane { this._cancelTimeout = null; this._forceTimeout = null; this._pendingEditSend = null; + // Voice I/O (mic STT + per-message TTS playback) + this._voiceRoles = { stt: false, tts: false }; + this._micBtn = null; + this._micIcon = null; + this._micDenied = false; + this._recorder = null; + this._recordingStream = null; + this._isRecording = false; + this._discardRecording = false; + this._recordAborted = false; + this._recordTimer = null; + this._recordStartMs = 0; + this._ttsAudio = null; + this._ttsBtnActive = null; + this._ttsSeq = 0; this._createDOM(); } @@ -48,6 +100,8 @@ class Pane { this._pendingEditSend = null; this.inputEl.disabled = false; this.attachments.clearChips(); + this._stopRecording(true); + this._stopTTS(); } updateWsName() { @@ -73,6 +127,8 @@ class Pane { this.evtSource.close(); this.evtSource = null; } + this._stopRecording(true); + this._stopTTS(); } // composer.setBusy runs unconditionally so the Stop button label / @@ -88,6 +144,16 @@ class Pane { const edge = next !== this.busy; this.busy = next; if (edge && !next) this.queue.onIdleEdge(); + // The mic produces composer input the user can only send when idle — keep + // it in lockstep with the send button (which composer.setBusy gates). + if (this._micBtn && !this._micDenied) { + if (next) { + this._stopRecording(true); // abandon any in-flight recording + this._micBtn.disabled = true; + } else if (!this._micBtn.classList.contains("is-busy")) { + this._micBtn.disabled = false; + } + } } showEmptyState() { @@ -672,6 +738,14 @@ class Pane { } }, }); + + // Voice input: mic button, hidden until the STT role is confirmed + // available (so it never appears when voice isn't configured). + this._buildMicButton(); + getVoiceRoles().then((roles) => { + this._voiceRoles = roles; + if (this._micBtn) this._micBtn.style.display = roles.stt ? "" : "none"; + }); } connectSSE(wsId) { @@ -1350,6 +1424,364 @@ class Pane { bar.insertBefore(btn, bar.firstChild); } + // ------------------------------------------------------------------------- + // Voice I/O: microphone dictation (STT) + per-message playback (TTS) + // ------------------------------------------------------------------------- + + _buildMicButton() { + if (!this.composer || !this.composer.actionsRowEl) return; + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "composer-mic-btn"; + btn.style.display = "none"; // revealed once the STT role is confirmed + btn.title = "Record speech to text"; + btn.setAttribute("aria-label", "Record speech to text"); + btn.setAttribute("aria-pressed", "false"); + const icon = document.createElement("span"); + icon.className = "composer-mic-icon icon-mic"; + icon.setAttribute("aria-hidden", "true"); + btn.appendChild(icon); + this._micIcon = icon; + btn.addEventListener("click", () => this._toggleRecording()); + this.composer.actionsRowEl.insertBefore(btn, this.sendBtn || null); + this._micBtn = btn; + } + + _syncMicButton() { + if (!this._micBtn) return; + const rec = !!this._isRecording; + this._micBtn.classList.toggle("is-recording", rec); + this._micBtn.setAttribute("aria-pressed", rec ? "true" : "false"); + if (this._micIcon) { + this._micIcon.classList.toggle("icon-stop", rec); + this._micIcon.classList.toggle("icon-mic", !rec); + } + const label = rec + ? "Stop recording and transcribe" + : "Record speech to text"; + this._micBtn.title = label; + this._micBtn.setAttribute("aria-label", label); + } + + _toggleRecording() { + if (this._isRecording) { + this._stopRecording(false); + } else { + this._startRecording(); + } + } + + _startRecording() { + if (this._isRecording || this.busy) return; + if ( + !navigator.mediaDevices || + !navigator.mediaDevices.getUserMedia || + typeof MediaRecorder === "undefined" + ) { + showToast("Microphone capture is not supported in this browser", "error"); + return; + } + // Synchronous abort latch: if teardown (reset / pane switch) runs while the + // permission prompt is open, the stream the promise later hands us must be + // stopped instead of going hot after teardown. + this._recordAborted = false; + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then((stream) => { + if (this._recordAborted) { + stream.getTracks().forEach((t) => t.stop()); + return; + } + this._recordingStream = stream; + let mimeType = ""; + const candidates = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/ogg;codecs=opus", + "audio/mp4", + ]; + for (let i = 0; i < candidates.length; i++) { + if ( + MediaRecorder.isTypeSupported && + MediaRecorder.isTypeSupported(candidates[i]) + ) { + mimeType = candidates[i]; + break; + } + } + const rec = mimeType + ? new MediaRecorder(stream, { mimeType }) + : new MediaRecorder(stream); + const chunks = []; + rec.addEventListener("dataavailable", (e) => { + if (e.data && e.data.size) chunks.push(e.data); + }); + rec.addEventListener("stop", () => { + this._teardownRecordingStream(); + this._isRecording = false; + this._stopRecordTimer(); + this._syncMicButton(); + const discard = this._discardRecording; + this._discardRecording = false; + if (discard) return; + const blob = new Blob(chunks, { + type: rec.mimeType || mimeType || "audio/webm", + }); + if (blob.size) this._uploadForTranscription(blob); + }); + this._recorder = rec; + this._isRecording = true; + this._discardRecording = false; + this._startRecordTimer(); + this._syncMicButton(); + voiceAnnounce( + "Recording. Activate the microphone button again to stop.", + ); + rec.start(); + }) + .catch(() => { + // Denied / hardware unavailable: leave a persistent disabled state with + // guidance — a hot button reads as dead once the browser blocks re-prompts. + this._teardownRecordingStream(); + this._isRecording = false; + this._stopRecordTimer(); + this._setMicDenied(); + }); + } + + _setMicDenied() { + this._micDenied = true; + showToast("Microphone access was denied", "error"); + this._syncMicButton(); + if (this._micBtn) { + this._micBtn.disabled = true; + const msg = + "Microphone blocked — enable it in your browser's site settings"; + this._micBtn.title = msg; + this._micBtn.setAttribute("aria-label", msg); + } + } + + _startRecordTimer() { + this._recordStartMs = Date.now(); + this._stopRecordTimer(); + this._recordTimer = window.setInterval(() => this._tickRecordTimer(), 500); + } + + _stopRecordTimer() { + if (this._recordTimer) { + window.clearInterval(this._recordTimer); + this._recordTimer = null; + } + } + + _tickRecordTimer() { + if (!this._isRecording || !this._micBtn) return; + const secs = Math.max( + 0, + Math.floor((Date.now() - this._recordStartMs) / 1000), + ); + const mmss = + Math.floor(secs / 60) + ":" + String(secs % 60).padStart(2, "0"); + this._micBtn.title = "Recording " + mmss + " — activate to stop"; + } + + _stopRecording(discard) { + this._discardRecording = !!discard; + this._recordAborted = true; // abort a getUserMedia still in flight + if (this._recorder && this._recorder.state !== "inactive") { + try { + this._recorder.stop(); + } catch (e) { + /* already stopped */ + } + return; + } + this._teardownRecordingStream(); + this._stopRecordTimer(); + if (this._isRecording) { + this._isRecording = false; + this._syncMicButton(); + } + } + + _teardownRecordingStream() { + if (this._recordingStream) { + this._recordingStream.getTracks().forEach((t) => t.stop()); + this._recordingStream = null; + } + this._recorder = null; + } + + _uploadForTranscription(blob) { + if (!this.wsId) return; + const ext = + blob.type.indexOf("ogg") !== -1 + ? "ogg" + : blob.type.indexOf("mp4") !== -1 + ? "mp4" + : "webm"; + const fd = new FormData(); + fd.append("audio", blob, "speech." + ext); + if (this._micBtn) { + this._micBtn.disabled = true; + this._micBtn.classList.add("is-busy"); + } + voiceAnnounce("Transcribing…"); + authFetch( + "/v1/api/workstreams/" + + encodeURIComponent(this.wsId) + + "/speech-to-text", + { method: "POST", body: fd }, + ) + .then((r) => r.json().then((body) => ({ ok: r.ok, body }))) + .then((res) => { + if (!res.ok) { + showToast( + (res.body && res.body.error) || "Transcription failed", + "error", + ); + return; + } + const text = (res.body && res.body.transcript) || ""; + if (text && this.inputEl) { + const cur = this.inputEl.value || ""; + this.inputEl.value = cur + ? cur.replace(/\s*$/, "") + " " + text + : text; + // Drive the composer's auto-resize + send-enable listeners. + this.inputEl.dispatchEvent(new Event("input", { bubbles: true })); + this.inputEl.focus(); + voiceAnnounce("Transcript added to message."); + } + }) + .catch((err) => { + showToast( + "Transcription failed: " + (err && err.message ? err.message : err), + "error", + ); + }) + .finally(() => { + if (this._micBtn && !this._micDenied) { + this._micBtn.disabled = !!this.busy; + this._micBtn.classList.remove("is-busy"); + } + }); + } + + _addTtsAction(el) { + let bar = el.querySelector(".msg-actions"); + if (!bar) { + bar = document.createElement("div"); + bar.className = "msg-actions"; + bar.setAttribute("role", "toolbar"); + bar.setAttribute("aria-label", "Message actions"); + el.appendChild(bar); + } + if (bar.querySelector(".msg-tts-btn")) return; // already added + const btn = document.createElement("button"); + btn.className = "msg-action-btn msg-tts-btn"; + btn.title = "Play response aloud"; + btn.setAttribute("aria-label", "Play response aloud"); + btn.setAttribute("aria-pressed", "false"); + const icon = document.createElement("span"); + icon.className = "icon-speaker"; + icon.setAttribute("aria-hidden", "true"); + btn.appendChild(icon); + btn.addEventListener("click", (e) => { + e.stopPropagation(); + this._playMessageTTS(el, btn); + }); + bar.appendChild(btn); + } + + // Strip code blocks / inline code / rendered math so TTS doesn't read source + // or KaTeX accessibility text out character-by-character. + _extractSpeakableText(bodyEl) { + const clone = bodyEl.cloneNode(true); + clone + .querySelectorAll("pre, code, .katex, .katex-display") + .forEach((n) => + n.replaceWith(document.createTextNode(" (code omitted) ")), + ); + return (clone.textContent || "").replace(/\s+/g, " ").trim(); + } + + _playMessageTTS(el, btn) { + // Toggle: clicking the active button (or any while playing) stops first. + if (this._ttsAudio) { + const wasThis = this._ttsBtnActive === btn; + this._stopTTS(); + if (wasThis) return; + } + const bodyEl = el.querySelector(".msg-body") || el; + const text = this._extractSpeakableText(bodyEl); + if (!text) return; + // Serialize: a monotonic token guards against an earlier (slower) request + // resolving after a newer one — which would double-play and leak the blob. + const token = ++this._ttsSeq; + btn.classList.add("is-busy"); + btn.disabled = true; + authFetch("/v1/api/tts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), + }) + .then((r) => { + if (!r.ok) { + return r.json().then((b) => { + throw new Error((b && b.error) || "Speech synthesis failed"); + }); + } + return r.blob(); + }) + .then((audioBlob) => { + const url = URL.createObjectURL(audioBlob); + if (token !== this._ttsSeq) { + URL.revokeObjectURL(url); // superseded — don't play or leak + return; + } + const audio = new Audio(url); + this._ttsAudio = audio; + this._ttsBtnActive = btn; + btn.classList.add("is-playing"); + btn.setAttribute("aria-pressed", "true"); + voiceAnnounce("Playing response."); + audio.addEventListener("ended", () => this._stopTTS()); + audio.addEventListener("error", () => this._stopTTS()); + audio.play().catch(() => this._stopTTS()); + }) + .catch((err) => { + showToast( + err && err.message ? err.message : "Speech synthesis failed", + "error", + ); + }) + .finally(() => { + btn.classList.remove("is-busy"); + btn.disabled = false; + }); + } + + _stopTTS() { + this._ttsSeq++; // invalidate any in-flight request + if (this._ttsAudio) { + try { + this._ttsAudio.pause(); + } catch (e) { + /* ignore */ + } + const src = this._ttsAudio.src || ""; + if (src.indexOf("blob:") === 0) URL.revokeObjectURL(src); + this._ttsAudio = null; + } + if (this._ttsBtnActive) { + this._ttsBtnActive.classList.remove("is-playing"); + this._ttsBtnActive.setAttribute("aria-pressed", "false"); + this._ttsBtnActive = null; + } + } + _retryLast() { if (this.busy) return; // Path-keyed retry (#549). Truncation + re-dispatch happen @@ -1836,7 +2268,11 @@ class Pane { } const assistants = this.messagesEl.querySelectorAll(".msg.assistant"); if (assistants.length) { - this._addRetryAction(assistants[assistants.length - 1]); + const lastAssistant = assistants[assistants.length - 1]; + this._addRetryAction(lastAssistant); + if (this._voiceRoles && this._voiceRoles.tts) { + this._addTtsAction(lastAssistant); + } } }