From 3f4564f56fb7c8e6e83c14f1af9bbc36660c4f7d Mon Sep 17 00:00:00 2001 From: "Vyctor H. Brzezowski" Date: Mon, 10 Aug 2026 01:43:28 -0300 Subject: [PATCH] fix(ui): microphone picker claims a selection while denying any input exists (#121397) * fix(ui): give the microphone picker one truthful state Discovery now reports why it stopped as a code instead of prose, so the composer popover can pick a single coherent rendering: with no selectable input it shows one empty state rather than a checked System default row stacked on two different ways of saying nothing was found. Rows drop wa-dropdown-item type="checkbox", whose own leading check toggles on click and disagrees with the trailing check bound to the stored device; the chat pane gateway picker had the same mix and is fixed with it. Co-Authored-By: Claude Opus 5 * feat(ui): keep the microphone picker current while it is open The empty state now promises the list keeps up, so both media surfaces subscribe to navigator.mediaDevices devicechange for as long as they are on screen and drop the listener when they close. The composer popover re-runs permission-requesting discovery, since a microphone that just appeared has hidden labels until the probe runs; the settings rows refresh passively so hardware changes never turn into an unasked-for browser dialog. Also drops the newly unused RealtimeTalkDeviceKind export that failed the deadcode gate. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- ui/src/i18n/locales/en.ts | 2 +- ui/src/pages/chat/chat-composer.test.ts | 151 ++++++++++++++++-- .../chat/components/chat-composer-controls.ts | 101 ++++++++---- .../chat/components/chat-composer-state.ts | 16 +- .../chat/components/chat-composer-types.ts | 10 +- ui/src/pages/chat/components/chat-composer.ts | 34 ++-- .../pages/chat/components/chat-pane-header.ts | 1 - ui/src/pages/chat/realtime-talk-input.test.ts | 58 ++++++- ui/src/pages/chat/realtime-talk-input.ts | 119 +++++++------- ui/src/pages/config/config-page.ts | 20 ++- ui/src/styles/chat/layout.css | 14 ++ 11 files changed, 399 insertions(+), 127 deletions(-) diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index 99639661b15d..b0f54377e5c3 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -5193,7 +5193,7 @@ export const en: TranslationMap = { microphoneListUnsupported: "This browser cannot list microphone inputs.", noCameras: "No additional cameras found", noMicrophones: "No additional microphones found", - microphoneNoneFound: "No microphone inputs were found.", + microphoneNoneFound: "No microphone found. Plug one in and it appears here.", microphonePageInactive: "Microphone inputs are unavailable while this page is inactive.", microphonePermissionBlocked: "Microphone access is blocked. Allow it in browser site settings to list inputs.", diff --git a/ui/src/pages/chat/chat-composer.test.ts b/ui/src/pages/chat/chat-composer.test.ts index 47d4d0311713..2e4ed39d318f 100644 --- a/ui/src/pages/chat/chat-composer.test.ts +++ b/ui/src/pages/chat/chat-composer.test.ts @@ -290,7 +290,7 @@ describe("renderChatComposer controls", () => { { deviceId: "studio-mic", label: "Studio microphone" }, { deviceId: "headset", label: "USB headset" }, ], - warning: null, + issue: null, }); patchSettings({ realtimeTalkInputDeviceId: "studio-mic" }); const container = document.createElement("div"); @@ -344,10 +344,55 @@ describe("renderChatComposer controls", () => { expect(dropdown?.open).toBe(true); }); - it("shows discovery warnings and the next-session hint during active Talk", async () => { + it.each([ + ["none-found", "chat.composer.microphoneNoneFound", false], + ["list-unsupported", "chat.composer.microphoneListUnsupported", false], + ["permission-blocked", "chat.composer.microphonePermissionBlocked", true], + ["busy", "chat.composer.microphoneBusy", true], + ["page-inactive", "chat.composer.microphonePageInactive", true], + ["failed", "chat.composer.microphoneAccessFailed", true], + ] as const)( + "renders %s as one empty state with no claimed selection", + async (issue, messageKey, fault) => { + discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue }); + const container = document.createElement("div"); + document.body.append(container); + const composerProps = props({ + onToggleRealtimeTalk: vi.fn(), + realtimeTalkActive: true, + realtimeTalkStatus: "listening", + }); + const draw = () => render(renderChatComposer(composerProps), container); + composerProps.onRequestUpdate = draw; + draw(); + + const dropdown = container.querySelector< + HTMLElement & { open: boolean; updateComplete: Promise } + >("wa-dropdown.chat-talk-input-picker"); + await dropdown?.updateComplete; + button(container, t("chat.composer.microphoneInput")).click(); + const empty = await vi.waitFor(() => { + const node = container.querySelector(".chat-talk-input-picker__empty"); + expect(node?.textContent?.trim()).toBe(t(messageKey)); + return node; + }); + + // One designed state: never a checked System default row, a second + // negative note, or a hint about a selection that cannot be made. + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(0); + expect(container.querySelector(".chat-talk-input-picker__note")).toBeNull(); + expect(container.querySelector(".chat-talk-input-picker__warning")).toBeNull(); + expect(container.querySelector(".chat-talk-input-picker__hint")).toBeNull(); + expect(container.querySelectorAll(".chat-talk-input-picker__empty")).toHaveLength(1); + expect(empty?.getAttribute("role")).toBe("status"); + expect(empty?.classList.contains("chat-talk-input-picker__empty--fault")).toBe(fault); + }, + ); + + it("keeps the list plus one warning when inputs exist but discovery reported an issue", async () => { discoverRealtimeTalkInputsMock.mockResolvedValue({ - devices: [], - warning: "Microphone permission is blocked.", + devices: [{ deviceId: "headset", label: "USB headset" }], + issue: "busy", }); const container = document.createElement("div"); document.body.append(container); @@ -366,17 +411,13 @@ describe("renderChatComposer controls", () => { await dropdown?.updateComplete; button(container, t("chat.composer.microphoneInput")).click(); await vi.waitFor(() => - expect(container.querySelector(".chat-talk-input-picker__warning")?.textContent).toContain( - "Microphone permission is blocked.", - ), + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(2), ); - expect(container.querySelector(".chat-talk-input-picker__warning")?.getAttribute("role")).toBe( - "alert", - ); - expect(container.querySelector(".chat-talk-input-picker__note")?.textContent).toContain( - t("chat.composer.noMicrophones"), + expect(container.querySelector(".chat-talk-input-picker__warning")?.textContent?.trim()).toBe( + t("chat.composer.microphoneBusy"), ); + expect(container.querySelector(".chat-talk-input-picker__empty")).toBeNull(); expect(container.querySelector(".chat-talk-input-picker__hint")?.textContent).toContain( t("chat.composer.microphoneAppliesNextSession"), ); @@ -388,6 +429,92 @@ describe("renderChatComposer controls", () => { expect(dropdown?.open).toBe(false); }); + it("marks the selected input with a single trailing check", async () => { + discoverRealtimeTalkInputsMock.mockResolvedValue({ + devices: [{ deviceId: "headset", label: "USB headset" }], + issue: null, + }); + const container = document.createElement("div"); + document.body.append(container); + const composerProps = props({ onToggleRealtimeTalk: vi.fn() }); + const draw = () => render(renderChatComposer(composerProps), container); + composerProps.onRequestUpdate = draw; + draw(); + + const dropdown = container.querySelector< + HTMLElement & { open: boolean; updateComplete: Promise } + >("wa-dropdown.chat-talk-input-picker"); + await dropdown?.updateComplete; + button(container, t("chat.composer.microphoneInput")).click(); + const items = await vi.waitFor(() => { + const rows = [...container.querySelectorAll(".chat-talk-input-picker__item")]; + expect(rows).toHaveLength(2); + return rows; + }); + + // type="checkbox" would make wa-dropdown-item paint its own leading check + // and toggle it on click, so the row would show two disagreeing marks. + expect(items.map((item) => item.getAttribute("type"))).toEqual(["normal", "normal"]); + expect(items.map((item) => item.querySelectorAll("svg").length)).toEqual([1, 0]); + expect(items[0]?.querySelector(".chat-talk-input-picker__check")?.getAttribute("slot")).toBe( + "details", + ); + expect(items.map((item) => item.getAttribute("aria-checked"))).toEqual(["true", "false"]); + }); + + it("follows devicechange while open and stops listening once closed", async () => { + const mediaDevices = new EventTarget(); + Object.defineProperty(globalThis.navigator, "mediaDevices", { + configurable: true, + value: mediaDevices, + }); + discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue: "none-found" }); + const container = document.createElement("div"); + document.body.append(container); + const composerProps = props({ onToggleRealtimeTalk: vi.fn() }); + const draw = () => render(renderChatComposer(composerProps), container); + composerProps.onRequestUpdate = draw; + draw(); + + const dropdown = container.querySelector< + HTMLElement & { open: boolean; updateComplete: Promise } + >("wa-dropdown.chat-talk-input-picker"); + await dropdown?.updateComplete; + button(container, t("chat.composer.microphoneInput")).click(); + await vi.waitFor(() => + expect(container.querySelector(".chat-talk-input-picker__empty")?.textContent?.trim()).toBe( + t("chat.composer.microphoneNoneFound"), + ), + ); + + // The empty state promises the list keeps up, so plugging in has to land + // without reopening the popover. + discoverRealtimeTalkInputsMock.mockResolvedValue({ + devices: [{ deviceId: "usb", label: "USB Audio Interface" }], + issue: null, + }); + mediaDevices.dispatchEvent(new Event("devicechange")); + await vi.waitFor(() => + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(2), + ); + expect(container.querySelector(".chat-talk-input-picker__empty")).toBeNull(); + + discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue: "none-found" }); + mediaDevices.dispatchEvent(new Event("devicechange")); + await vi.waitFor(() => + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(0), + ); + + dropdown?.dispatchEvent( + new KeyboardEvent("keydown", { key: "Escape", bubbles: true, cancelable: true }), + ); + await dropdown?.updateComplete; + const callsWhileClosed = discoverRealtimeTalkInputsMock.mock.calls.length; + mediaDevices.dispatchEvent(new Event("devicechange")); + await Promise.resolve(); + expect(discoverRealtimeTalkInputsMock.mock.calls.length).toBe(callsWhileClosed); + }); + it("offers camera only inside a video-capable active talk session", () => { const onToggleRealtimeCamera = vi.fn(); const { container } = renderComposer({ diff --git a/ui/src/pages/chat/components/chat-composer-controls.ts b/ui/src/pages/chat/components/chat-composer-controls.ts index 1b7ba4027ec4..8f8d00bd722d 100644 --- a/ui/src/pages/chat/components/chat-composer-controls.ts +++ b/ui/src/pages/chat/components/chat-composer-controls.ts @@ -5,7 +5,11 @@ import { syncDropdownItemRadio } from "../../../components/web-awesome.ts"; import { t } from "../../../i18n/index.ts"; import type { ControlUiFollowUpMode } from "../../../lib/chat/follow-up-mode.ts"; import type { ComposerDictationController } from "../composer-dictation.ts"; -import type { RealtimeTalkInputDevice } from "../realtime-talk-input.ts"; +import { + realtimeTalkDeviceIssueMessage, + type RealtimeTalkDeviceIssue, + type RealtimeTalkInputDevice, +} from "../realtime-talk-input.ts"; import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts"; import type { RealtimeTalkStatus } from "../realtime-talk.ts"; import { renderMicrophoneActivity, voiceStatusLabel } from "./chat-voice-activity.ts"; @@ -49,18 +53,30 @@ type MicrophonePickerProps = { open: boolean; selectedDeviceId: string; voiceActive: boolean; - warning: string | null; + issue: RealtimeTalkDeviceIssue | null; onOpen: () => void; onClose: () => void; onSelect: (deviceId: string) => void; }; export function renderMicrophonePicker(props: MicrophonePickerProps) { + // Discovery reporting an issue with nothing enumerated is the browser stating + // there is no capture route at all: a "System default" row would claim a + // selection that cannot exist, so the popover shows one empty state instead + // of a checked row stacked on two ways of saying the same thing. + const unavailable = !props.loading && props.devices.length === 0 ? props.issue : null; // System default renders even while discovery runs: the dropdown's one-time // focus step needs at least one item or keyboard users never enter the menu. - const options = props.loading - ? [{ deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }] - : [{ deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }, ...props.devices]; + const options = unavailable + ? [] + : [ + { deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }, + ...(props.loading ? [] : props.devices), + ]; + // A machine without a microphone and a browser that cannot enumerate are + // facts, not faults; only the recoverable reasons earn the warn tone. + const unavailableIsFault = + unavailable !== null && unavailable !== "none-found" && unavailable !== "list-unsupported"; const label = t("chat.composer.microphoneInput"); return html`
${label}
- ${options.map((option) => { - const selected = option.deviceId === props.selectedDeviceId; - return html` - syncDropdownItemRadio(element, selected))} + ${unavailable + ? html`
- ${option.label} - - - `; - })} - ${props.loading - ? html`
${t("common.loading")}
` - : nothing} - ${!props.loading && props.devices.length === 0 - ? html`
${t("chat.composer.noMicrophones")}
` - : nothing} - ${props.warning - ? html`` - : nothing} - ${props.voiceActive - ? html`
- ${t("chat.composer.microphoneAppliesNextSession")} + ${realtimeTalkDeviceIssueMessage(unavailable, "audioinput")}
` - : nothing} + : html` + ${options.map((option) => { + const selected = option.deviceId === props.selectedDeviceId; + // Selection is radio-shaped, so the row stays a plain menu item: + // wa-dropdown-item type="checkbox" paints its own leading check + // and flips it on click, which would contradict this trailing + // check whenever the click does not change the stored device. + return html` + syncDropdownItemRadio(element, selected))} + > + ${option.label} + + + `; + })} + ${props.loading + ? html`
+ ${t("common.loading")} +
` + : nothing} + ${props.issue + ? html`` + : nothing} + ${props.voiceActive + ? html`
+ ${t("chat.composer.microphoneAppliesNextSession")} +
` + : nothing} + `} `; } diff --git a/ui/src/pages/chat/components/chat-composer-state.ts b/ui/src/pages/chat/components/chat-composer-state.ts index 2aefe964de97..71b19c4587b6 100644 --- a/ui/src/pages/chat/components/chat-composer-state.ts +++ b/ui/src/pages/chat/components/chat-composer-state.ts @@ -34,7 +34,8 @@ function createChatComposerState(): ChatComposerState { microphonePickerOpen: false, microphonePickerLoading: false, microphoneDevices: [], - microphoneWarning: null, + microphoneIssue: null, + microphoneDeviceWatch: null, microphoneDiscoveryRequest: 0, capabilityMenuOpen: false, capabilityMenuView: "root", @@ -144,16 +145,27 @@ export function suppressStaleSubmittedDraftReplay( return true; } +/** Drops the devicechange subscription so a closed picker stops refreshing. */ +export function releaseMicrophoneDeviceWatch(state: ChatComposerState) { + state.microphoneDeviceWatch?.(); + state.microphoneDeviceWatch = null; +} + export function resetChatComposerState(paneId?: string) { if (paneId) { // Goal elapsed timers are keyed by element and cleaned up when their // element leaves the DOM, so a per-pane reset does not need to touch them. - composerStates.get(paneId)?.dictation?.dispose(); + const paneState = composerStates.get(paneId); + paneState?.dictation?.dispose(); + if (paneState) { + releaseMicrophoneDeviceWatch(paneState); + } composerStates.delete(paneId); return; } for (const state of composerStates.values()) { state.dictation?.dispose(); + releaseMicrophoneDeviceWatch(state); } composerStates.clear(); clearGoalElapsedTimers(); diff --git a/ui/src/pages/chat/components/chat-composer-types.ts b/ui/src/pages/chat/components/chat-composer-types.ts index c20d2eeedc27..489fe25b8d2a 100644 --- a/ui/src/pages/chat/components/chat-composer-types.ts +++ b/ui/src/pages/chat/components/chat-composer-types.ts @@ -11,7 +11,11 @@ import type { SessionToolOverrides } from "../../../lib/sessions/patch.ts"; import type { ComposerDictationController } from "../composer-dictation.ts"; import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "../input-history.ts"; import type { RealtimeTalkConversationEntry } from "../realtime-talk-conversation.ts"; -import type { RealtimeTalkCameraDevice, RealtimeTalkInputDevice } from "../realtime-talk-input.ts"; +import type { + RealtimeTalkCameraDevice, + RealtimeTalkDeviceIssue, + RealtimeTalkInputDevice, +} from "../realtime-talk-input.ts"; import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts"; import type { RealtimeTalkStatus } from "../realtime-talk.ts"; import type { ChatRunUiStatus } from "../run-lifecycle.ts"; @@ -172,7 +176,9 @@ export type ChatComposerState = { microphonePickerOpen: boolean; microphonePickerLoading: boolean; microphoneDevices: RealtimeTalkInputDevice[]; - microphoneWarning: string | null; + microphoneIssue: RealtimeTalkDeviceIssue | null; + /** Unsubscribe for the devicechange watch; non-null only while the picker is open. */ + microphoneDeviceWatch: (() => void) | null; microphoneDiscoveryRequest: number; capabilityMenuOpen: boolean; capabilityMenuView: ChatComposerPlusMenuView; diff --git a/ui/src/pages/chat/components/chat-composer.ts b/ui/src/pages/chat/components/chat-composer.ts index 7e3092908771..7205ff7ec9c0 100644 --- a/ui/src/pages/chat/components/chat-composer.ts +++ b/ui/src/pages/chat/components/chat-composer.ts @@ -5,7 +5,7 @@ import "../../../components/tooltip.ts"; import { t } from "../../../i18n/index.ts"; import { areUiSessionKeysEquivalent } from "../../../lib/sessions/session-key.ts"; import { ComposerDictationController, insertComposerDictation } from "../composer-dictation.ts"; -import { discoverRealtimeTalkInputs } from "../realtime-talk-input.ts"; +import { discoverRealtimeTalkInputs, observeRealtimeTalkDevices } from "../realtime-talk-input.ts"; import { isLargePastedTextAttachment } from "./chat-attachments.ts"; import { renderContextNotice } from "./chat-composer-context.ts"; import { renderMicrophonePicker, type ChatRunControlsProps } from "./chat-composer-controls.ts"; @@ -48,6 +48,7 @@ import { hasTerminalRunStatus, isCurrentSessionSubmittedProgress, markComposerInputIntent, + releaseMicrophoneDeviceWatch, suppressStaleSubmittedDraftReplay, } from "./chat-composer-state.ts"; import type { ChatComposerProps, ChatComposerState } from "./chat-composer-types.ts"; @@ -505,30 +506,29 @@ export function renderChatComposer(props: ChatComposerProps) { } props.onToggleRealtimeTalk?.(); }; - const openMicrophonePicker = () => { - if (state.microphonePickerOpen) { - return; - } - state.microphonePickerOpen = true; + const discoverMicrophones = () => { state.microphonePickerLoading = true; - state.microphoneWarning = null; + state.microphoneIssue = null; const request = ++state.microphoneDiscoveryRequest; requestUpdate(); + // Permission-requesting discovery on every pass, including device changes: + // a microphone that just appeared has hidden labels until the probe runs, + // and the probe only prompts while the picker is the surface in front of + // the user. void discoverRealtimeTalkInputs(true) .then((result) => { if (request !== state.microphoneDiscoveryRequest) { return; } state.microphoneDevices = result.devices; - state.microphoneWarning = result.warning; + state.microphoneIssue = result.issue; }) - .catch((error: unknown) => { + .catch(() => { if (request !== state.microphoneDiscoveryRequest) { return; } state.microphoneDevices = []; - state.microphoneWarning = - error instanceof Error ? error.message : t("chat.composer.microphoneAccessFailed"); + state.microphoneIssue = "failed"; }) .finally(() => { if (request !== state.microphoneDiscoveryRequest) { @@ -538,15 +538,25 @@ export function renderChatComposer(props: ChatComposerProps) { requestUpdate(); }); }; + const openMicrophonePicker = () => { + if (state.microphonePickerOpen) { + return; + } + state.microphonePickerOpen = true; + state.microphoneDeviceWatch ??= observeRealtimeTalkDevices(discoverMicrophones); + discoverMicrophones(); + }; const closeMicrophonePicker = () => { if (!state.microphonePickerOpen) { return; } + releaseMicrophoneDeviceWatch(state); state.microphonePickerOpen = false; requestUpdate(); }; const selectMicrophone = (deviceId: string) => { patchSettings({ realtimeTalkInputDeviceId: deviceId.trim() || undefined }); + releaseMicrophoneDeviceWatch(state); state.microphonePickerOpen = false; requestUpdate(); }; @@ -558,7 +568,7 @@ export function renderChatComposer(props: ChatComposerProps) { open: state.microphonePickerOpen, selectedDeviceId: selectedMicrophoneId, voiceActive: Boolean(props.realtimeTalkActive), - warning: state.microphoneWarning, + issue: state.microphoneIssue, onOpen: openMicrophonePicker, onClose: closeMicrophonePicker, onSelect: selectMicrophone, diff --git a/ui/src/pages/chat/components/chat-pane-header.ts b/ui/src/pages/chat/components/chat-pane-header.ts index c40c91972cfd..c3eb917797bb 100644 --- a/ui/src/pages/chat/components/chat-pane-header.ts +++ b/ui/src/pages/chat/components/chat-pane-header.ts @@ -177,7 +177,6 @@ function renderGatewayPicker(props: ChatPaneHeaderProps) { const selected = gateway.id === snapshot.currentId; return html` syncDropdownItemRadio(element, selected))} diff --git a/ui/src/pages/chat/realtime-talk-input.test.ts b/ui/src/pages/chat/realtime-talk-input.test.ts index d0883d9a7d52..4c8877457bb9 100644 --- a/ui/src/pages/chat/realtime-talk-input.test.ts +++ b/ui/src/pages/chat/realtime-talk-input.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { discoverRealtimeTalkCameras, discoverRealtimeTalkInputs, + observeRealtimeTalkDevices, openRealtimeTalkCamera, openRealtimeTalkInput, } from "./realtime-talk-input.ts"; @@ -37,7 +38,7 @@ describe("realtime Talk microphone inputs", () => { { deviceId: "usb", label: "Microphone 2" }, ], permissionRequired: true, - warning: null, + issue: null, }); expect(getUserMedia).not.toHaveBeenCalled(); }); @@ -63,7 +64,7 @@ describe("realtime Talk microphone inputs", () => { { deviceId: "loopback", label: "Loopback Audio" }, ], permissionRequired: false, - warning: null, + issue: null, }); expect(getUserMedia).toHaveBeenCalledWith({ audio: true }); expect(stopFirst).toHaveBeenCalledOnce(); @@ -71,7 +72,7 @@ describe("realtime Talk microphone inputs", () => { expect(enumerateDevices).toHaveBeenCalledTimes(2); }); - it("keeps System default usable when microphone permission is denied", async () => { + it("reports the blocked reason when microphone permission is denied", async () => { vi.stubGlobal("navigator", { mediaDevices: { enumerateDevices: vi.fn(async () => [mediaDevice("audioinput", "", "")]), @@ -85,7 +86,52 @@ describe("realtime Talk microphone inputs", () => { expect(result.devices).toEqual([]); expect(result.permissionRequired).toBe(true); - expect(result.warning).toContain("Microphone access is blocked"); + expect(result.issue).toBe("permission-blocked"); + }); + + it("separates an empty machine from a blocked browser", async () => { + vi.stubGlobal("navigator", { + mediaDevices: { + enumerateDevices: vi.fn(async () => []), + getUserMedia: vi.fn(async () => { + throw new DOMException("none", "NotFoundError"); + }), + }, + }); + + await expect(discoverRealtimeTalkInputs(true)).resolves.toEqual({ + devices: [], + permissionRequired: true, + issue: "none-found", + }); + }); + + it("subscribes to devicechange and releases the listener on unsubscribe", () => { + const mediaDevices = new EventTarget(); + let changes = 0; + vi.stubGlobal("navigator", { mediaDevices }); + + const unsubscribe = observeRealtimeTalkDevices(() => (changes += 1)); + mediaDevices.dispatchEvent(new Event("devicechange")); + unsubscribe(); + mediaDevices.dispatchEvent(new Event("devicechange")); + + expect(changes).toBe(1); + }); + + it("stays inert where the browser exposes no media devices to watch", () => { + vi.stubGlobal("navigator", {}); + expect(() => observeRealtimeTalkDevices(() => undefined)()).not.toThrow(); + }); + + it("reports an unsupported enumeration instead of a generic access failure", async () => { + vi.stubGlobal("navigator", { mediaDevices: {} }); + + await expect(discoverRealtimeTalkInputs(true)).resolves.toEqual({ + devices: [], + permissionRequired: false, + issue: "list-unsupported", + }); }); it("does not silently fall back when the selected microphone is unavailable", async () => { @@ -288,7 +334,7 @@ describe("realtime Talk camera inputs", () => { { deviceId: "back", label: "Camera 2" }, ], permissionRequired: true, - warning: null, + issue: null, }); expect(getUserMedia).not.toHaveBeenCalled(); }); @@ -305,7 +351,7 @@ describe("realtime Talk camera inputs", () => { await expect(discoverRealtimeTalkCameras(true)).resolves.toEqual({ devices: [{ deviceId: "camera", label: "Desk Camera" }], permissionRequired: false, - warning: null, + issue: null, }); expect(getUserMedia).toHaveBeenCalledWith({ video: true }); expect(stop).toHaveBeenCalledOnce(); diff --git a/ui/src/pages/chat/realtime-talk-input.ts b/ui/src/pages/chat/realtime-talk-input.ts index e0a69c751ce1..574bfbae6c2b 100644 --- a/ui/src/pages/chat/realtime-talk-input.ts +++ b/ui/src/pages/chat/realtime-talk-input.ts @@ -7,28 +7,36 @@ export type RealtimeTalkInputDevice = { export type RealtimeTalkCameraDevice = RealtimeTalkInputDevice; +/** + * Why discovery stopped is a fact only this module observes. Callers need the + * reason itself — not prose — to pick a coherent rendering, so the code travels + * and each surface owns its own wording and tone. + */ +const deviceIssueMessageKeys = { + "list-unsupported": [ + "chat.composer.microphoneListUnsupported", + "chat.composer.cameraListUnsupported", + ], + "none-found": ["chat.composer.microphoneNoneFound", "chat.composer.cameraNoneFound"], + "permission-blocked": [ + "chat.composer.microphonePermissionBlocked", + "chat.composer.cameraPermissionBlocked", + ], + busy: ["chat.composer.microphoneBusy", "chat.composer.cameraBusy"], + "page-inactive": ["chat.composer.microphonePageInactive", "chat.composer.cameraPageInactive"], + failed: ["chat.composer.microphoneAccessFailed", "chat.composer.cameraAccessFailed"], +} as const; + +export type RealtimeTalkDeviceIssue = keyof typeof deviceIssueMessageKeys; + type RealtimeTalkDeviceDiscovery = { devices: RealtimeTalkInputDevice[]; permissionRequired: boolean; - warning: string | null; + issue: RealtimeTalkDeviceIssue | null; }; type RealtimeTalkDeviceKind = "audioinput" | "videoinput"; -function mediaDevices(kind: RealtimeTalkDeviceKind): MediaDevices { - const devices = globalThis.navigator?.mediaDevices; - if (!devices?.enumerateDevices) { - throw new Error( - t( - kind === "audioinput" - ? "chat.composer.microphoneListUnsupported" - : "chat.composer.cameraListUnsupported", - ), - ); - } - return devices; -} - function normalizeDevices( devices: MediaDeviceInfo[], kind: RealtimeTalkDeviceKind, @@ -63,60 +71,63 @@ function deviceDetailsHidden(devices: MediaDeviceInfo[], kind: RealtimeTalkDevic return inputs.length === 0 || inputs.some((device) => !device.deviceId || !device.label); } -function describeDeviceError(error: unknown, kind: RealtimeTalkDeviceKind): string { - const name = error instanceof DOMException ? error.name : ""; - if (name === "NotAllowedError") { - return t( - kind === "audioinput" - ? "chat.composer.microphonePermissionBlocked" - : "chat.composer.cameraPermissionBlocked", - ); - } - if (name === "NotFoundError") { - return t( - kind === "audioinput" ? "chat.composer.microphoneNoneFound" : "chat.composer.cameraNoneFound", - ); - } - if (name === "NotReadableError") { - return t(kind === "audioinput" ? "chat.composer.microphoneBusy" : "chat.composer.cameraBusy"); - } - if (name === "InvalidStateError") { - return t( - kind === "audioinput" - ? "chat.composer.microphonePageInactive" - : "chat.composer.cameraPageInactive", - ); - } - return t( - kind === "audioinput" - ? "chat.composer.microphoneAccessFailed" - : "chat.composer.cameraAccessFailed", +const deviceIssueByDomErrorName: Record = { + NotAllowedError: "permission-blocked", + NotFoundError: "none-found", + NotReadableError: "busy", + InvalidStateError: "page-inactive", +}; + +function deviceIssueFromError(error: unknown): RealtimeTalkDeviceIssue { + return ( + (error instanceof DOMException ? deviceIssueByDomErrorName[error.name] : undefined) ?? "failed" ); } +export function realtimeTalkDeviceIssueMessage( + issue: RealtimeTalkDeviceIssue, + kind: RealtimeTalkDeviceKind, +): string { + const [microphoneKey, cameraKey] = deviceIssueMessageKeys[issue]; + return t(kind === "audioinput" ? microphoneKey : cameraKey); +} + +/** + * Hardware appears and disappears while a picker is on screen, and the empty + * state promises the list keeps up. The caller owns the subscription window: + * run the returned unsubscribe when its surface closes, or the listener + * outlives the state it refreshes. + */ +export function observeRealtimeTalkDevices(onChange: () => void): () => void { + const devices = globalThis.navigator?.mediaDevices; + if (!devices?.addEventListener) { + return () => undefined; + } + devices.addEventListener("devicechange", onChange); + return () => devices.removeEventListener("devicechange", onChange); +} + export function describeRealtimeTalkInputError(error: unknown): string { - return describeDeviceError(error, "audioinput"); + return realtimeTalkDeviceIssueMessage(deviceIssueFromError(error), "audioinput"); } async function discoverRealtimeTalkDevices( requestPermission: boolean, kind: RealtimeTalkDeviceKind, ): Promise { - let devices: MediaDevices; + const devices = globalThis.navigator?.mediaDevices; + if (!devices?.enumerateDevices) { + return { devices: [], permissionRequired: false, issue: "list-unsupported" }; + } let entries: MediaDeviceInfo[]; try { - devices = mediaDevices(kind); entries = await devices.enumerateDevices(); } catch (error) { - return { - devices: [], - permissionRequired: false, - warning: describeDeviceError(error, kind), - }; + return { devices: [], permissionRequired: false, issue: deviceIssueFromError(error) }; } const permissionRequired = deviceDetailsHidden(entries, kind); if (!requestPermission || !permissionRequired || !devices.getUserMedia) { - return { devices: normalizeDevices(entries, kind), permissionRequired, warning: null }; + return { devices: normalizeDevices(entries, kind), permissionRequired, issue: null }; } try { @@ -128,13 +139,13 @@ async function discoverRealtimeTalkDevices( return { devices: normalizeDevices(entries, kind), permissionRequired: deviceDetailsHidden(entries, kind), - warning: null, + issue: null, }; } catch (error) { return { devices: normalizeDevices(entries, kind), permissionRequired, - warning: describeDeviceError(error, kind), + issue: deviceIssueFromError(error), }; } } diff --git a/ui/src/pages/config/config-page.ts b/ui/src/pages/config/config-page.ts index 4b999fe9f633..beec698d98d4 100644 --- a/ui/src/pages/config/config-page.ts +++ b/ui/src/pages/config/config-page.ts @@ -53,6 +53,8 @@ import { loadModels } from "../chat/models.ts"; import { discoverRealtimeTalkCameras, discoverRealtimeTalkInputs, + observeRealtimeTalkDevices, + realtimeTalkDeviceIssueMessage, type RealtimeTalkCameraDevice, type RealtimeTalkInputDevice, } from "../chat/realtime-talk-input.ts"; @@ -242,6 +244,7 @@ export class ConfigPage extends OpenClawLightDomElement { @state() private systemInfoUnavailable = false; @state() private sessionObserverModels: ModelCatalogEntry[] = []; @state() private sessionObserverModelsUnavailable = false; + private mediaDeviceWatch: (() => void) | null = null; @state() private microphoneDevices: RealtimeTalkInputDevice[] = []; @state() private microphonePermissionRequired = true; @state() private microphoneLoading = false; @@ -390,6 +393,13 @@ export class ConfigPage extends OpenClawLightDomElement { this.context.theme.serverSelection, ); this.settings = loadSettings(); + // Passive refresh only: the media rows already own the permission prompt + // behind their own controls, and a hardware change must never turn into an + // unasked-for browser dialog on a settings page. + this.mediaDeviceWatch = observeRealtimeTalkDevices(() => { + void this.refreshMicrophones(false); + void this.refreshCameras(false); + }); this.syncRouteData(); } @@ -399,6 +409,8 @@ export class ConfigPage extends OpenClawLightDomElement { this.hiddenSessionCatalogsChanged, ); this.customThemeImportOwner.retireImport(); + this.mediaDeviceWatch?.(); + this.mediaDeviceWatch = null; this.systemInfoPolling.stop(); this.updateCountdownPolling.stop(); this.invalidateSystemInfoRequest(); @@ -455,7 +467,9 @@ export class ConfigPage extends OpenClawLightDomElement { const result = await discoverRealtimeTalkInputs(requestPermission); this.microphoneDevices = result.devices; this.microphonePermissionRequired = result.permissionRequired; - this.microphoneError = result.warning; + this.microphoneError = result.issue + ? realtimeTalkDeviceIssueMessage(result.issue, "audioinput") + : null; } catch (error) { // Discovery is best-effort in blocked/inactive contexts; a rejection // must not wedge the picker in its loading state. @@ -484,7 +498,9 @@ export class ConfigPage extends OpenClawLightDomElement { const result = await discoverRealtimeTalkCameras(requestPermission); this.cameraDevices = result.devices; this.cameraPermissionRequired = result.permissionRequired; - this.cameraError = result.warning; + this.cameraError = result.issue + ? realtimeTalkDeviceIssueMessage(result.issue, "videoinput") + : null; } catch (error) { this.cameraError = error instanceof Error ? error.message : String(error); } finally { diff --git a/ui/src/styles/chat/layout.css b/ui/src/styles/chat/layout.css index 1106027979ed..456fe3bfd012 100644 --- a/ui/src/styles/chat/layout.css +++ b/ui/src/styles/chat/layout.css @@ -3208,6 +3208,20 @@ openclaw-chat-video-player { color: var(--warn); } +/* Sole content of the popover when no input can be selected, so it carries the + row's own weight instead of reading as a footnote under the heading. */ +.chat-talk-input-picker__empty { + padding: 4px 8px 8px; + color: var(--muted); + font-size: 12px; + line-height: 1.4; + text-wrap: pretty; +} + +.chat-talk-input-picker__empty--fault { + color: var(--warn); +} + .chat-talk-input-picker__hint { margin-top: 3px; border-top: 1px solid color-mix(in srgb, var(--border) 65%, transparent);