diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index 99639661b15d..b0f54377e5c3 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -5193,7 +5193,7 @@ export const en: TranslationMap = { microphoneListUnsupported: "This browser cannot list microphone inputs.", noCameras: "No additional cameras found", noMicrophones: "No additional microphones found", - microphoneNoneFound: "No microphone inputs were found.", + microphoneNoneFound: "No microphone found. Plug one in and it appears here.", microphonePageInactive: "Microphone inputs are unavailable while this page is inactive.", microphonePermissionBlocked: "Microphone access is blocked. Allow it in browser site settings to list inputs.", diff --git a/ui/src/pages/chat/chat-composer.test.ts b/ui/src/pages/chat/chat-composer.test.ts index 47d4d0311713..2e4ed39d318f 100644 --- a/ui/src/pages/chat/chat-composer.test.ts +++ b/ui/src/pages/chat/chat-composer.test.ts @@ -290,7 +290,7 @@ describe("renderChatComposer controls", () => { { deviceId: "studio-mic", label: "Studio microphone" }, { deviceId: "headset", label: "USB headset" }, ], - warning: null, + issue: null, }); patchSettings({ realtimeTalkInputDeviceId: "studio-mic" }); const container = document.createElement("div"); @@ -344,10 +344,55 @@ describe("renderChatComposer controls", () => { expect(dropdown?.open).toBe(true); }); - it("shows discovery warnings and the next-session hint during active Talk", async () => { + it.each([ + ["none-found", "chat.composer.microphoneNoneFound", false], + ["list-unsupported", "chat.composer.microphoneListUnsupported", false], + ["permission-blocked", "chat.composer.microphonePermissionBlocked", true], + ["busy", "chat.composer.microphoneBusy", true], + ["page-inactive", "chat.composer.microphonePageInactive", true], + ["failed", "chat.composer.microphoneAccessFailed", true], + ] as const)( + "renders %s as one empty state with no claimed selection", + async (issue, messageKey, fault) => { + discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue }); + const container = document.createElement("div"); + document.body.append(container); + const composerProps = props({ + onToggleRealtimeTalk: vi.fn(), + realtimeTalkActive: true, + realtimeTalkStatus: "listening", + }); + const draw = () => render(renderChatComposer(composerProps), container); + composerProps.onRequestUpdate = draw; + draw(); + + const dropdown = container.querySelector< + HTMLElement & { open: boolean; updateComplete: Promise } + >("wa-dropdown.chat-talk-input-picker"); + await dropdown?.updateComplete; + button(container, t("chat.composer.microphoneInput")).click(); + const empty = await vi.waitFor(() => { + const node = container.querySelector(".chat-talk-input-picker__empty"); + expect(node?.textContent?.trim()).toBe(t(messageKey)); + return node; + }); + + // One designed state: never a checked System default row, a second + // negative note, or a hint about a selection that cannot be made. + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(0); + expect(container.querySelector(".chat-talk-input-picker__note")).toBeNull(); + expect(container.querySelector(".chat-talk-input-picker__warning")).toBeNull(); + expect(container.querySelector(".chat-talk-input-picker__hint")).toBeNull(); + expect(container.querySelectorAll(".chat-talk-input-picker__empty")).toHaveLength(1); + expect(empty?.getAttribute("role")).toBe("status"); + expect(empty?.classList.contains("chat-talk-input-picker__empty--fault")).toBe(fault); + }, + ); + + it("keeps the list plus one warning when inputs exist but discovery reported an issue", async () => { discoverRealtimeTalkInputsMock.mockResolvedValue({ - devices: [], - warning: "Microphone permission is blocked.", + devices: [{ deviceId: "headset", label: "USB headset" }], + issue: "busy", }); const container = document.createElement("div"); document.body.append(container); @@ -366,17 +411,13 @@ describe("renderChatComposer controls", () => { await dropdown?.updateComplete; button(container, t("chat.composer.microphoneInput")).click(); await vi.waitFor(() => - expect(container.querySelector(".chat-talk-input-picker__warning")?.textContent).toContain( - "Microphone permission is blocked.", - ), + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(2), ); - expect(container.querySelector(".chat-talk-input-picker__warning")?.getAttribute("role")).toBe( - "alert", - ); - expect(container.querySelector(".chat-talk-input-picker__note")?.textContent).toContain( - t("chat.composer.noMicrophones"), + expect(container.querySelector(".chat-talk-input-picker__warning")?.textContent?.trim()).toBe( + t("chat.composer.microphoneBusy"), ); + expect(container.querySelector(".chat-talk-input-picker__empty")).toBeNull(); expect(container.querySelector(".chat-talk-input-picker__hint")?.textContent).toContain( t("chat.composer.microphoneAppliesNextSession"), ); @@ -388,6 +429,92 @@ describe("renderChatComposer controls", () => { expect(dropdown?.open).toBe(false); }); + it("marks the selected input with a single trailing check", async () => { + discoverRealtimeTalkInputsMock.mockResolvedValue({ + devices: [{ deviceId: "headset", label: "USB headset" }], + issue: null, + }); + const container = document.createElement("div"); + document.body.append(container); + const composerProps = props({ onToggleRealtimeTalk: vi.fn() }); + const draw = () => render(renderChatComposer(composerProps), container); + composerProps.onRequestUpdate = draw; + draw(); + + const dropdown = container.querySelector< + HTMLElement & { open: boolean; updateComplete: Promise } + >("wa-dropdown.chat-talk-input-picker"); + await dropdown?.updateComplete; + button(container, t("chat.composer.microphoneInput")).click(); + const items = await vi.waitFor(() => { + const rows = [...container.querySelectorAll(".chat-talk-input-picker__item")]; + expect(rows).toHaveLength(2); + return rows; + }); + + // type="checkbox" would make wa-dropdown-item paint its own leading check + // and toggle it on click, so the row would show two disagreeing marks. + expect(items.map((item) => item.getAttribute("type"))).toEqual(["normal", "normal"]); + expect(items.map((item) => item.querySelectorAll("svg").length)).toEqual([1, 0]); + expect(items[0]?.querySelector(".chat-talk-input-picker__check")?.getAttribute("slot")).toBe( + "details", + ); + expect(items.map((item) => item.getAttribute("aria-checked"))).toEqual(["true", "false"]); + }); + + it("follows devicechange while open and stops listening once closed", async () => { + const mediaDevices = new EventTarget(); + Object.defineProperty(globalThis.navigator, "mediaDevices", { + configurable: true, + value: mediaDevices, + }); + discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue: "none-found" }); + const container = document.createElement("div"); + document.body.append(container); + const composerProps = props({ onToggleRealtimeTalk: vi.fn() }); + const draw = () => render(renderChatComposer(composerProps), container); + composerProps.onRequestUpdate = draw; + draw(); + + const dropdown = container.querySelector< + HTMLElement & { open: boolean; updateComplete: Promise } + >("wa-dropdown.chat-talk-input-picker"); + await dropdown?.updateComplete; + button(container, t("chat.composer.microphoneInput")).click(); + await vi.waitFor(() => + expect(container.querySelector(".chat-talk-input-picker__empty")?.textContent?.trim()).toBe( + t("chat.composer.microphoneNoneFound"), + ), + ); + + // The empty state promises the list keeps up, so plugging in has to land + // without reopening the popover. + discoverRealtimeTalkInputsMock.mockResolvedValue({ + devices: [{ deviceId: "usb", label: "USB Audio Interface" }], + issue: null, + }); + mediaDevices.dispatchEvent(new Event("devicechange")); + await vi.waitFor(() => + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(2), + ); + expect(container.querySelector(".chat-talk-input-picker__empty")).toBeNull(); + + discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue: "none-found" }); + mediaDevices.dispatchEvent(new Event("devicechange")); + await vi.waitFor(() => + expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(0), + ); + + dropdown?.dispatchEvent( + new KeyboardEvent("keydown", { key: "Escape", bubbles: true, cancelable: true }), + ); + await dropdown?.updateComplete; + const callsWhileClosed = discoverRealtimeTalkInputsMock.mock.calls.length; + mediaDevices.dispatchEvent(new Event("devicechange")); + await Promise.resolve(); + expect(discoverRealtimeTalkInputsMock.mock.calls.length).toBe(callsWhileClosed); + }); + it("offers camera only inside a video-capable active talk session", () => { const onToggleRealtimeCamera = vi.fn(); const { container } = renderComposer({ diff --git a/ui/src/pages/chat/components/chat-composer-controls.ts b/ui/src/pages/chat/components/chat-composer-controls.ts index 1b7ba4027ec4..8f8d00bd722d 100644 --- a/ui/src/pages/chat/components/chat-composer-controls.ts +++ b/ui/src/pages/chat/components/chat-composer-controls.ts @@ -5,7 +5,11 @@ import { syncDropdownItemRadio } from "../../../components/web-awesome.ts"; import { t } from "../../../i18n/index.ts"; import type { ControlUiFollowUpMode } from "../../../lib/chat/follow-up-mode.ts"; import type { ComposerDictationController } from "../composer-dictation.ts"; -import type { RealtimeTalkInputDevice } from "../realtime-talk-input.ts"; +import { + realtimeTalkDeviceIssueMessage, + type RealtimeTalkDeviceIssue, + type RealtimeTalkInputDevice, +} from "../realtime-talk-input.ts"; import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts"; import type { RealtimeTalkStatus } from "../realtime-talk.ts"; import { renderMicrophoneActivity, voiceStatusLabel } from "./chat-voice-activity.ts"; @@ -49,18 +53,30 @@ type MicrophonePickerProps = { open: boolean; selectedDeviceId: string; voiceActive: boolean; - warning: string | null; + issue: RealtimeTalkDeviceIssue | null; onOpen: () => void; onClose: () => void; onSelect: (deviceId: string) => void; }; export function renderMicrophonePicker(props: MicrophonePickerProps) { + // Discovery reporting an issue with nothing enumerated is the browser stating + // there is no capture route at all: a "System default" row would claim a + // selection that cannot exist, so the popover shows one empty state instead + // of a checked row stacked on two ways of saying the same thing. + const unavailable = !props.loading && props.devices.length === 0 ? props.issue : null; // System default renders even while discovery runs: the dropdown's one-time // focus step needs at least one item or keyboard users never enter the menu. - const options = props.loading - ? [{ deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }] - : [{ deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }, ...props.devices]; + const options = unavailable + ? [] + : [ + { deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }, + ...(props.loading ? [] : props.devices), + ]; + // A machine without a microphone and a browser that cannot enumerate are + // facts, not faults; only the recoverable reasons earn the warn tone. + const unavailableIsFault = + unavailable !== null && unavailable !== "none-found" && unavailable !== "list-unsupported"; const label = t("chat.composer.microphoneInput"); return html`
${label}
- ${options.map((option) => { - const selected = option.deviceId === props.selectedDeviceId; - return html` - syncDropdownItemRadio(element, selected))} + ${unavailable + ? html`
- ${option.label} - - - `; - })} - ${props.loading - ? html`
${t("common.loading")}
` - : nothing} - ${!props.loading && props.devices.length === 0 - ? html`
${t("chat.composer.noMicrophones")}
` - : nothing} - ${props.warning - ? html`` - : nothing} - ${props.voiceActive - ? html`
- ${t("chat.composer.microphoneAppliesNextSession")} + ${realtimeTalkDeviceIssueMessage(unavailable, "audioinput")}
` - : nothing} + : html` + ${options.map((option) => { + const selected = option.deviceId === props.selectedDeviceId; + // Selection is radio-shaped, so the row stays a plain menu item: + // wa-dropdown-item type="checkbox" paints its own leading check + // and flips it on click, which would contradict this trailing + // check whenever the click does not change the stored device. + return html` + syncDropdownItemRadio(element, selected))} + > + ${option.label} + + + `; + })} + ${props.loading + ? html`
+ ${t("common.loading")} +
` + : nothing} + ${props.issue + ? html`` + : nothing} + ${props.voiceActive + ? html`
+ ${t("chat.composer.microphoneAppliesNextSession")} +
` + : nothing} + `} `; } diff --git a/ui/src/pages/chat/components/chat-composer-state.ts b/ui/src/pages/chat/components/chat-composer-state.ts index 2aefe964de97..71b19c4587b6 100644 --- a/ui/src/pages/chat/components/chat-composer-state.ts +++ b/ui/src/pages/chat/components/chat-composer-state.ts @@ -34,7 +34,8 @@ function createChatComposerState(): ChatComposerState { microphonePickerOpen: false, microphonePickerLoading: false, microphoneDevices: [], - microphoneWarning: null, + microphoneIssue: null, + microphoneDeviceWatch: null, microphoneDiscoveryRequest: 0, capabilityMenuOpen: false, capabilityMenuView: "root", @@ -144,16 +145,27 @@ export function suppressStaleSubmittedDraftReplay( return true; } +/** Drops the devicechange subscription so a closed picker stops refreshing. */ +export function releaseMicrophoneDeviceWatch(state: ChatComposerState) { + state.microphoneDeviceWatch?.(); + state.microphoneDeviceWatch = null; +} + export function resetChatComposerState(paneId?: string) { if (paneId) { // Goal elapsed timers are keyed by element and cleaned up when their // element leaves the DOM, so a per-pane reset does not need to touch them. - composerStates.get(paneId)?.dictation?.dispose(); + const paneState = composerStates.get(paneId); + paneState?.dictation?.dispose(); + if (paneState) { + releaseMicrophoneDeviceWatch(paneState); + } composerStates.delete(paneId); return; } for (const state of composerStates.values()) { state.dictation?.dispose(); + releaseMicrophoneDeviceWatch(state); } composerStates.clear(); clearGoalElapsedTimers(); diff --git a/ui/src/pages/chat/components/chat-composer-types.ts b/ui/src/pages/chat/components/chat-composer-types.ts index c20d2eeedc27..489fe25b8d2a 100644 --- a/ui/src/pages/chat/components/chat-composer-types.ts +++ b/ui/src/pages/chat/components/chat-composer-types.ts @@ -11,7 +11,11 @@ import type { SessionToolOverrides } from "../../../lib/sessions/patch.ts"; import type { ComposerDictationController } from "../composer-dictation.ts"; import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "../input-history.ts"; import type { RealtimeTalkConversationEntry } from "../realtime-talk-conversation.ts"; -import type { RealtimeTalkCameraDevice, RealtimeTalkInputDevice } from "../realtime-talk-input.ts"; +import type { + RealtimeTalkCameraDevice, + RealtimeTalkDeviceIssue, + RealtimeTalkInputDevice, +} from "../realtime-talk-input.ts"; import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts"; import type { RealtimeTalkStatus } from "../realtime-talk.ts"; import type { ChatRunUiStatus } from "../run-lifecycle.ts"; @@ -172,7 +176,9 @@ export type ChatComposerState = { microphonePickerOpen: boolean; microphonePickerLoading: boolean; microphoneDevices: RealtimeTalkInputDevice[]; - microphoneWarning: string | null; + microphoneIssue: RealtimeTalkDeviceIssue | null; + /** Unsubscribe for the devicechange watch; non-null only while the picker is open. */ + microphoneDeviceWatch: (() => void) | null; microphoneDiscoveryRequest: number; capabilityMenuOpen: boolean; capabilityMenuView: ChatComposerPlusMenuView; diff --git a/ui/src/pages/chat/components/chat-composer.ts b/ui/src/pages/chat/components/chat-composer.ts index 7e3092908771..7205ff7ec9c0 100644 --- a/ui/src/pages/chat/components/chat-composer.ts +++ b/ui/src/pages/chat/components/chat-composer.ts @@ -5,7 +5,7 @@ import "../../../components/tooltip.ts"; import { t } from "../../../i18n/index.ts"; import { areUiSessionKeysEquivalent } from "../../../lib/sessions/session-key.ts"; import { ComposerDictationController, insertComposerDictation } from "../composer-dictation.ts"; -import { discoverRealtimeTalkInputs } from "../realtime-talk-input.ts"; +import { discoverRealtimeTalkInputs, observeRealtimeTalkDevices } from "../realtime-talk-input.ts"; import { isLargePastedTextAttachment } from "./chat-attachments.ts"; import { renderContextNotice } from "./chat-composer-context.ts"; import { renderMicrophonePicker, type ChatRunControlsProps } from "./chat-composer-controls.ts"; @@ -48,6 +48,7 @@ import { hasTerminalRunStatus, isCurrentSessionSubmittedProgress, markComposerInputIntent, + releaseMicrophoneDeviceWatch, suppressStaleSubmittedDraftReplay, } from "./chat-composer-state.ts"; import type { ChatComposerProps, ChatComposerState } from "./chat-composer-types.ts"; @@ -505,30 +506,29 @@ export function renderChatComposer(props: ChatComposerProps) { } props.onToggleRealtimeTalk?.(); }; - const openMicrophonePicker = () => { - if (state.microphonePickerOpen) { - return; - } - state.microphonePickerOpen = true; + const discoverMicrophones = () => { state.microphonePickerLoading = true; - state.microphoneWarning = null; + state.microphoneIssue = null; const request = ++state.microphoneDiscoveryRequest; requestUpdate(); + // Permission-requesting discovery on every pass, including device changes: + // a microphone that just appeared has hidden labels until the probe runs, + // and the probe only prompts while the picker is the surface in front of + // the user. void discoverRealtimeTalkInputs(true) .then((result) => { if (request !== state.microphoneDiscoveryRequest) { return; } state.microphoneDevices = result.devices; - state.microphoneWarning = result.warning; + state.microphoneIssue = result.issue; }) - .catch((error: unknown) => { + .catch(() => { if (request !== state.microphoneDiscoveryRequest) { return; } state.microphoneDevices = []; - state.microphoneWarning = - error instanceof Error ? error.message : t("chat.composer.microphoneAccessFailed"); + state.microphoneIssue = "failed"; }) .finally(() => { if (request !== state.microphoneDiscoveryRequest) { @@ -538,15 +538,25 @@ export function renderChatComposer(props: ChatComposerProps) { requestUpdate(); }); }; + const openMicrophonePicker = () => { + if (state.microphonePickerOpen) { + return; + } + state.microphonePickerOpen = true; + state.microphoneDeviceWatch ??= observeRealtimeTalkDevices(discoverMicrophones); + discoverMicrophones(); + }; const closeMicrophonePicker = () => { if (!state.microphonePickerOpen) { return; } + releaseMicrophoneDeviceWatch(state); state.microphonePickerOpen = false; requestUpdate(); }; const selectMicrophone = (deviceId: string) => { patchSettings({ realtimeTalkInputDeviceId: deviceId.trim() || undefined }); + releaseMicrophoneDeviceWatch(state); state.microphonePickerOpen = false; requestUpdate(); }; @@ -558,7 +568,7 @@ export function renderChatComposer(props: ChatComposerProps) { open: state.microphonePickerOpen, selectedDeviceId: selectedMicrophoneId, voiceActive: Boolean(props.realtimeTalkActive), - warning: state.microphoneWarning, + issue: state.microphoneIssue, onOpen: openMicrophonePicker, onClose: closeMicrophonePicker, onSelect: selectMicrophone, diff --git a/ui/src/pages/chat/components/chat-pane-header.ts b/ui/src/pages/chat/components/chat-pane-header.ts index c40c91972cfd..c3eb917797bb 100644 --- a/ui/src/pages/chat/components/chat-pane-header.ts +++ b/ui/src/pages/chat/components/chat-pane-header.ts @@ -177,7 +177,6 @@ function renderGatewayPicker(props: ChatPaneHeaderProps) { const selected = gateway.id === snapshot.currentId; return html` syncDropdownItemRadio(element, selected))} diff --git a/ui/src/pages/chat/realtime-talk-input.test.ts b/ui/src/pages/chat/realtime-talk-input.test.ts index d0883d9a7d52..4c8877457bb9 100644 --- a/ui/src/pages/chat/realtime-talk-input.test.ts +++ b/ui/src/pages/chat/realtime-talk-input.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { discoverRealtimeTalkCameras, discoverRealtimeTalkInputs, + observeRealtimeTalkDevices, openRealtimeTalkCamera, openRealtimeTalkInput, } from "./realtime-talk-input.ts"; @@ -37,7 +38,7 @@ describe("realtime Talk microphone inputs", () => { { deviceId: "usb", label: "Microphone 2" }, ], permissionRequired: true, - warning: null, + issue: null, }); expect(getUserMedia).not.toHaveBeenCalled(); }); @@ -63,7 +64,7 @@ describe("realtime Talk microphone inputs", () => { { deviceId: "loopback", label: "Loopback Audio" }, ], permissionRequired: false, - warning: null, + issue: null, }); expect(getUserMedia).toHaveBeenCalledWith({ audio: true }); expect(stopFirst).toHaveBeenCalledOnce(); @@ -71,7 +72,7 @@ describe("realtime Talk microphone inputs", () => { expect(enumerateDevices).toHaveBeenCalledTimes(2); }); - it("keeps System default usable when microphone permission is denied", async () => { + it("reports the blocked reason when microphone permission is denied", async () => { vi.stubGlobal("navigator", { mediaDevices: { enumerateDevices: vi.fn(async () => [mediaDevice("audioinput", "", "")]), @@ -85,7 +86,52 @@ describe("realtime Talk microphone inputs", () => { expect(result.devices).toEqual([]); expect(result.permissionRequired).toBe(true); - expect(result.warning).toContain("Microphone access is blocked"); + expect(result.issue).toBe("permission-blocked"); + }); + + it("separates an empty machine from a blocked browser", async () => { + vi.stubGlobal("navigator", { + mediaDevices: { + enumerateDevices: vi.fn(async () => []), + getUserMedia: vi.fn(async () => { + throw new DOMException("none", "NotFoundError"); + }), + }, + }); + + await expect(discoverRealtimeTalkInputs(true)).resolves.toEqual({ + devices: [], + permissionRequired: true, + issue: "none-found", + }); + }); + + it("subscribes to devicechange and releases the listener on unsubscribe", () => { + const mediaDevices = new EventTarget(); + let changes = 0; + vi.stubGlobal("navigator", { mediaDevices }); + + const unsubscribe = observeRealtimeTalkDevices(() => (changes += 1)); + mediaDevices.dispatchEvent(new Event("devicechange")); + unsubscribe(); + mediaDevices.dispatchEvent(new Event("devicechange")); + + expect(changes).toBe(1); + }); + + it("stays inert where the browser exposes no media devices to watch", () => { + vi.stubGlobal("navigator", {}); + expect(() => observeRealtimeTalkDevices(() => undefined)()).not.toThrow(); + }); + + it("reports an unsupported enumeration instead of a generic access failure", async () => { + vi.stubGlobal("navigator", { mediaDevices: {} }); + + await expect(discoverRealtimeTalkInputs(true)).resolves.toEqual({ + devices: [], + permissionRequired: false, + issue: "list-unsupported", + }); }); it("does not silently fall back when the selected microphone is unavailable", async () => { @@ -288,7 +334,7 @@ describe("realtime Talk camera inputs", () => { { deviceId: "back", label: "Camera 2" }, ], permissionRequired: true, - warning: null, + issue: null, }); expect(getUserMedia).not.toHaveBeenCalled(); }); @@ -305,7 +351,7 @@ describe("realtime Talk camera inputs", () => { await expect(discoverRealtimeTalkCameras(true)).resolves.toEqual({ devices: [{ deviceId: "camera", label: "Desk Camera" }], permissionRequired: false, - warning: null, + issue: null, }); expect(getUserMedia).toHaveBeenCalledWith({ video: true }); expect(stop).toHaveBeenCalledOnce(); diff --git a/ui/src/pages/chat/realtime-talk-input.ts b/ui/src/pages/chat/realtime-talk-input.ts index e0a69c751ce1..574bfbae6c2b 100644 --- a/ui/src/pages/chat/realtime-talk-input.ts +++ b/ui/src/pages/chat/realtime-talk-input.ts @@ -7,28 +7,36 @@ export type RealtimeTalkInputDevice = { export type RealtimeTalkCameraDevice = RealtimeTalkInputDevice; +/** + * Why discovery stopped is a fact only this module observes. Callers need the + * reason itself — not prose — to pick a coherent rendering, so the code travels + * and each surface owns its own wording and tone. + */ +const deviceIssueMessageKeys = { + "list-unsupported": [ + "chat.composer.microphoneListUnsupported", + "chat.composer.cameraListUnsupported", + ], + "none-found": ["chat.composer.microphoneNoneFound", "chat.composer.cameraNoneFound"], + "permission-blocked": [ + "chat.composer.microphonePermissionBlocked", + "chat.composer.cameraPermissionBlocked", + ], + busy: ["chat.composer.microphoneBusy", "chat.composer.cameraBusy"], + "page-inactive": ["chat.composer.microphonePageInactive", "chat.composer.cameraPageInactive"], + failed: ["chat.composer.microphoneAccessFailed", "chat.composer.cameraAccessFailed"], +} as const; + +export type RealtimeTalkDeviceIssue = keyof typeof deviceIssueMessageKeys; + type RealtimeTalkDeviceDiscovery = { devices: RealtimeTalkInputDevice[]; permissionRequired: boolean; - warning: string | null; + issue: RealtimeTalkDeviceIssue | null; }; type RealtimeTalkDeviceKind = "audioinput" | "videoinput"; -function mediaDevices(kind: RealtimeTalkDeviceKind): MediaDevices { - const devices = globalThis.navigator?.mediaDevices; - if (!devices?.enumerateDevices) { - throw new Error( - t( - kind === "audioinput" - ? "chat.composer.microphoneListUnsupported" - : "chat.composer.cameraListUnsupported", - ), - ); - } - return devices; -} - function normalizeDevices( devices: MediaDeviceInfo[], kind: RealtimeTalkDeviceKind, @@ -63,60 +71,63 @@ function deviceDetailsHidden(devices: MediaDeviceInfo[], kind: RealtimeTalkDevic return inputs.length === 0 || inputs.some((device) => !device.deviceId || !device.label); } -function describeDeviceError(error: unknown, kind: RealtimeTalkDeviceKind): string { - const name = error instanceof DOMException ? error.name : ""; - if (name === "NotAllowedError") { - return t( - kind === "audioinput" - ? "chat.composer.microphonePermissionBlocked" - : "chat.composer.cameraPermissionBlocked", - ); - } - if (name === "NotFoundError") { - return t( - kind === "audioinput" ? "chat.composer.microphoneNoneFound" : "chat.composer.cameraNoneFound", - ); - } - if (name === "NotReadableError") { - return t(kind === "audioinput" ? "chat.composer.microphoneBusy" : "chat.composer.cameraBusy"); - } - if (name === "InvalidStateError") { - return t( - kind === "audioinput" - ? "chat.composer.microphonePageInactive" - : "chat.composer.cameraPageInactive", - ); - } - return t( - kind === "audioinput" - ? "chat.composer.microphoneAccessFailed" - : "chat.composer.cameraAccessFailed", +const deviceIssueByDomErrorName: Record = { + NotAllowedError: "permission-blocked", + NotFoundError: "none-found", + NotReadableError: "busy", + InvalidStateError: "page-inactive", +}; + +function deviceIssueFromError(error: unknown): RealtimeTalkDeviceIssue { + return ( + (error instanceof DOMException ? deviceIssueByDomErrorName[error.name] : undefined) ?? "failed" ); } +export function realtimeTalkDeviceIssueMessage( + issue: RealtimeTalkDeviceIssue, + kind: RealtimeTalkDeviceKind, +): string { + const [microphoneKey, cameraKey] = deviceIssueMessageKeys[issue]; + return t(kind === "audioinput" ? microphoneKey : cameraKey); +} + +/** + * Hardware appears and disappears while a picker is on screen, and the empty + * state promises the list keeps up. The caller owns the subscription window: + * run the returned unsubscribe when its surface closes, or the listener + * outlives the state it refreshes. + */ +export function observeRealtimeTalkDevices(onChange: () => void): () => void { + const devices = globalThis.navigator?.mediaDevices; + if (!devices?.addEventListener) { + return () => undefined; + } + devices.addEventListener("devicechange", onChange); + return () => devices.removeEventListener("devicechange", onChange); +} + export function describeRealtimeTalkInputError(error: unknown): string { - return describeDeviceError(error, "audioinput"); + return realtimeTalkDeviceIssueMessage(deviceIssueFromError(error), "audioinput"); } async function discoverRealtimeTalkDevices( requestPermission: boolean, kind: RealtimeTalkDeviceKind, ): Promise { - let devices: MediaDevices; + const devices = globalThis.navigator?.mediaDevices; + if (!devices?.enumerateDevices) { + return { devices: [], permissionRequired: false, issue: "list-unsupported" }; + } let entries: MediaDeviceInfo[]; try { - devices = mediaDevices(kind); entries = await devices.enumerateDevices(); } catch (error) { - return { - devices: [], - permissionRequired: false, - warning: describeDeviceError(error, kind), - }; + return { devices: [], permissionRequired: false, issue: deviceIssueFromError(error) }; } const permissionRequired = deviceDetailsHidden(entries, kind); if (!requestPermission || !permissionRequired || !devices.getUserMedia) { - return { devices: normalizeDevices(entries, kind), permissionRequired, warning: null }; + return { devices: normalizeDevices(entries, kind), permissionRequired, issue: null }; } try { @@ -128,13 +139,13 @@ async function discoverRealtimeTalkDevices( return { devices: normalizeDevices(entries, kind), permissionRequired: deviceDetailsHidden(entries, kind), - warning: null, + issue: null, }; } catch (error) { return { devices: normalizeDevices(entries, kind), permissionRequired, - warning: describeDeviceError(error, kind), + issue: deviceIssueFromError(error), }; } } diff --git a/ui/src/pages/config/config-page.ts b/ui/src/pages/config/config-page.ts index 4b999fe9f633..beec698d98d4 100644 --- a/ui/src/pages/config/config-page.ts +++ b/ui/src/pages/config/config-page.ts @@ -53,6 +53,8 @@ import { loadModels } from "../chat/models.ts"; import { discoverRealtimeTalkCameras, discoverRealtimeTalkInputs, + observeRealtimeTalkDevices, + realtimeTalkDeviceIssueMessage, type RealtimeTalkCameraDevice, type RealtimeTalkInputDevice, } from "../chat/realtime-talk-input.ts"; @@ -242,6 +244,7 @@ export class ConfigPage extends OpenClawLightDomElement { @state() private systemInfoUnavailable = false; @state() private sessionObserverModels: ModelCatalogEntry[] = []; @state() private sessionObserverModelsUnavailable = false; + private mediaDeviceWatch: (() => void) | null = null; @state() private microphoneDevices: RealtimeTalkInputDevice[] = []; @state() private microphonePermissionRequired = true; @state() private microphoneLoading = false; @@ -390,6 +393,13 @@ export class ConfigPage extends OpenClawLightDomElement { this.context.theme.serverSelection, ); this.settings = loadSettings(); + // Passive refresh only: the media rows already own the permission prompt + // behind their own controls, and a hardware change must never turn into an + // unasked-for browser dialog on a settings page. + this.mediaDeviceWatch = observeRealtimeTalkDevices(() => { + void this.refreshMicrophones(false); + void this.refreshCameras(false); + }); this.syncRouteData(); } @@ -399,6 +409,8 @@ export class ConfigPage extends OpenClawLightDomElement { this.hiddenSessionCatalogsChanged, ); this.customThemeImportOwner.retireImport(); + this.mediaDeviceWatch?.(); + this.mediaDeviceWatch = null; this.systemInfoPolling.stop(); this.updateCountdownPolling.stop(); this.invalidateSystemInfoRequest(); @@ -455,7 +467,9 @@ export class ConfigPage extends OpenClawLightDomElement { const result = await discoverRealtimeTalkInputs(requestPermission); this.microphoneDevices = result.devices; this.microphonePermissionRequired = result.permissionRequired; - this.microphoneError = result.warning; + this.microphoneError = result.issue + ? realtimeTalkDeviceIssueMessage(result.issue, "audioinput") + : null; } catch (error) { // Discovery is best-effort in blocked/inactive contexts; a rejection // must not wedge the picker in its loading state. @@ -484,7 +498,9 @@ export class ConfigPage extends OpenClawLightDomElement { const result = await discoverRealtimeTalkCameras(requestPermission); this.cameraDevices = result.devices; this.cameraPermissionRequired = result.permissionRequired; - this.cameraError = result.warning; + this.cameraError = result.issue + ? realtimeTalkDeviceIssueMessage(result.issue, "videoinput") + : null; } catch (error) { this.cameraError = error instanceof Error ? error.message : String(error); } finally { diff --git a/ui/src/styles/chat/layout.css b/ui/src/styles/chat/layout.css index 1106027979ed..456fe3bfd012 100644 --- a/ui/src/styles/chat/layout.css +++ b/ui/src/styles/chat/layout.css @@ -3208,6 +3208,20 @@ openclaw-chat-video-player { color: var(--warn); } +/* Sole content of the popover when no input can be selected, so it carries the + row's own weight instead of reading as a footnote under the heading. */ +.chat-talk-input-picker__empty { + padding: 4px 8px 8px; + color: var(--muted); + font-size: 12px; + line-height: 1.4; + text-wrap: pretty; +} + +.chat-talk-input-picker__empty--fault { + color: var(--warn); +} + .chat-talk-input-picker__hint { margin-top: 3px; border-top: 1px solid color-mix(in srgb, var(--border) 65%, transparent);