fix(ui): microphone picker claims a selection while denying any input exists (#121397)

* fix(ui): give the microphone picker one truthful state

Discovery now reports why it stopped as a code instead of prose, so the
composer popover can pick a single coherent rendering: with no selectable
input it shows one empty state rather than a checked System default row
stacked on two different ways of saying nothing was found. Rows drop
wa-dropdown-item type="checkbox", whose own leading check toggles on
click and disagrees with the trailing check bound to the stored device;
the chat pane gateway picker had the same mix and is fixed with it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* feat(ui): keep the microphone picker current while it is open

The empty state now promises the list keeps up, so both media surfaces
subscribe to navigator.mediaDevices devicechange for as long as they are
on screen and drop the listener when they close. The composer popover
re-runs permission-requesting discovery, since a microphone that just
appeared has hidden labels until the probe runs; the settings rows
refresh passively so hardware changes never turn into an unasked-for
browser dialog. Also drops the newly unused RealtimeTalkDeviceKind
export that failed the deadcode gate.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Vyctor H. Brzezowski
2026-08-10 01:43:28 -03:00
committed by GitHub
parent 22e6f5b5e0
commit 3f4564f56f
11 changed files with 399 additions and 127 deletions
+1 -1
View File
@@ -5193,7 +5193,7 @@ export const en: TranslationMap = {
microphoneListUnsupported: "This browser cannot list microphone inputs.",
noCameras: "No additional cameras found",
noMicrophones: "No additional microphones found",
microphoneNoneFound: "No microphone inputs were found.",
microphoneNoneFound: "No microphone found. Plug one in and it appears here.",
microphonePageInactive: "Microphone inputs are unavailable while this page is inactive.",
microphonePermissionBlocked:
"Microphone access is blocked. Allow it in browser site settings to list inputs.",
+139 -12
View File
@@ -290,7 +290,7 @@ describe("renderChatComposer controls", () => {
{ deviceId: "studio-mic", label: "Studio microphone" },
{ deviceId: "headset", label: "USB headset" },
],
warning: null,
issue: null,
});
patchSettings({ realtimeTalkInputDeviceId: "studio-mic" });
const container = document.createElement("div");
@@ -344,10 +344,55 @@ describe("renderChatComposer controls", () => {
expect(dropdown?.open).toBe(true);
});
it("shows discovery warnings and the next-session hint during active Talk", async () => {
it.each([
["none-found", "chat.composer.microphoneNoneFound", false],
["list-unsupported", "chat.composer.microphoneListUnsupported", false],
["permission-blocked", "chat.composer.microphonePermissionBlocked", true],
["busy", "chat.composer.microphoneBusy", true],
["page-inactive", "chat.composer.microphonePageInactive", true],
["failed", "chat.composer.microphoneAccessFailed", true],
] as const)(
"renders %s as one empty state with no claimed selection",
async (issue, messageKey, fault) => {
discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue });
const container = document.createElement("div");
document.body.append(container);
const composerProps = props({
onToggleRealtimeTalk: vi.fn(),
realtimeTalkActive: true,
realtimeTalkStatus: "listening",
});
const draw = () => render(renderChatComposer(composerProps), container);
composerProps.onRequestUpdate = draw;
draw();
const dropdown = container.querySelector<
HTMLElement & { open: boolean; updateComplete: Promise<unknown> }
>("wa-dropdown.chat-talk-input-picker");
await dropdown?.updateComplete;
button(container, t("chat.composer.microphoneInput")).click();
const empty = await vi.waitFor(() => {
const node = container.querySelector(".chat-talk-input-picker__empty");
expect(node?.textContent?.trim()).toBe(t(messageKey));
return node;
});
// One designed state: never a checked System default row, a second
// negative note, or a hint about a selection that cannot be made.
expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(0);
expect(container.querySelector(".chat-talk-input-picker__note")).toBeNull();
expect(container.querySelector(".chat-talk-input-picker__warning")).toBeNull();
expect(container.querySelector(".chat-talk-input-picker__hint")).toBeNull();
expect(container.querySelectorAll(".chat-talk-input-picker__empty")).toHaveLength(1);
expect(empty?.getAttribute("role")).toBe("status");
expect(empty?.classList.contains("chat-talk-input-picker__empty--fault")).toBe(fault);
},
);
it("keeps the list plus one warning when inputs exist but discovery reported an issue", async () => {
discoverRealtimeTalkInputsMock.mockResolvedValue({
devices: [],
warning: "Microphone permission is blocked.",
devices: [{ deviceId: "headset", label: "USB headset" }],
issue: "busy",
});
const container = document.createElement("div");
document.body.append(container);
@@ -366,17 +411,13 @@ describe("renderChatComposer controls", () => {
await dropdown?.updateComplete;
button(container, t("chat.composer.microphoneInput")).click();
await vi.waitFor(() =>
expect(container.querySelector(".chat-talk-input-picker__warning")?.textContent).toContain(
"Microphone permission is blocked.",
),
expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(2),
);
expect(container.querySelector(".chat-talk-input-picker__warning")?.getAttribute("role")).toBe(
"alert",
);
expect(container.querySelector(".chat-talk-input-picker__note")?.textContent).toContain(
t("chat.composer.noMicrophones"),
expect(container.querySelector(".chat-talk-input-picker__warning")?.textContent?.trim()).toBe(
t("chat.composer.microphoneBusy"),
);
expect(container.querySelector(".chat-talk-input-picker__empty")).toBeNull();
expect(container.querySelector(".chat-talk-input-picker__hint")?.textContent).toContain(
t("chat.composer.microphoneAppliesNextSession"),
);
@@ -388,6 +429,92 @@ describe("renderChatComposer controls", () => {
expect(dropdown?.open).toBe(false);
});
it("marks the selected input with a single trailing check", async () => {
discoverRealtimeTalkInputsMock.mockResolvedValue({
devices: [{ deviceId: "headset", label: "USB headset" }],
issue: null,
});
const container = document.createElement("div");
document.body.append(container);
const composerProps = props({ onToggleRealtimeTalk: vi.fn() });
const draw = () => render(renderChatComposer(composerProps), container);
composerProps.onRequestUpdate = draw;
draw();
const dropdown = container.querySelector<
HTMLElement & { open: boolean; updateComplete: Promise<unknown> }
>("wa-dropdown.chat-talk-input-picker");
await dropdown?.updateComplete;
button(container, t("chat.composer.microphoneInput")).click();
const items = await vi.waitFor(() => {
const rows = [...container.querySelectorAll(".chat-talk-input-picker__item")];
expect(rows).toHaveLength(2);
return rows;
});
// type="checkbox" would make wa-dropdown-item paint its own leading check
// and toggle it on click, so the row would show two disagreeing marks.
expect(items.map((item) => item.getAttribute("type"))).toEqual(["normal", "normal"]);
expect(items.map((item) => item.querySelectorAll("svg").length)).toEqual([1, 0]);
expect(items[0]?.querySelector(".chat-talk-input-picker__check")?.getAttribute("slot")).toBe(
"details",
);
expect(items.map((item) => item.getAttribute("aria-checked"))).toEqual(["true", "false"]);
});
it("follows devicechange while open and stops listening once closed", async () => {
const mediaDevices = new EventTarget();
Object.defineProperty(globalThis.navigator, "mediaDevices", {
configurable: true,
value: mediaDevices,
});
discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue: "none-found" });
const container = document.createElement("div");
document.body.append(container);
const composerProps = props({ onToggleRealtimeTalk: vi.fn() });
const draw = () => render(renderChatComposer(composerProps), container);
composerProps.onRequestUpdate = draw;
draw();
const dropdown = container.querySelector<
HTMLElement & { open: boolean; updateComplete: Promise<unknown> }
>("wa-dropdown.chat-talk-input-picker");
await dropdown?.updateComplete;
button(container, t("chat.composer.microphoneInput")).click();
await vi.waitFor(() =>
expect(container.querySelector(".chat-talk-input-picker__empty")?.textContent?.trim()).toBe(
t("chat.composer.microphoneNoneFound"),
),
);
// The empty state promises the list keeps up, so plugging in has to land
// without reopening the popover.
discoverRealtimeTalkInputsMock.mockResolvedValue({
devices: [{ deviceId: "usb", label: "USB Audio Interface" }],
issue: null,
});
mediaDevices.dispatchEvent(new Event("devicechange"));
await vi.waitFor(() =>
expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(2),
);
expect(container.querySelector(".chat-talk-input-picker__empty")).toBeNull();
discoverRealtimeTalkInputsMock.mockResolvedValue({ devices: [], issue: "none-found" });
mediaDevices.dispatchEvent(new Event("devicechange"));
await vi.waitFor(() =>
expect(container.querySelectorAll(".chat-talk-input-picker__item")).toHaveLength(0),
);
dropdown?.dispatchEvent(
new KeyboardEvent("keydown", { key: "Escape", bubbles: true, cancelable: true }),
);
await dropdown?.updateComplete;
const callsWhileClosed = discoverRealtimeTalkInputsMock.mock.calls.length;
mediaDevices.dispatchEvent(new Event("devicechange"));
await Promise.resolve();
expect(discoverRealtimeTalkInputsMock.mock.calls.length).toBe(callsWhileClosed);
});
it("offers camera only inside a video-capable active talk session", () => {
const onToggleRealtimeCamera = vi.fn();
const { container } = renderComposer({
@@ -5,7 +5,11 @@ import { syncDropdownItemRadio } from "../../../components/web-awesome.ts";
import { t } from "../../../i18n/index.ts";
import type { ControlUiFollowUpMode } from "../../../lib/chat/follow-up-mode.ts";
import type { ComposerDictationController } from "../composer-dictation.ts";
import type { RealtimeTalkInputDevice } from "../realtime-talk-input.ts";
import {
realtimeTalkDeviceIssueMessage,
type RealtimeTalkDeviceIssue,
type RealtimeTalkInputDevice,
} from "../realtime-talk-input.ts";
import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts";
import type { RealtimeTalkStatus } from "../realtime-talk.ts";
import { renderMicrophoneActivity, voiceStatusLabel } from "./chat-voice-activity.ts";
@@ -49,18 +53,30 @@ type MicrophonePickerProps = {
open: boolean;
selectedDeviceId: string;
voiceActive: boolean;
warning: string | null;
issue: RealtimeTalkDeviceIssue | null;
onOpen: () => void;
onClose: () => void;
onSelect: (deviceId: string) => void;
};
export function renderMicrophonePicker(props: MicrophonePickerProps) {
// Discovery reporting an issue with nothing enumerated is the browser stating
// there is no capture route at all: a "System default" row would claim a
// selection that cannot exist, so the popover shows one empty state instead
// of a checked row stacked on two ways of saying the same thing.
const unavailable = !props.loading && props.devices.length === 0 ? props.issue : null;
// System default renders even while discovery runs: the dropdown's one-time
// focus step needs at least one item or keyboard users never enter the menu.
const options = props.loading
? [{ deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }]
: [{ deviceId: "", label: t("chat.composer.systemDefaultMicrophone") }, ...props.devices];
const options = unavailable
? []
: [
{ deviceId: "", label: t("chat.composer.systemDefaultMicrophone") },
...(props.loading ? [] : props.devices),
];
// A machine without a microphone and a browser that cannot enumerate are
// facts, not faults; only the recoverable reasons earn the warn tone.
const unavailableIsFault =
unavailable !== null && unavailable !== "none-found" && unavailable !== "list-unsupported";
const label = t("chat.composer.microphoneInput");
return html`
<wa-dropdown
@@ -84,38 +100,53 @@ export function renderMicrophonePicker(props: MicrophonePickerProps) {
${icons.chevronDown}
</button>
<div class="chat-talk-input-picker__heading">${label}</div>
${options.map((option) => {
const selected = option.deviceId === props.selectedDeviceId;
return html`
<wa-dropdown-item
class="chat-talk-input-picker__item"
value=${option.deviceId}
type="checkbox"
role="menuitemradio"
aria-checked=${String(selected)}
${ref((element) => syncDropdownItemRadio(element, selected))}
${unavailable
? html`<div
class="chat-talk-input-picker__empty${unavailableIsFault
? " chat-talk-input-picker__empty--fault"
: ""}"
role="status"
>
<span class="chat-talk-input-picker__label">${option.label}</span>
<span slot="details" class="chat-talk-input-picker__check" aria-hidden="true"
>${selected ? icons.check : nothing}</span
>
</wa-dropdown-item>
`;
})}
${props.loading
? html`<div class="chat-talk-input-picker__note" role="status">${t("common.loading")}</div>`
: nothing}
${!props.loading && props.devices.length === 0
? html`<div class="chat-talk-input-picker__note">${t("chat.composer.noMicrophones")}</div>`
: nothing}
${props.warning
? html`<div class="chat-talk-input-picker__warning" role="alert">${props.warning}</div>`
: nothing}
${props.voiceActive
? html`<div class="chat-talk-input-picker__hint">
${t("chat.composer.microphoneAppliesNextSession")}
${realtimeTalkDeviceIssueMessage(unavailable, "audioinput")}
</div>`
: nothing}
: html`
${options.map((option) => {
const selected = option.deviceId === props.selectedDeviceId;
// Selection is radio-shaped, so the row stays a plain menu item:
// wa-dropdown-item type="checkbox" paints its own leading check
// and flips it on click, which would contradict this trailing
// check whenever the click does not change the stored device.
return html`
<wa-dropdown-item
class="chat-talk-input-picker__item"
value=${option.deviceId}
role="menuitemradio"
aria-checked=${String(selected)}
${ref((element) => syncDropdownItemRadio(element, selected))}
>
<span class="chat-talk-input-picker__label">${option.label}</span>
<span slot="details" class="chat-talk-input-picker__check" aria-hidden="true"
>${selected ? icons.check : nothing}</span
>
</wa-dropdown-item>
`;
})}
${props.loading
? html`<div class="chat-talk-input-picker__note" role="status">
${t("common.loading")}
</div>`
: nothing}
${props.issue
? html`<div class="chat-talk-input-picker__warning" role="alert">
${realtimeTalkDeviceIssueMessage(props.issue, "audioinput")}
</div>`
: nothing}
${props.voiceActive
? html`<div class="chat-talk-input-picker__hint">
${t("chat.composer.microphoneAppliesNextSession")}
</div>`
: nothing}
`}
</wa-dropdown>
`;
}
@@ -34,7 +34,8 @@ function createChatComposerState(): ChatComposerState {
microphonePickerOpen: false,
microphonePickerLoading: false,
microphoneDevices: [],
microphoneWarning: null,
microphoneIssue: null,
microphoneDeviceWatch: null,
microphoneDiscoveryRequest: 0,
capabilityMenuOpen: false,
capabilityMenuView: "root",
@@ -144,16 +145,27 @@ export function suppressStaleSubmittedDraftReplay(
return true;
}
/** Drops the devicechange subscription so a closed picker stops refreshing. */
export function releaseMicrophoneDeviceWatch(state: ChatComposerState) {
state.microphoneDeviceWatch?.();
state.microphoneDeviceWatch = null;
}
export function resetChatComposerState(paneId?: string) {
if (paneId) {
// Goal elapsed timers are keyed by element and cleaned up when their
// element leaves the DOM, so a per-pane reset does not need to touch them.
composerStates.get(paneId)?.dictation?.dispose();
const paneState = composerStates.get(paneId);
paneState?.dictation?.dispose();
if (paneState) {
releaseMicrophoneDeviceWatch(paneState);
}
composerStates.delete(paneId);
return;
}
for (const state of composerStates.values()) {
state.dictation?.dispose();
releaseMicrophoneDeviceWatch(state);
}
composerStates.clear();
clearGoalElapsedTimers();
@@ -11,7 +11,11 @@ import type { SessionToolOverrides } from "../../../lib/sessions/patch.ts";
import type { ComposerDictationController } from "../composer-dictation.ts";
import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "../input-history.ts";
import type { RealtimeTalkConversationEntry } from "../realtime-talk-conversation.ts";
import type { RealtimeTalkCameraDevice, RealtimeTalkInputDevice } from "../realtime-talk-input.ts";
import type {
RealtimeTalkCameraDevice,
RealtimeTalkDeviceIssue,
RealtimeTalkInputDevice,
} from "../realtime-talk-input.ts";
import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts";
import type { RealtimeTalkStatus } from "../realtime-talk.ts";
import type { ChatRunUiStatus } from "../run-lifecycle.ts";
@@ -172,7 +176,9 @@ export type ChatComposerState = {
microphonePickerOpen: boolean;
microphonePickerLoading: boolean;
microphoneDevices: RealtimeTalkInputDevice[];
microphoneWarning: string | null;
microphoneIssue: RealtimeTalkDeviceIssue | null;
/** Unsubscribe for the devicechange watch; non-null only while the picker is open. */
microphoneDeviceWatch: (() => void) | null;
microphoneDiscoveryRequest: number;
capabilityMenuOpen: boolean;
capabilityMenuView: ChatComposerPlusMenuView;
+22 -12
View File
@@ -5,7 +5,7 @@ import "../../../components/tooltip.ts";
import { t } from "../../../i18n/index.ts";
import { areUiSessionKeysEquivalent } from "../../../lib/sessions/session-key.ts";
import { ComposerDictationController, insertComposerDictation } from "../composer-dictation.ts";
import { discoverRealtimeTalkInputs } from "../realtime-talk-input.ts";
import { discoverRealtimeTalkInputs, observeRealtimeTalkDevices } from "../realtime-talk-input.ts";
import { isLargePastedTextAttachment } from "./chat-attachments.ts";
import { renderContextNotice } from "./chat-composer-context.ts";
import { renderMicrophonePicker, type ChatRunControlsProps } from "./chat-composer-controls.ts";
@@ -48,6 +48,7 @@ import {
hasTerminalRunStatus,
isCurrentSessionSubmittedProgress,
markComposerInputIntent,
releaseMicrophoneDeviceWatch,
suppressStaleSubmittedDraftReplay,
} from "./chat-composer-state.ts";
import type { ChatComposerProps, ChatComposerState } from "./chat-composer-types.ts";
@@ -505,30 +506,29 @@ export function renderChatComposer(props: ChatComposerProps) {
}
props.onToggleRealtimeTalk?.();
};
const openMicrophonePicker = () => {
if (state.microphonePickerOpen) {
return;
}
state.microphonePickerOpen = true;
const discoverMicrophones = () => {
state.microphonePickerLoading = true;
state.microphoneWarning = null;
state.microphoneIssue = null;
const request = ++state.microphoneDiscoveryRequest;
requestUpdate();
// Permission-requesting discovery on every pass, including device changes:
// a microphone that just appeared has hidden labels until the probe runs,
// and the probe only prompts while the picker is the surface in front of
// the user.
void discoverRealtimeTalkInputs(true)
.then((result) => {
if (request !== state.microphoneDiscoveryRequest) {
return;
}
state.microphoneDevices = result.devices;
state.microphoneWarning = result.warning;
state.microphoneIssue = result.issue;
})
.catch((error: unknown) => {
.catch(() => {
if (request !== state.microphoneDiscoveryRequest) {
return;
}
state.microphoneDevices = [];
state.microphoneWarning =
error instanceof Error ? error.message : t("chat.composer.microphoneAccessFailed");
state.microphoneIssue = "failed";
})
.finally(() => {
if (request !== state.microphoneDiscoveryRequest) {
@@ -538,15 +538,25 @@ export function renderChatComposer(props: ChatComposerProps) {
requestUpdate();
});
};
const openMicrophonePicker = () => {
if (state.microphonePickerOpen) {
return;
}
state.microphonePickerOpen = true;
state.microphoneDeviceWatch ??= observeRealtimeTalkDevices(discoverMicrophones);
discoverMicrophones();
};
const closeMicrophonePicker = () => {
if (!state.microphonePickerOpen) {
return;
}
releaseMicrophoneDeviceWatch(state);
state.microphonePickerOpen = false;
requestUpdate();
};
const selectMicrophone = (deviceId: string) => {
patchSettings({ realtimeTalkInputDeviceId: deviceId.trim() || undefined });
releaseMicrophoneDeviceWatch(state);
state.microphonePickerOpen = false;
requestUpdate();
};
@@ -558,7 +568,7 @@ export function renderChatComposer(props: ChatComposerProps) {
open: state.microphonePickerOpen,
selectedDeviceId: selectedMicrophoneId,
voiceActive: Boolean(props.realtimeTalkActive),
warning: state.microphoneWarning,
issue: state.microphoneIssue,
onOpen: openMicrophonePicker,
onClose: closeMicrophonePicker,
onSelect: selectMicrophone,
@@ -177,7 +177,6 @@ function renderGatewayPicker(props: ChatPaneHeaderProps) {
const selected = gateway.id === snapshot.currentId;
return html`<wa-dropdown-item
class="chat-pane__gateway-menu-item chat-pane__gateway-item"
type="checkbox"
role="menuitemradio"
aria-checked=${String(selected)}
${ref((element) => syncDropdownItemRadio(element, selected))}
+52 -6
View File
@@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import {
discoverRealtimeTalkCameras,
discoverRealtimeTalkInputs,
observeRealtimeTalkDevices,
openRealtimeTalkCamera,
openRealtimeTalkInput,
} from "./realtime-talk-input.ts";
@@ -37,7 +38,7 @@ describe("realtime Talk microphone inputs", () => {
{ deviceId: "usb", label: "Microphone 2" },
],
permissionRequired: true,
warning: null,
issue: null,
});
expect(getUserMedia).not.toHaveBeenCalled();
});
@@ -63,7 +64,7 @@ describe("realtime Talk microphone inputs", () => {
{ deviceId: "loopback", label: "Loopback Audio" },
],
permissionRequired: false,
warning: null,
issue: null,
});
expect(getUserMedia).toHaveBeenCalledWith({ audio: true });
expect(stopFirst).toHaveBeenCalledOnce();
@@ -71,7 +72,7 @@ describe("realtime Talk microphone inputs", () => {
expect(enumerateDevices).toHaveBeenCalledTimes(2);
});
it("keeps System default usable when microphone permission is denied", async () => {
it("reports the blocked reason when microphone permission is denied", async () => {
vi.stubGlobal("navigator", {
mediaDevices: {
enumerateDevices: vi.fn(async () => [mediaDevice("audioinput", "", "")]),
@@ -85,7 +86,52 @@ describe("realtime Talk microphone inputs", () => {
expect(result.devices).toEqual([]);
expect(result.permissionRequired).toBe(true);
expect(result.warning).toContain("Microphone access is blocked");
expect(result.issue).toBe("permission-blocked");
});
it("separates an empty machine from a blocked browser", async () => {
vi.stubGlobal("navigator", {
mediaDevices: {
enumerateDevices: vi.fn(async () => []),
getUserMedia: vi.fn(async () => {
throw new DOMException("none", "NotFoundError");
}),
},
});
await expect(discoverRealtimeTalkInputs(true)).resolves.toEqual({
devices: [],
permissionRequired: true,
issue: "none-found",
});
});
it("subscribes to devicechange and releases the listener on unsubscribe", () => {
const mediaDevices = new EventTarget();
let changes = 0;
vi.stubGlobal("navigator", { mediaDevices });
const unsubscribe = observeRealtimeTalkDevices(() => (changes += 1));
mediaDevices.dispatchEvent(new Event("devicechange"));
unsubscribe();
mediaDevices.dispatchEvent(new Event("devicechange"));
expect(changes).toBe(1);
});
it("stays inert where the browser exposes no media devices to watch", () => {
vi.stubGlobal("navigator", {});
expect(() => observeRealtimeTalkDevices(() => undefined)()).not.toThrow();
});
it("reports an unsupported enumeration instead of a generic access failure", async () => {
vi.stubGlobal("navigator", { mediaDevices: {} });
await expect(discoverRealtimeTalkInputs(true)).resolves.toEqual({
devices: [],
permissionRequired: false,
issue: "list-unsupported",
});
});
it("does not silently fall back when the selected microphone is unavailable", async () => {
@@ -288,7 +334,7 @@ describe("realtime Talk camera inputs", () => {
{ deviceId: "back", label: "Camera 2" },
],
permissionRequired: true,
warning: null,
issue: null,
});
expect(getUserMedia).not.toHaveBeenCalled();
});
@@ -305,7 +351,7 @@ describe("realtime Talk camera inputs", () => {
await expect(discoverRealtimeTalkCameras(true)).resolves.toEqual({
devices: [{ deviceId: "camera", label: "Desk Camera" }],
permissionRequired: false,
warning: null,
issue: null,
});
expect(getUserMedia).toHaveBeenCalledWith({ video: true });
expect(stop).toHaveBeenCalledOnce();
+65 -54
View File
@@ -7,28 +7,36 @@ export type RealtimeTalkInputDevice = {
export type RealtimeTalkCameraDevice = RealtimeTalkInputDevice;
/**
* Why discovery stopped is a fact only this module observes. Callers need the
* reason itself — not prose — to pick a coherent rendering, so the code travels
* and each surface owns its own wording and tone.
*/
const deviceIssueMessageKeys = {
"list-unsupported": [
"chat.composer.microphoneListUnsupported",
"chat.composer.cameraListUnsupported",
],
"none-found": ["chat.composer.microphoneNoneFound", "chat.composer.cameraNoneFound"],
"permission-blocked": [
"chat.composer.microphonePermissionBlocked",
"chat.composer.cameraPermissionBlocked",
],
busy: ["chat.composer.microphoneBusy", "chat.composer.cameraBusy"],
"page-inactive": ["chat.composer.microphonePageInactive", "chat.composer.cameraPageInactive"],
failed: ["chat.composer.microphoneAccessFailed", "chat.composer.cameraAccessFailed"],
} as const;
export type RealtimeTalkDeviceIssue = keyof typeof deviceIssueMessageKeys;
type RealtimeTalkDeviceDiscovery = {
devices: RealtimeTalkInputDevice[];
permissionRequired: boolean;
warning: string | null;
issue: RealtimeTalkDeviceIssue | null;
};
type RealtimeTalkDeviceKind = "audioinput" | "videoinput";
function mediaDevices(kind: RealtimeTalkDeviceKind): MediaDevices {
const devices = globalThis.navigator?.mediaDevices;
if (!devices?.enumerateDevices) {
throw new Error(
t(
kind === "audioinput"
? "chat.composer.microphoneListUnsupported"
: "chat.composer.cameraListUnsupported",
),
);
}
return devices;
}
function normalizeDevices(
devices: MediaDeviceInfo[],
kind: RealtimeTalkDeviceKind,
@@ -63,60 +71,63 @@ function deviceDetailsHidden(devices: MediaDeviceInfo[], kind: RealtimeTalkDevic
return inputs.length === 0 || inputs.some((device) => !device.deviceId || !device.label);
}
function describeDeviceError(error: unknown, kind: RealtimeTalkDeviceKind): string {
const name = error instanceof DOMException ? error.name : "";
if (name === "NotAllowedError") {
return t(
kind === "audioinput"
? "chat.composer.microphonePermissionBlocked"
: "chat.composer.cameraPermissionBlocked",
);
}
if (name === "NotFoundError") {
return t(
kind === "audioinput" ? "chat.composer.microphoneNoneFound" : "chat.composer.cameraNoneFound",
);
}
if (name === "NotReadableError") {
return t(kind === "audioinput" ? "chat.composer.microphoneBusy" : "chat.composer.cameraBusy");
}
if (name === "InvalidStateError") {
return t(
kind === "audioinput"
? "chat.composer.microphonePageInactive"
: "chat.composer.cameraPageInactive",
);
}
return t(
kind === "audioinput"
? "chat.composer.microphoneAccessFailed"
: "chat.composer.cameraAccessFailed",
const deviceIssueByDomErrorName: Record<string, RealtimeTalkDeviceIssue> = {
NotAllowedError: "permission-blocked",
NotFoundError: "none-found",
NotReadableError: "busy",
InvalidStateError: "page-inactive",
};
function deviceIssueFromError(error: unknown): RealtimeTalkDeviceIssue {
return (
(error instanceof DOMException ? deviceIssueByDomErrorName[error.name] : undefined) ?? "failed"
);
}
export function realtimeTalkDeviceIssueMessage(
issue: RealtimeTalkDeviceIssue,
kind: RealtimeTalkDeviceKind,
): string {
const [microphoneKey, cameraKey] = deviceIssueMessageKeys[issue];
return t(kind === "audioinput" ? microphoneKey : cameraKey);
}
/**
* Hardware appears and disappears while a picker is on screen, and the empty
* state promises the list keeps up. The caller owns the subscription window:
* run the returned unsubscribe when its surface closes, or the listener
* outlives the state it refreshes.
*/
export function observeRealtimeTalkDevices(onChange: () => void): () => void {
const devices = globalThis.navigator?.mediaDevices;
if (!devices?.addEventListener) {
return () => undefined;
}
devices.addEventListener("devicechange", onChange);
return () => devices.removeEventListener("devicechange", onChange);
}
export function describeRealtimeTalkInputError(error: unknown): string {
return describeDeviceError(error, "audioinput");
return realtimeTalkDeviceIssueMessage(deviceIssueFromError(error), "audioinput");
}
async function discoverRealtimeTalkDevices(
requestPermission: boolean,
kind: RealtimeTalkDeviceKind,
): Promise<RealtimeTalkDeviceDiscovery> {
let devices: MediaDevices;
const devices = globalThis.navigator?.mediaDevices;
if (!devices?.enumerateDevices) {
return { devices: [], permissionRequired: false, issue: "list-unsupported" };
}
let entries: MediaDeviceInfo[];
try {
devices = mediaDevices(kind);
entries = await devices.enumerateDevices();
} catch (error) {
return {
devices: [],
permissionRequired: false,
warning: describeDeviceError(error, kind),
};
return { devices: [], permissionRequired: false, issue: deviceIssueFromError(error) };
}
const permissionRequired = deviceDetailsHidden(entries, kind);
if (!requestPermission || !permissionRequired || !devices.getUserMedia) {
return { devices: normalizeDevices(entries, kind), permissionRequired, warning: null };
return { devices: normalizeDevices(entries, kind), permissionRequired, issue: null };
}
try {
@@ -128,13 +139,13 @@ async function discoverRealtimeTalkDevices(
return {
devices: normalizeDevices(entries, kind),
permissionRequired: deviceDetailsHidden(entries, kind),
warning: null,
issue: null,
};
} catch (error) {
return {
devices: normalizeDevices(entries, kind),
permissionRequired,
warning: describeDeviceError(error, kind),
issue: deviceIssueFromError(error),
};
}
}
+18 -2
View File
@@ -53,6 +53,8 @@ import { loadModels } from "../chat/models.ts";
import {
discoverRealtimeTalkCameras,
discoverRealtimeTalkInputs,
observeRealtimeTalkDevices,
realtimeTalkDeviceIssueMessage,
type RealtimeTalkCameraDevice,
type RealtimeTalkInputDevice,
} from "../chat/realtime-talk-input.ts";
@@ -242,6 +244,7 @@ export class ConfigPage extends OpenClawLightDomElement {
@state() private systemInfoUnavailable = false;
@state() private sessionObserverModels: ModelCatalogEntry[] = [];
@state() private sessionObserverModelsUnavailable = false;
private mediaDeviceWatch: (() => void) | null = null;
@state() private microphoneDevices: RealtimeTalkInputDevice[] = [];
@state() private microphonePermissionRequired = true;
@state() private microphoneLoading = false;
@@ -390,6 +393,13 @@ export class ConfigPage extends OpenClawLightDomElement {
this.context.theme.serverSelection,
);
this.settings = loadSettings();
// Passive refresh only: the media rows already own the permission prompt
// behind their own controls, and a hardware change must never turn into an
// unasked-for browser dialog on a settings page.
this.mediaDeviceWatch = observeRealtimeTalkDevices(() => {
void this.refreshMicrophones(false);
void this.refreshCameras(false);
});
this.syncRouteData();
}
@@ -399,6 +409,8 @@ export class ConfigPage extends OpenClawLightDomElement {
this.hiddenSessionCatalogsChanged,
);
this.customThemeImportOwner.retireImport();
this.mediaDeviceWatch?.();
this.mediaDeviceWatch = null;
this.systemInfoPolling.stop();
this.updateCountdownPolling.stop();
this.invalidateSystemInfoRequest();
@@ -455,7 +467,9 @@ export class ConfigPage extends OpenClawLightDomElement {
const result = await discoverRealtimeTalkInputs(requestPermission);
this.microphoneDevices = result.devices;
this.microphonePermissionRequired = result.permissionRequired;
this.microphoneError = result.warning;
this.microphoneError = result.issue
? realtimeTalkDeviceIssueMessage(result.issue, "audioinput")
: null;
} catch (error) {
// Discovery is best-effort in blocked/inactive contexts; a rejection
// must not wedge the picker in its loading state.
@@ -484,7 +498,9 @@ export class ConfigPage extends OpenClawLightDomElement {
const result = await discoverRealtimeTalkCameras(requestPermission);
this.cameraDevices = result.devices;
this.cameraPermissionRequired = result.permissionRequired;
this.cameraError = result.warning;
this.cameraError = result.issue
? realtimeTalkDeviceIssueMessage(result.issue, "videoinput")
: null;
} catch (error) {
this.cameraError = error instanceof Error ? error.message : String(error);
} finally {
+14
View File
@@ -3208,6 +3208,20 @@ openclaw-chat-video-player {
color: var(--warn);
}
/* Sole content of the popover when no input can be selected, so it carries the
row's own weight instead of reading as a footnote under the heading. */
.chat-talk-input-picker__empty {
padding: 4px 8px 8px;
color: var(--muted);
font-size: 12px;
line-height: 1.4;
text-wrap: pretty;
}
.chat-talk-input-picker__empty--fault {
color: var(--warn);
}
.chat-talk-input-picker__hint {
margin-top: 3px;
border-top: 1px solid color-mix(in srgb, var(--border) 65%, transparent);