From d60436a4eee2fbacba4aec86f909604b5edd244e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Jul 2026 16:44:05 -0700 Subject: [PATCH] feat(ui): transcode renditions and waveform audio cards (#116115) * feat(ui): add chat media renditions and waveforms * fix(ui): tighten media playback guards * fix(ui): bound waveform decode memory * fix(ui): harden media playback retries * fix(ui): preserve valid media state * fix(ui): stop detached video playback * fix(ui): bound media artifact resolution * fix(ui): reject failed media fallback sources * refactor(ui): split attachment availability helpers * fix(ui): unload media across auth resets * fix(ui): retain the longest-lived media ticket --- ui/src/i18n/locales/en.ts | 1 + ui/src/lib/chat/chat-types.ts | 4 + ui/src/lib/chat/message-normalizer.test.ts | 36 ++ ui/src/lib/chat/message-normalizer.ts | 66 ++ ui/src/pages/chat/chat-pane-state.test.ts | 7 +- ui/src/pages/chat/chat-pane-state.ts | 2 + .../chat/components/chat-audio-player.test.ts | 459 +++++++++++++- .../chat/components/chat-audio-player.ts | 480 ++++++++++++-- .../components/chat-audio-waveform.test.ts | 113 ++++ .../chat/components/chat-audio-waveform.ts | 180 ++++++ .../components/chat-media-playback.test.ts | 149 +++++ .../chat/components/chat-media-playback.ts | 144 +++++ .../chat/components/chat-media-source.test.ts | 30 + .../chat/components/chat-media-source.ts | 36 +- .../chat-message-attachment-availability.ts | 145 +++++ .../chat-message-attachment-status.ts | 33 + .../components/chat-message-attachments.ts | 595 +++++++++++------- .../chat/components/chat-message-bubble.ts | 2 + .../components/chat-message-local-media.ts | 19 + .../chat/components/chat-message-media.ts | 6 +- .../chat/components/chat-message.test.ts | 396 +++++++++++- .../chat/components/chat-video-player.test.ts | 221 +++++++ .../chat/components/chat-video-player.ts | 264 ++++++++ ui/src/styles/chat/layout.css | 49 ++ 24 files changed, 3144 insertions(+), 293 deletions(-) create mode 100644 ui/src/pages/chat/components/chat-audio-waveform.test.ts create mode 100644 ui/src/pages/chat/components/chat-audio-waveform.ts create mode 100644 ui/src/pages/chat/components/chat-media-playback.test.ts create mode 100644 ui/src/pages/chat/components/chat-media-playback.ts create mode 100644 ui/src/pages/chat/components/chat-message-attachment-availability.ts create mode 100644 ui/src/pages/chat/components/chat-message-attachment-status.ts create mode 100644 ui/src/pages/chat/components/chat-video-player.test.ts create mode 100644 ui/src/pages/chat/components/chat-video-player.ts diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index e9b0e293fa82..4e56da94b9e1 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -4844,6 +4844,7 @@ export const en: TranslationMap = { pause: "Pause", seek: "Seek media", download: "Download {filename}", + preparing: "Preparing playback…", videoUnavailable: "Can't play this format — download instead.", }, modelControls: { diff --git a/ui/src/lib/chat/chat-types.ts b/ui/src/lib/chat/chat-types.ts index a9dc1438cecb..dcffcaab4996 100644 --- a/ui/src/lib/chat/chat-types.ts +++ b/ui/src/lib/chat/chat-types.ts @@ -119,6 +119,10 @@ export type MessageContentItem = label: string; mimeType?: string; isVoiceNote?: boolean; + artifactId?: string; + playback?: "native" | "transcode"; + sizeBytes?: number; + durationMs?: number; width?: number; height?: number; }; diff --git a/ui/src/lib/chat/message-normalizer.test.ts b/ui/src/lib/chat/message-normalizer.test.ts index 9df1fc1a7919..75668e23a92a 100644 --- a/ui/src/lib/chat/message-normalizer.test.ts +++ b/ui/src/lib/chat/message-normalizer.test.ts @@ -242,6 +242,42 @@ describe("message-normalizer", () => { ]); }); + it("preserves managed media playback and artifact metadata", () => { + const result = normalizeMessage({ + role: "assistant", + content: [ + { + type: "audio", + artifactId: "artifact_managed_media_audio", + url: "/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full", + fileName: "voice.caf", + mimeType: "audio/x-caf", + playback: "transcode", + sizeBytes: 4096, + durationMs: 2_345, + isVoiceNote: true, + }, + ], + }); + + expect(result.content).toEqual([ + { + type: "attachment", + attachment: { + artifactId: "artifact_managed_media_audio", + url: "/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full", + kind: "audio", + label: "voice.caf", + mimeType: "audio/x-caf", + playback: "transcode", + sizeBytes: 4096, + durationMs: 2_345, + isVoiceNote: true, + }, + }, + ]); + }); + it("does not normalize non-assistant structured audio blocks as attachments", () => { const result = normalizeMessage({ role: "user", diff --git a/ui/src/lib/chat/message-normalizer.ts b/ui/src/lib/chat/message-normalizer.ts index 38f60a3f76f3..50fc5c9d4423 100644 --- a/ui/src/lib/chat/message-normalizer.ts +++ b/ui/src/lib/chat/message-normalizer.ts @@ -294,6 +294,52 @@ function coerceAudioContentBlock( return null; } +function coerceManagedMediaContentBlock( + item: Record, +): Extract | null { + if ((item.type !== "audio" && item.type !== "video") || typeof item.url !== "string") { + return null; + } + const url = item.url.trim(); + if (!url) { + return null; + } + const kind = item.type; + const fallbackLabel = kind === "audio" ? "Audio" : "Video"; + const label = + typeof item.fileName === "string" && item.fileName.trim() + ? item.fileName.trim() + : typeof item.label === "string" && item.label.trim() + ? item.label.trim() + : fallbackLabel; + return { + type: "attachment", + attachment: { + url, + kind, + label, + ...(typeof item.mimeType === "string" ? { mimeType: item.mimeType } : {}), + ...(typeof item.artifactId === "string" ? { artifactId: item.artifactId } : {}), + ...(kind === "audio" && item.isVoiceNote === true ? { isVoiceNote: true } : {}), + ...(item.playback === "native" || item.playback === "transcode" + ? { playback: item.playback } + : {}), + ...(typeof item.sizeBytes === "number" && item.sizeBytes >= 0 + ? { sizeBytes: item.sizeBytes } + : {}), + ...(typeof item.durationMs === "number" && item.durationMs >= 0 + ? { durationMs: item.durationMs } + : {}), + ...(kind === "video" && typeof item.width === "number" && item.width > 0 + ? { width: item.width } + : {}), + ...(kind === "video" && typeof item.height === "number" && item.height > 0 + ? { height: item.height } + : {}), + }, + }; +} + function mergeAdjacentTextItems(items: MessageContentItem[]): MessageContentItem[] { const merged: MessageContentItem[] = []; for (const item of items) { @@ -444,6 +490,10 @@ export function normalizeMessage(message: unknown): NormalizedMessage { } else if (contentItems) { content = contentItems.flatMap((item) => { if (isAssistantMessage) { + const managedMediaAttachment = coerceManagedMediaContentBlock(item); + if (managedMediaAttachment) { + return [managedMediaAttachment]; + } const audioAttachment = coerceAudioContentBlock(item); if (audioAttachment) { return [audioAttachment]; @@ -463,6 +513,10 @@ export function normalizeMessage(message: unknown): NormalizedMessage { label?: unknown; mimeType?: unknown; isVoiceNote?: unknown; + artifactId?: unknown; + playback?: unknown; + sizeBytes?: unknown; + durationMs?: unknown; width?: unknown; height?: unknown; }; @@ -485,6 +539,18 @@ export function normalizeMessage(message: unknown): NormalizedMessage { label: attachment.label, ...(typeof attachment.mimeType === "string" ? { mimeType: attachment.mimeType } : {}), ...(attachment.isVoiceNote === true ? { isVoiceNote: true } : {}), + ...(typeof attachment.artifactId === "string" + ? { artifactId: attachment.artifactId } + : {}), + ...(attachment.playback === "native" || attachment.playback === "transcode" + ? { playback: attachment.playback } + : {}), + ...(typeof attachment.sizeBytes === "number" && attachment.sizeBytes >= 0 + ? { sizeBytes: attachment.sizeBytes } + : {}), + ...(typeof attachment.durationMs === "number" && attachment.durationMs >= 0 + ? { durationMs: attachment.durationMs } + : {}), ...(typeof attachment.width === "number" && attachment.width > 0 ? { width: attachment.width } : {}), diff --git a/ui/src/pages/chat/chat-pane-state.test.ts b/ui/src/pages/chat/chat-pane-state.test.ts index 988db8b7b227..8836f9fb2886 100644 --- a/ui/src/pages/chat/chat-pane-state.test.ts +++ b/ui/src/pages/chat/chat-pane-state.test.ts @@ -86,13 +86,13 @@ describe("applySelectedSessionProjection", () => { describe("resolveChatArtifactDownload", () => { it("returns a trimmed ticket without exposing a gateway bearer credential", async () => { - const requests: Array<{ method: string; params: unknown }> = []; + const requests: Array<{ method: string; params: unknown; options: unknown }> = []; const result = await resolveChatArtifactDownload( { connected: true, client: { - request: async (method: string, params: unknown) => { - requests.push({ method, params }); + request: async (method: string, params: unknown, options: unknown) => { + requests.push({ method, params, options }); return { artifact: { id: "artifact-1", @@ -113,6 +113,7 @@ describe("resolveChatArtifactDownload", () => { { method: "artifacts.download", params: { sessionKey: "agent:main:main", artifactId: "artifact-1" }, + options: { timeoutMs: 30_000 }, }, ]); expect(result).toEqual({ diff --git a/ui/src/pages/chat/chat-pane-state.ts b/ui/src/pages/chat/chat-pane-state.ts index 628cde3a8b1b..00bfffc4cffd 100644 --- a/ui/src/pages/chat/chat-pane-state.ts +++ b/ui/src/pages/chat/chat-pane-state.ts @@ -22,6 +22,7 @@ export function applySelectedSessionProjection( } const MAX_TRACKED_SESSION_ROWS = 256; +const CHAT_ARTIFACT_DOWNLOAD_TIMEOUT_MS = 30_000; export class SessionParticipationTracker { private readonly lastBlocked = new Map(); @@ -94,6 +95,7 @@ export async function resolveChatArtifactDownload( const result = await state.client.request( "artifacts.download", params, + { timeoutMs: CHAT_ARTIFACT_DOWNLOAD_TIMEOUT_MS }, ); const url = typeof result?.url === "string" ? result.url.trim() : ""; if (!url) { diff --git a/ui/src/pages/chat/components/chat-audio-player.test.ts b/ui/src/pages/chat/components/chat-audio-player.test.ts index 445ffc59aa4b..c025e30ac604 100644 --- a/ui/src/pages/chat/components/chat-audio-player.test.ts +++ b/ui/src/pages/chat/components/chat-audio-player.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import "./chat-audio-player.ts"; +import { CHAT_AUDIO_WAVEFORM_MAX_BYTES } from "./chat-audio-waveform.ts"; type ChatAudioPlayer = HTMLElementTagNameMap["openclaw-chat-audio-player"]; @@ -25,6 +26,8 @@ async function createPlayer(label: string): Promise { afterEach(() => { document.body.replaceChildren(); + vi.useRealTimers(); + vi.unstubAllGlobals(); vi.restoreAllMocks(); }); @@ -71,10 +74,11 @@ describe("ChatAudioPlayer", () => { expect(player.querySelector(".chat-audio-player__time")?.textContent).toContain("2:05"); player.querySelector(".chat-audio-player__toggle")!.click(); - await player.updateComplete; - expect(play).toHaveBeenCalledOnce(); - expect(player.querySelector(".chat-audio-player__toggle")?.getAttribute("aria-label")).toBe( - "Pause", + await vi.waitFor(() => expect(play).toHaveBeenCalledOnce()); + await vi.waitFor(() => + expect(player.querySelector(".chat-audio-player__toggle")?.getAttribute("aria-label")).toBe( + "Pause", + ), ); expect( player @@ -153,6 +157,453 @@ describe("ChatAudioPlayer", () => { expect(pauseFirst).not.toHaveBeenCalled(); }); + it("shows preparing, retries a 202 rendition, and then plays", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 202 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-audio-player"); + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket"; + player.sourceIdentity = "/tmp/voice.caf"; + player.label = "voice.caf"; + player.playback = "transcode"; + document.body.append(player); + await player.updateComplete; + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await player.updateComplete; + + expect(player.textContent).toContain("Preparing playback…"); + await vi.advanceTimersByTimeAsync(2_000); + await player.updateComplete; + + const media = player.querySelector("audio")!; + let paused = true; + Object.defineProperty(media, "paused", { configurable: true, get: () => paused }); + const play = vi.spyOn(media, "play").mockImplementation(async () => { + paused = false; + media.dispatchEvent(new Event("play")); + }); + expect(media.getAttribute("src")).toContain("mediaTicket=ticket&playback=1"); + player.querySelector(".chat-audio-player__toggle")!.click(); + await vi.advanceTimersByTimeAsync(0); + await player.updateComplete; + expect(play).toHaveBeenCalledOnce(); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("falls back to download after rendition retries are exhausted", async () => { + vi.useFakeTimers(); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(null, { status: 202 })), + ); + const player = document.createElement("openclaw-chat-audio-player"); + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket"; + player.sourceIdentity = "/tmp/voice.caf"; + player.label = "voice.caf"; + player.playback = "transcode"; + document.body.append(player); + await player.updateComplete; + await vi.runAllTimersAsync(); + await player.updateComplete; + + expect(player.querySelector(".chat-assistant-attachment-card__reason")?.textContent).toContain( + "Can't play this format — download instead.", + ); + expect( + player + .querySelector(".chat-assistant-attachment-card__reason a") + ?.getAttribute("href"), + ).not.toContain("playback=1"); + }); + + it("does not retain an errored source when a refreshed rendition is unavailable", async () => { + let resolveFirstRefresh: ((response: Response) => void) | undefined; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockImplementationOnce( + async () => + await new Promise((resolve) => { + resolveFirstRefresh = resolve; + }), + ) + .mockResolvedValueOnce(new Response(null, { status: 500 })); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-audio-player"); + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=old"; + player.sourceIdentity = "/tmp/voice.caf"; + player.label = "voice.caf"; + player.playback = "transcode"; + document.body.append(player); + await vi.waitFor(() => + expect(player.querySelector("audio")?.getAttribute("src")).toContain("mediaTicket=old"), + ); + + player.querySelector("audio")?.dispatchEvent(new Event("error")); + await player.updateComplete; + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=refresh-1"; + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=refresh-2"; + + await vi.waitFor(() => + expect( + player.querySelector(".chat-assistant-attachment-card__reason")?.textContent, + ).toContain("Can't play this format — download instead."), + ); + expect(fetchMock).toHaveBeenCalledTimes(3); + resolveFirstRefresh?.(new Response(null, { status: 200 })); + }); + + it("reuses the waveform fetch as the audio element Blob source", async () => { + const samples = new Float32Array([0, 0.5, -1, 0.25]); + const decodeAudioData = vi.fn(async () => ({ + duration: 4, + length: samples.length, + numberOfChannels: 1, + getChannelData: () => samples, + })); + const close = vi.fn(async () => undefined); + vi.stubGlobal( + "AudioContext", + class { + decodeAudioData = decodeAudioData; + close = close; + }, + ); + let resolveFetch: ((response: Response) => void) | undefined; + const fetchMock = vi.fn( + async () => + await new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(URL, "createObjectURL").mockReturnValue("blob:waveform-audio"); + vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => undefined); + const player = await createPlayer("waveform-reuse"); + player.serverDurationMs = 4_000; + const media = player.querySelector("audio")!; + let paused = true; + Object.defineProperty(media, "paused", { configurable: true, get: () => paused }); + const play = vi.spyOn(media, "play").mockImplementation(async () => { + paused = false; + media.dispatchEvent(new Event("play")); + }); + vi.spyOn(media, "pause").mockImplementation(() => { + paused = true; + media.dispatchEvent(new Event("pause")); + }); + + player.querySelector(".chat-audio-player__toggle")!.click(); + expect(play).toHaveBeenCalledOnce(); + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce()); + resolveFetch?.( + new Response(new Uint8Array([1, 2, 3, 4]), { + status: 200, + headers: { "Content-Type": "audio/mpeg", "Content-Length": "4" }, + }), + ); + await vi.waitFor(() => + expect(player.querySelectorAll(".chat-audio-player__waveform rect")).toHaveLength(96), + ); + await player.updateComplete; + expect(fetchMock).toHaveBeenCalledOnce(); + expect(decodeAudioData).toHaveBeenCalledOnce(); + expect(media.getAttribute("src")).toBe("https://example.com/waveform-reuse.mp3"); + + media.pause(); + player.querySelector(".chat-audio-player__toggle")!.click(); + await vi.waitFor(() => expect(play).toHaveBeenCalledTimes(2)); + expect(media.getAttribute("src")).toBe("blob:waveform-audio"); + expect(fetchMock).toHaveBeenCalledOnce(); + + player.authToken = "new-principal"; + await player.updateComplete; + expect(media.getAttribute("src")).toBe("https://example.com/waveform-reuse.mp3"); + + player.remove(); + const refreshed = document.createElement("openclaw-chat-audio-player"); + refreshed.src = "https://example.com/waveform-reuse.mp3?mediaTicket=fresh"; + refreshed.sourceIdentity = "media://waveform-reuse"; + refreshed.authToken = "different-principal"; + refreshed.label = "waveform-reuse.mp3"; + refreshed.serverDurationMs = 4_000; + document.body.append(refreshed); + await refreshed.updateComplete; + const refreshedMedia = refreshed.querySelector("audio")!; + Object.defineProperty(refreshedMedia, "paused", { configurable: true, value: true }); + vi.spyOn(refreshedMedia, "play").mockResolvedValue(undefined); + refreshed.querySelector(".chat-audio-player__toggle")!.click(); + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + resolveFetch?.(new Response(null, { status: 500 })); + }); + + it("shows preparation while the initial rendition HEAD is pending", async () => { + let resolveFetch: ((response: Response) => void) | undefined; + vi.stubGlobal( + "fetch", + vi.fn( + async () => + await new Promise((resolve) => { + resolveFetch = resolve; + }), + ), + ); + const player = document.createElement("openclaw-chat-audio-player"); + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket"; + player.sourceIdentity = "/tmp/voice.caf"; + player.label = "voice.caf"; + player.playback = "transcode"; + document.body.append(player); + + await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…")); + resolveFetch?.(new Response(null, { status: 500 })); + }); + + it("skips waveform work without a server-probed duration", async () => { + const decodeAudioData = vi.fn(); + vi.stubGlobal( + "AudioContext", + class { + decodeAudioData = decodeAudioData; + close = vi.fn(async () => undefined); + }, + ); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const player = await createPlayer("unknown-duration"); + const media = player.querySelector("audio")!; + Object.defineProperty(media, "paused", { configurable: true, value: true }); + vi.spyOn(media, "play").mockResolvedValue(undefined); + + player.querySelector(".chat-audio-player__toggle")!.click(); + await vi.waitFor(() => + expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(), + ); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(decodeAudioData).not.toHaveBeenCalled(); + expect(player.querySelector(".chat-audio-player__waveform")).toBeNull(); + }); + + it("discards waveform peaks when decoded duration exceeds the server gate by 20 percent", async () => { + const samples = new Float32Array([0, 0.5, -1, 0.25]); + const decodeAudioData = vi.fn(async () => ({ + duration: 121, + length: samples.length, + numberOfChannels: 1, + getChannelData: () => samples, + })); + const AudioContextMock = vi.fn(function (options?: AudioContextOptions) { + expect(options?.sampleRate).toBe(16_000); + return { decodeAudioData, close: vi.fn(async () => undefined) }; + }); + vi.stubGlobal("AudioContext", AudioContextMock); + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(new Uint8Array([1, 2, 3, 4]), { + status: 200, + headers: { "Content-Type": "audio/mpeg", "Content-Length": "4" }, + }), + ), + ); + vi.spyOn(URL, "createObjectURL").mockReturnValue("blob:duration-mismatch"); + const player = await createPlayer("duration-mismatch"); + player.serverDurationMs = 100_000; + const media = player.querySelector("audio")!; + Object.defineProperty(media, "paused", { configurable: true, value: true }); + setMediaNumber(media, "duration", 100); + media.dispatchEvent(new Event("loadedmetadata")); + await player.updateComplete; + vi.spyOn(media, "play").mockResolvedValue(undefined); + + player.querySelector(".chat-audio-player__toggle")!.click(); + await vi.waitFor(() => + expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(), + ); + + expect(decodeAudioData).toHaveBeenCalledOnce(); + expect(player.querySelector(".chat-audio-player__waveform")).toBeNull(); + expect( + Array.from(player.querySelectorAll(".chat-audio-player__time span"), (item) => + item.textContent?.trim(), + ), + ).toEqual(["0:00", "1:40"]); + }); + + it("does not buffer or cache a chunked waveform response above 8 MiB", async () => { + const decodeAudioData = vi.fn(); + vi.stubGlobal( + "AudioContext", + class { + decodeAudioData = decodeAudioData; + close = vi.fn(async () => undefined); + }, + ); + const fetchMock = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(CHAT_AUDIO_WAVEFORM_MAX_BYTES + 1)); + controller.close(); + }, + }), + { status: 200, headers: { "Content-Type": "audio/mpeg" } }, + ), + ); + vi.stubGlobal("fetch", fetchMock); + const createObjectURL = vi.spyOn(URL, "createObjectURL"); + const player = await createPlayer("oversized-waveform"); + player.serverDurationMs = 4_000; + const media = player.querySelector("audio")!; + Object.defineProperty(media, "paused", { configurable: true, value: true }); + const play = vi.spyOn(media, "play").mockResolvedValue(undefined); + + player.querySelector(".chat-audio-player__toggle")!.click(); + expect(play).toHaveBeenCalledOnce(); + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce()); + await vi.waitFor(() => + expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(), + ); + + expect(createObjectURL).not.toHaveBeenCalled(); + expect(decodeAudioData).not.toHaveBeenCalled(); + expect(player.querySelector(".chat-audio-player__waveform")).toBeNull(); + expect(media.getAttribute("src")).toBe("https://example.com/oversized-waveform.mp3"); + }); + + it("transfers Blob ownership when AudioContext construction fails", async () => { + const FailingAudioContext = vi.fn(function () { + throw new Error("audio context limit reached"); + }); + vi.stubGlobal("AudioContext", FailingAudioContext); + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(new Uint8Array([1, 2, 3, 4]), { + status: 200, + headers: { "Content-Type": "audio/mpeg", "Content-Length": "4" }, + }), + ), + ); + vi.spyOn(URL, "createObjectURL").mockReturnValue("blob:context-fallback"); + const player = await createPlayer("context-fallback"); + player.serverDurationMs = 4_000; + const media = player.querySelector("audio")!; + let paused = true; + Object.defineProperty(media, "paused", { configurable: true, get: () => paused }); + const play = vi.spyOn(media, "play").mockImplementation(async () => { + paused = false; + media.dispatchEvent(new Event("play")); + }); + vi.spyOn(media, "pause").mockImplementation(() => { + paused = true; + media.dispatchEvent(new Event("pause")); + }); + + player.querySelector(".chat-audio-player__toggle")!.click(); + await vi.waitFor(() => + expect((player as unknown as { playRequest: unknown }).playRequest).toBeNull(), + ); + media.pause(); + player.querySelector(".chat-audio-player__toggle")!.click(); + + expect(play).toHaveBeenCalledTimes(2); + expect(media.getAttribute("src")).toBe("blob:context-fallback"); + }); + + it("restarts transcode readiness after disconnecting during preparation", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 202 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-audio-player"); + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket"; + player.sourceIdentity = "/tmp/voice.caf"; + player.label = "voice.caf"; + player.playback = "transcode"; + document.body.append(player); + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce()); + + player.remove(); + document.body.append(player); + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + await vi.waitFor(() => + expect(player.querySelector("audio")?.getAttribute("src")).toContain("playback=1"), + ); + }); + + it("cancels the old source when identity changes during rendition preparation", async () => { + let resolveFetch: ((response: Response) => void) | undefined; + vi.stubGlobal( + "fetch", + vi.fn( + async () => + await new Promise((resolve) => { + resolveFetch = resolve; + }), + ), + ); + const player = await createPlayer("identity-before"); + const media = player.querySelector("audio")!; + let paused = false; + Object.defineProperty(media, "paused", { configurable: true, get: () => paused }); + setMediaNumber(media, "currentTime", 20); + setMediaNumber(media, "duration", 80); + const play = vi.spyOn(media, "play").mockResolvedValue(undefined); + const pause = vi.spyOn(media, "pause").mockImplementation(() => { + paused = true; + }); + + player.src = "/__openclaw__/assistant-media?source=after.caf&mediaTicket=ticket"; + player.sourceIdentity = "media://identity-after"; + player.label = "after.caf"; + player.playback = "transcode"; + await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…")); + + expect(pause).toHaveBeenCalledOnce(); + expect(media.hasAttribute("src")).toBe(false); + resolveFetch?.(new Response(null, { status: 200 })); + await vi.waitFor(() => expect(media.getAttribute("src")).toContain("playback=1")); + media.dispatchEvent(new Event("loadedmetadata")); + expect(play).not.toHaveBeenCalled(); + }); + + it("clears a failed rendition after reconnect succeeds", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 500 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-audio-player"); + player.src = "/__openclaw__/assistant-media?source=voice.caf&mediaTicket=ticket"; + player.sourceIdentity = "/tmp/retry-voice.caf"; + player.label = "voice.caf"; + player.playback = "transcode"; + document.body.append(player); + await vi.waitFor(() => + expect( + player.querySelector(".chat-assistant-attachment-card__reason")?.textContent, + ).toContain("Can't play this format"), + ); + + player.remove(); + document.body.append(player); + await vi.waitFor(() => + expect(player.querySelector("audio")?.getAttribute("src")).toContain("playback=1"), + ); + expect(player.textContent).not.toContain("Can't play this format — download instead."); + }); + it("does not auto-resume a refreshed source after disconnecting before metadata", async () => { const player = await createPlayer("disconnect-resume"); const media = player.querySelector("audio")!; diff --git a/ui/src/pages/chat/components/chat-audio-player.ts b/ui/src/pages/chat/components/chat-audio-player.ts index 165ac982e868..84b13c4493e5 100644 --- a/ui/src/pages/chat/components/chat-audio-player.ts +++ b/ui/src/pages/chat/components/chat-audio-player.ts @@ -1,4 +1,4 @@ -import { html, type PropertyValues } from "lit"; +import { html, svg, type PropertyValues } from "lit"; import { property, state } from "lit/decorators.js"; import { ref } from "lit/directives/ref.js"; import { styleMap } from "lit/directives/style-map.js"; @@ -11,9 +11,69 @@ import { claimChatAudioPlayback, releaseChatAudioPlayback, } from "./chat-audio-coordinator.ts"; +import { + cacheAndRetainChatAudioBlob, + canDecodeChatAudioWaveform, + CHAT_AUDIO_WAVEFORM_MAX_BYTES, + CHAT_AUDIO_WAVEFORM_SAMPLE_RATE, + computeChatAudioWaveformPeaks, + retainCachedChatAudioBlob, + shouldFetchChatAudioWaveform, + type CachedChatAudioBlob, +} from "./chat-audio-waveform.ts"; +import { + appendChatMediaPlaybackParam, + buildChatMediaFetchHeaders, + waitForChatMediaPlayback, + type ChatMediaPlaybackMode, +} from "./chat-media-playback.ts"; import { ChatMediaSourceController } from "./chat-media-source.ts"; const SEEK_STEP_SECONDS = 5; +const WAVEFORM_FETCH_TIMEOUT_MS = 30_000; +const WAVEFORM_DECODE_DURATION_TOLERANCE = 1.2; + +async function readResponseBytesWithinLimit( + response: Response, + maxBytes: number, +): Promise { + const contentLengthHeader = response.headers.get("Content-Length"); + const contentLength = contentLengthHeader === null ? undefined : Number(contentLengthHeader); + if (contentLength !== undefined && Number.isFinite(contentLength) && contentLength > maxBytes) { + await response.body?.cancel().catch(() => undefined); + return null; + } + if (!response.body) { + const bytes = await response.arrayBuffer(); + return bytes.byteLength <= maxBytes ? bytes : null; + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + totalBytes += value.byteLength; + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined); + return null; + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const combined = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + combined.set(chunk, offset); + offset += chunk.byteLength; + } + return combined.buffer; +} function formatChatMediaTime(seconds: number): string { if (!Number.isFinite(seconds) || seconds < 0) { @@ -29,6 +89,10 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement { @property() src = ""; @property() sourceIdentity = ""; @property() label = ""; + @property() playback: ChatMediaPlaybackMode = "native"; + @property() authToken: string | null = null; + @property({ type: Number }) sizeBytes: number | undefined; + @property({ type: Number }) serverDurationMs: number | undefined; @property({ type: Boolean }) voiceNote = false; @property({ attribute: false }) onMediaLoaded: (() => void) | undefined; @@ -37,12 +101,40 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement { @state() private buffered = 0; @state() private playing = false; @state() private failed = false; + @state() private preparing = false; + @state() private playbackReady = true; + @state() private waveformPeaks: readonly number[] | null = null; private media: HTMLAudioElement | null = null; private readonly sourceController = new ChatMediaSourceController(); + private currentSourceFailed = false; private readonly cancelPendingResume = () => this.sourceController.cancelPendingResume(); + private readinessController: AbortController | null = null; + private readinessKey = ""; + private readinessPromise: Promise | null = null; + private readySource = ""; + private playRequest: Promise | null = null; + private releaseWaveformBlob: (() => void) | undefined; + private waveformController: AbortController | null = null; + private waveformAttempted = false; + + override connectedCallback(): void { + super.connectedCallback(); + queueMicrotask(() => this.syncSource()); + } override disconnectedCallback(): void { + this.readinessController?.abort(); + this.readinessController = null; + this.readinessKey = ""; + this.readinessPromise = null; + this.readySource = ""; + this.playbackReady = this.playback !== "transcode"; + this.releaseWaveformBlob?.(); + this.releaseWaveformBlob = undefined; + this.waveformController?.abort(); + this.waveformController = null; + this.waveformAttempted = false; if (this.media) { this.sourceController.cancelPendingResume(); if (!this.media.paused) { @@ -54,29 +146,294 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement { } override updated(changedProperties: PropertyValues): void { - if (changedProperties.has("src")) { - this.failed = false; - } - if (this.media) { - this.sourceController.updateSource(this.media, this.src, this.sourceIdentity); + if ( + changedProperties.has("src") || + changedProperties.has("sourceIdentity") || + changedProperties.has("playback") || + changedProperties.has("authToken") + ) { + const sourceIdentityChanged = + changedProperties.has("sourceIdentity") && + Boolean(this.sourceController.currentIdentity) && + this.sourceController.currentIdentity !== this.sourceIdentity.trim(); + if ( + sourceIdentityChanged || + changedProperties.has("playback") || + changedProperties.has("authToken") + ) { + this.waveformController?.abort(); + this.waveformController = null; + this.releaseWaveformBlob?.(); + this.releaseWaveformBlob = undefined; + this.waveformPeaks = null; + this.waveformAttempted = false; + this.currentTime = 0; + this.duration = 0; + this.buffered = 0; + if (this.media && !this.media.paused) { + this.media.pause(); + releaseChatAudioPlayback(this.media); + } + if ((sourceIdentityChanged || changedProperties.has("authToken")) && this.media) { + this.sourceController.reset(this.media); + this.currentSourceFailed = false; + } + } + this.syncSource(); } } private setMedia = (element: Element | undefined) => { this.media = element instanceof HTMLAudioElement ? element : null; + this.syncSource(); }; - private togglePlayback(): void { + private syncSource(): void { + const media = this.media; + const source = this.src.trim(); + if (!media || !source || !this.sourceIdentity.trim() || !this.isConnected) { + return; + } + if (this.releaseWaveformBlob) { + return; + } + const playbackSource = + this.playback === "transcode" ? appendChatMediaPlaybackParam(source) : source; + const hasCurrentAttachmentSource = + this.sourceController.currentIdentity === this.sourceIdentity.trim(); + const hasUsableCurrentAttachmentSource = + hasCurrentAttachmentSource && !this.currentSourceFailed; + if (this.playback !== "transcode") { + this.readinessController?.abort(); + this.readinessController = null; + this.readinessKey = ""; + this.readinessPromise = null; + this.preparing = false; + this.playbackReady = true; + this.readySource = playbackSource; + this.failed = false; + this.currentSourceFailed = false; + this.sourceController.updateSource(media, playbackSource, this.sourceIdentity); + return; + } + + const readinessKey = `${playbackSource}\0${this.authToken?.trim() ?? ""}`; + if (readinessKey === this.readinessKey && this.readinessController && this.readinessPromise) { + return; + } + this.readinessController?.abort(); + const controller = new AbortController(); + this.readinessController = controller; + this.readinessKey = readinessKey; + this.readySource = ""; + this.preparing = !hasUsableCurrentAttachmentSource; + this.playbackReady = hasUsableCurrentAttachmentSource; + this.failed = false; + const pending = waitForChatMediaPlayback({ + source: playbackSource, + authToken: this.authToken, + signal: controller.signal, + onPreparing: () => { + if (this.readinessController === controller && !hasUsableCurrentAttachmentSource) { + this.preparing = true; + } + }, + }).then((result) => { + if (this.readinessController !== controller || result === "aborted") { + return false; + } + this.preparing = false; + if (result !== "ready") { + if (hasUsableCurrentAttachmentSource) { + this.readySource = this.sourceController.currentSource; + this.playbackReady = true; + return true; + } + this.failed = true; + this.playbackReady = false; + return false; + } + this.readySource = playbackSource; + this.playbackReady = true; + this.failed = false; + this.currentSourceFailed = false; + this.sourceController.updateSource(media, playbackSource, this.sourceIdentity); + return true; + }); + this.readinessPromise = pending; + } + + private resolveWaveformCacheKey(): string { + return [ + this.sourceIdentity.trim(), + this.playback, + this.src.trim(), + this.authToken?.trim() ?? "", + ].join("\0"); + } + + private applyPreparedAudio( + cacheKey: string, + prepared: { value: CachedChatAudioBlob; release: () => void }, + ): void { + const media = this.media; + if (!media || cacheKey !== this.resolveWaveformCacheKey()) { + prepared.release(); + return; + } + this.releaseWaveformBlob?.(); + this.releaseWaveformBlob = prepared.release; + this.waveformPeaks = prepared.value.peaks ?? null; + if (prepared.value.durationSeconds !== undefined) { + this.duration = prepared.value.durationSeconds; + } + this.sourceController.updateSource(media, prepared.value.blobUrl, this.sourceIdentity); + } + + private adoptPreparedAudioForPlayback(): void { const media = this.media; if (!media) { return; } - if (media.paused) { - claimChatAudioPlayback(media, this.cancelPendingResume); - void media.play().catch(() => { - releaseChatAudioPlayback(media); - this.playing = false; + if (!this.releaseWaveformBlob) { + const cached = retainCachedChatAudioBlob(this.resolveWaveformCacheKey()); + if (cached) { + this.applyPreparedAudio(this.resolveWaveformCacheKey(), cached); + } + } + this.sourceController.applyPendingSource(media); + } + + private async prepareWaveformAudio(): Promise { + const media = this.media; + const source = this.readySource; + if (!media || !source || this.releaseWaveformBlob || this.waveformAttempted) { + return; + } + const cacheKey = this.resolveWaveformCacheKey(); + const cached = retainCachedChatAudioBlob(cacheKey); + if (cached) { + this.applyPreparedAudio(cacheKey, cached); + return; + } + const durationSeconds = + this.serverDurationMs !== undefined ? this.serverDurationMs / 1_000 : undefined; + if ( + durationSeconds === undefined || + !shouldFetchChatAudioWaveform({ sizeBytes: this.sizeBytes, durationSeconds }) + ) { + return; + } + const AudioContextConstructor = globalThis.AudioContext; + if (!AudioContextConstructor) { + return; + } + this.waveformAttempted = true; + + const headers = buildChatMediaFetchHeaders(this.authToken); + headers.set("Accept", "audio/*"); + const controller = new AbortController(); + this.waveformController = controller; + const timeout = setTimeout( + () => controller.abort(new DOMException("waveform fetch timed out", "TimeoutError")), + WAVEFORM_FETCH_TIMEOUT_MS, + ); + let response: Response; + let bytes: ArrayBuffer; + try { + response = await fetch(source, { + method: "GET", + headers, + credentials: "same-origin", + signal: controller.signal, }); + if (!response.ok) { + return; + } + const boundedBytes = await readResponseBytesWithinLimit( + response, + CHAT_AUDIO_WAVEFORM_MAX_BYTES, + ); + if (!boundedBytes) { + return; + } + bytes = boundedBytes; + } finally { + clearTimeout(timeout); + if (this.waveformController === controller) { + this.waveformController = null; + } + } + const blob = new Blob([bytes], { + type: response.headers.get("Content-Type")?.split(";", 1)[0]?.trim() || "audio/mpeg", + }); + const blobUrl = URL.createObjectURL(blob); + let peaks: readonly number[] | undefined; + let acceptedDecodedDuration: number | undefined; + if (canDecodeChatAudioWaveform({ sizeBytes: bytes.byteLength, durationSeconds })) { + let context: AudioContext | null = null; + try { + // Duration is trusted only from the server-side ffprobe metadata. + // A 16 kHz decode bounds PCM; >20% duration mismatches are discarded. + context = new AudioContextConstructor({ sampleRate: CHAT_AUDIO_WAVEFORM_SAMPLE_RATE }); + const decoded = await context.decodeAudioData(bytes.slice(0)); + const decodedDuration = Number.isFinite(decoded.duration) ? decoded.duration : undefined; + if ( + decodedDuration !== undefined && + decodedDuration <= durationSeconds * WAVEFORM_DECODE_DURATION_TOLERANCE + ) { + peaks = computeChatAudioWaveformPeaks(decoded); + acceptedDecodedDuration = decodedDuration; + } + } catch { + // A playable browser source can still use the fetched Blob when Web Audio cannot decode it. + } finally { + await context?.close().catch(() => undefined); + } + } + if (!this.isConnected || cacheKey !== this.resolveWaveformCacheKey()) { + URL.revokeObjectURL(blobUrl); + return; + } + const retained = cacheAndRetainChatAudioBlob(cacheKey, { + blobUrl, + sizeBytes: bytes.byteLength, + ...(peaks ? { peaks } : {}), + ...(acceptedDecodedDuration !== undefined + ? { durationSeconds: acceptedDecodedDuration } + : {}), + }); + if (retained) { + this.applyPreparedAudio(cacheKey, retained); + } + } + + private togglePlayback(): void { + const media = this.media; + if (!media || !this.playbackReady) { + return; + } + if (media.paused) { + this.adoptPreparedAudioForPlayback(); + claimChatAudioPlayback(media, this.cancelPendingResume); + const playback = media.play(); + if (!this.playRequest) { + // Invoke play in the click task so strict browser media policies retain user activation. + this.playRequest = playback + .then(() => this.prepareWaveformAudio().catch(() => undefined)) + .catch(() => { + releaseChatAudioPlayback(media); + this.playing = false; + }) + .finally(() => { + this.playRequest = null; + }); + } else { + void playback.catch(() => { + releaseChatAudioPlayback(media); + this.playing = false; + }); + } } else { media.pause(); } @@ -117,6 +474,46 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement { this.buffered = Math.min(1, media.buffered.end(media.buffered.length - 1) / this.duration); } + private renderSeek(progress: number) { + const seek = html` + this.seekTo(Number((event.currentTarget as HTMLInputElement).value))} + />`; + if (!this.waveformPeaks) { + return seek; + } + const count = this.waveformPeaks.length; + return html`
+ + ${seek} +
`; + } + override render() { const progress = this.duration > 0 ? Math.min(1, this.currentTime / this.duration) : 0; const downloadHref = safeAttachmentHref(this.src); @@ -160,41 +557,32 @@ class ChatAudioPlayer extends OpenClawLightDomContentsElement { >` : null} ` - : html`
this.handlePlayerKeydown(event)} - > - -
- - this.seekTo(Number((event.currentTarget as HTMLInputElement).value))} - /> -
- ${formatChatMediaTime(this.currentTime)} - ${formatChatMediaTime(this.duration)} + +
+ ${this.renderSeek(progress)} +
+ ${formatChatMediaTime(this.currentTime)} + ${formatChatMediaTime(this.duration)} +
-
-
`} +
`} diff --git a/ui/src/pages/chat/components/chat-audio-waveform.test.ts b/ui/src/pages/chat/components/chat-audio-waveform.test.ts new file mode 100644 index 000000000000..9677c30642a1 --- /dev/null +++ b/ui/src/pages/chat/components/chat-audio-waveform.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; +import { + cacheAndRetainChatAudioBlob, + canDecodeChatAudioWaveform, + CHAT_AUDIO_WAVEFORM_MAX_BYTES, + computeChatAudioWaveformPeaks, + shouldFetchChatAudioWaveform, +} from "./chat-audio-waveform.ts"; + +describe("chat audio waveform", () => { + it("caps waveform decoding by byte size and duration", () => { + expect( + canDecodeChatAudioWaveform({ + sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES, + durationSeconds: 300, + }), + ).toBe(true); + expect(canDecodeChatAudioWaveform({ sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES + 1 })).toBe( + false, + ); + expect( + canDecodeChatAudioWaveform({ + sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES, + durationSeconds: 301, + }), + ).toBe(false); + expect(shouldFetchChatAudioWaveform({})).toBe(false); + expect(canDecodeChatAudioWaveform({ sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES })).toBe(false); + expect( + canDecodeChatAudioWaveform({ + sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES, + durationSeconds: Number.NaN, + }), + ).toBe(false); + }); + + it("computes normalized peak buckets", () => { + const channels = [new Float32Array([0.1, -0.5, 0.2, 0.4, -0.25, 0.75, 0, -1])]; + const peaks = computeChatAudioWaveformPeaks( + { + length: channels[0]!.length, + numberOfChannels: channels.length, + getChannelData: (channel) => channels[channel]!, + }, + 4, + ); + + expect(peaks).toHaveLength(4); + expect(peaks[0]).toBeCloseTo(0.5); + expect(peaks[1]).toBeCloseTo(0.4); + expect(peaks[2]).toBeCloseTo(0.75); + expect(peaks[3]).toBe(1); + }); + + it("includes peaks carried only by a non-first channel", () => { + const channels = [new Float32Array([0, 0, 0, 0]), new Float32Array([0, 0.25, 0, 1])]; + const peaks = computeChatAudioWaveformPeaks( + { + length: channels[0]!.length, + numberOfChannels: channels.length, + getChannelData: (channel) => channels[channel]!, + }, + 2, + ); + + expect(peaks).toEqual([0.25, 1]); + }); + + it("declines a new Blob when all 32 cache entries are retained", () => { + const releases: Array<() => void> = []; + for (let index = 0; index < 32; index += 1) { + const retained = cacheAndRetainChatAudioBlob(`retained-${index}`, { + blobUrl: `blob:retained-${index}`, + sizeBytes: 1, + }); + expect(retained).not.toBeNull(); + if (!retained) { + throw new Error("expected retained audio blob"); + } + expect(retained.value.blobUrl).toBe(`blob:retained-${index}`); + releases.push(retained.release); + } + expect( + cacheAndRetainChatAudioBlob("retained-overflow", { + blobUrl: "blob:retained-overflow", + sizeBytes: 1, + }), + ).toBeNull(); + + for (const release of releases) { + release(); + } + }); + + it("caps retained audio bytes at 48 MiB", () => { + const releases = Array.from({ length: 6 }, (_, index) => + cacheAndRetainChatAudioBlob(`large-${index}`, { + blobUrl: `blob:large-${index}`, + sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES, + }), + ); + expect(releases.every((retained) => retained !== null)).toBe(true); + expect( + cacheAndRetainChatAudioBlob("large-overflow", { + blobUrl: "blob:large-overflow", + sizeBytes: CHAT_AUDIO_WAVEFORM_MAX_BYTES, + }), + ).toBeNull(); + for (const retained of releases) { + retained?.release(); + } + }); +}); diff --git a/ui/src/pages/chat/components/chat-audio-waveform.ts b/ui/src/pages/chat/components/chat-audio-waveform.ts new file mode 100644 index 000000000000..5d8fef81f9f6 --- /dev/null +++ b/ui/src/pages/chat/components/chat-audio-waveform.ts @@ -0,0 +1,180 @@ +export const CHAT_AUDIO_WAVEFORM_MAX_BYTES = 8 * 1024 * 1024; +export const CHAT_AUDIO_WAVEFORM_SAMPLE_RATE = 16_000; +const CHAT_AUDIO_WAVEFORM_MAX_DURATION_SECONDS = 5 * 60; +const CHAT_AUDIO_WAVEFORM_BUCKET_COUNT = 96; + +type ChatAudioBufferLike = { + length: number; + numberOfChannels: number; + getChannelData(channel: number): Float32Array; +}; + +export type CachedChatAudioBlob = { + blobUrl: string; + sizeBytes: number; + peaks?: readonly number[]; + durationSeconds?: number; +}; + +type ChatAudioBlobCacheEntry = CachedChatAudioBlob & { retainCount: number }; + +const chatAudioBlobCache = new Map(); +const CHAT_AUDIO_BLOB_CACHE_MAX_ENTRIES = 32; +const CHAT_AUDIO_BLOB_CACHE_MAX_BYTES = 48 * 1024 * 1024; + +export function shouldFetchChatAudioWaveform(params: { + sizeBytes?: number; + durationSeconds?: number; +}): boolean { + return ( + params.durationSeconds !== undefined && + Number.isFinite(params.durationSeconds) && + params.durationSeconds >= 0 && + params.durationSeconds <= CHAT_AUDIO_WAVEFORM_MAX_DURATION_SECONDS && + (params.sizeBytes === undefined || + (Number.isFinite(params.sizeBytes) && + params.sizeBytes >= 0 && + params.sizeBytes <= CHAT_AUDIO_WAVEFORM_MAX_BYTES)) + ); +} + +export function canDecodeChatAudioWaveform(params: { + sizeBytes: number; + durationSeconds?: number; +}): boolean { + return ( + Number.isFinite(params.sizeBytes) && + params.sizeBytes >= 0 && + params.sizeBytes <= CHAT_AUDIO_WAVEFORM_MAX_BYTES && + params.durationSeconds !== undefined && + Number.isFinite(params.durationSeconds) && + params.durationSeconds >= 0 && + params.durationSeconds <= CHAT_AUDIO_WAVEFORM_MAX_DURATION_SECONDS + ); +} + +export function computeChatAudioWaveformPeaks( + buffer: ChatAudioBufferLike, + bucketCount = CHAT_AUDIO_WAVEFORM_BUCKET_COUNT, +): number[] { + const count = Math.max(1, Math.floor(bucketCount)); + if (buffer.length <= 0 || buffer.numberOfChannels <= 0) { + return Array.from({ length: count }, () => 0); + } + const channels = Array.from({ length: buffer.numberOfChannels }, (_, channel) => + buffer.getChannelData(channel), + ); + const peaks = Array.from({ length: count }, () => 0); + for (let bucket = 0; bucket < count; bucket += 1) { + const start = Math.floor((bucket * buffer.length) / count); + const end = Math.max(start + 1, Math.floor(((bucket + 1) * buffer.length) / count)); + let peak = 0; + for (const samples of channels) { + for (let index = start; index < Math.min(end, samples.length); index += 1) { + peak = Math.max(peak, Math.abs(samples[index] ?? 0)); + } + } + peaks[bucket] = peak; + } + const maximum = Math.max(...peaks); + return maximum > 0 ? peaks.map((peak) => peak / maximum) : peaks; +} + +function trimChatAudioBlobCache(): void { + while ( + chatAudioBlobCache.size > CHAT_AUDIO_BLOB_CACHE_MAX_ENTRIES || + [...chatAudioBlobCache.values()].reduce((total, entry) => total + entry.sizeBytes, 0) > + CHAT_AUDIO_BLOB_CACHE_MAX_BYTES + ) { + const evictable = [...chatAudioBlobCache].find(([, entry]) => entry.retainCount === 0); + if (!evictable) { + return; + } + const [cacheKey, entry] = evictable; + chatAudioBlobCache.delete(cacheKey); + URL.revokeObjectURL(entry.blobUrl); + } +} + +function makeRoomForChatAudioBlob(sizeBytes: number): boolean { + if (sizeBytes > CHAT_AUDIO_BLOB_CACHE_MAX_BYTES) { + return false; + } + const cachedBytes = () => + [...chatAudioBlobCache.values()].reduce((total, entry) => total + entry.sizeBytes, 0); + while ( + chatAudioBlobCache.size + 1 > CHAT_AUDIO_BLOB_CACHE_MAX_ENTRIES || + cachedBytes() + sizeBytes > CHAT_AUDIO_BLOB_CACHE_MAX_BYTES + ) { + const evictable = [...chatAudioBlobCache].find(([, entry]) => entry.retainCount === 0); + if (!evictable) { + return false; + } + const [cacheKey, entry] = evictable; + chatAudioBlobCache.delete(cacheKey); + URL.revokeObjectURL(entry.blobUrl); + } + return true; +} + +export function retainCachedChatAudioBlob( + cacheKey: string, +): { value: CachedChatAudioBlob; release: () => void } | null { + const entry = chatAudioBlobCache.get(cacheKey); + if (!entry) { + return null; + } + chatAudioBlobCache.delete(cacheKey); + chatAudioBlobCache.set(cacheKey, entry); + entry.retainCount += 1; + let released = false; + return { + value: entry, + release: () => { + if (released) { + return; + } + released = true; + const current = chatAudioBlobCache.get(cacheKey); + if (current && current.retainCount > 0) { + current.retainCount -= 1; + } + trimChatAudioBlobCache(); + }, + }; +} + +export function cacheAndRetainChatAudioBlob( + cacheKey: string, + value: CachedChatAudioBlob, +): { value: CachedChatAudioBlob; release: () => void } | null { + const previous = chatAudioBlobCache.get(cacheKey); + if (previous) { + if (previous.blobUrl !== value.blobUrl) { + URL.revokeObjectURL(value.blobUrl); + } + return retainCachedChatAudioBlob(cacheKey)!; + } + if (!makeRoomForChatAudioBlob(value.sizeBytes)) { + URL.revokeObjectURL(value.blobUrl); + return null; + } + const entry = { ...value, retainCount: 1 }; + chatAudioBlobCache.set(cacheKey, entry); + trimChatAudioBlobCache(); + let released = false; + return { + value: entry, + release: () => { + if (released) { + return; + } + released = true; + const current = chatAudioBlobCache.get(cacheKey); + if (current && current.retainCount > 0) { + current.retainCount -= 1; + } + trimChatAudioBlobCache(); + }, + }; +} diff --git a/ui/src/pages/chat/components/chat-media-playback.test.ts b/ui/src/pages/chat/components/chat-media-playback.test.ts new file mode 100644 index 000000000000..3f40ed1b1cc1 --- /dev/null +++ b/ui/src/pages/chat/components/chat-media-playback.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it, vi } from "vitest"; +import { appendChatMediaPlaybackParam, waitForChatMediaPlayback } from "./chat-media-playback.ts"; + +const EXPECTED_RETRY_DELAYS_MS = [2_000, 4_000, 8_000, 16_000, 30_000, 30_000, 20_000]; + +describe("chat media playback renditions", () => { + it("appends playback=1 without dropping assistant or managed media tickets", () => { + expect( + appendChatMediaPlaybackParam( + "/__openclaw__/assistant-media?source=%2Ftmp%2Fvoice.caf&mediaTicket=assistant", + ), + ).toBe( + "/__openclaw__/assistant-media?source=%2Ftmp%2Fvoice.caf&mediaTicket=assistant&playback=1", + ); + expect( + appendChatMediaPlaybackParam( + "/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full?mediaTicket=managed", + ), + ).toBe( + "/api/chat/media/outgoing/agent%3Amain%3Amain/audio/full?mediaTicket=managed&playback=1", + ); + expect(appendChatMediaPlaybackParam("media/clip.avi?mediaTicket=relative#preview")).toBe( + "media/clip.avi?mediaTicket=relative&playback=1#preview", + ); + expect(appendChatMediaPlaybackParam("//cdn.example/clip.avi?mediaTicket=cdn")).toBe( + "//cdn.example/clip.avi?mediaTicket=cdn&playback=1", + ); + }); + + it("reports preparing and retries until the rendition is ready", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 202 })) + .mockResolvedValueOnce(new Response(null, { status: 202 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + const waitImpl = vi.fn<(delayMs: number, signal: AbortSignal) => Promise>( + async () => undefined, + ); + const onPreparing = vi.fn(); + + await expect( + waitForChatMediaPlayback({ + source: "/media?playback=1", + authToken: "secret-token", + signal: new AbortController().signal, + fetchImpl, + waitImpl, + onPreparing, + }), + ).resolves.toBe("ready"); + + expect(onPreparing).toHaveBeenCalledTimes(2); + expect(waitImpl.mock.calls.map(([delay]) => delay)).toEqual([2_000, 4_000]); + expect(fetchImpl).toHaveBeenCalledTimes(3); + const firstRequest = fetchImpl.mock.calls[0]; + expect(firstRequest?.[0]).toBe("/media?playback=1"); + expect(firstRequest?.[1]?.method).toBe("HEAD"); + expect(new Headers(firstRequest?.[1]?.headers).get("Authorization")).toBe( + "Bearer secret-token", + ); + }); + + it("stops after the bounded two-minute retry schedule", async () => { + const fetchImpl = vi.fn(async () => new Response(null, { status: 202 })); + const waitImpl = vi.fn<(delayMs: number, signal: AbortSignal) => Promise>( + async () => undefined, + ); + + await expect( + waitForChatMediaPlayback({ + source: "/media?playback=1", + signal: new AbortController().signal, + fetchImpl, + waitImpl, + }), + ).resolves.toBe("unavailable"); + + expect(waitImpl.mock.calls.map(([delay]) => delay)).toEqual(EXPECTED_RETRY_DELAYS_MS); + expect(fetchImpl).toHaveBeenCalledTimes(EXPECTED_RETRY_DELAYS_MS.length + 1); + expect(EXPECTED_RETRY_DELAYS_MS.reduce((sum, delay) => sum + delay, 0)).toBe(110_000); + }); + + it("performs a definitive final HEAD after the last backoff", async () => { + vi.useFakeTimers(); + try { + const fetchImpl = vi.fn(); + for (let attempt = 0; attempt < 7; attempt += 1) { + fetchImpl.mockResolvedValueOnce(new Response(null, { status: 202 })); + } + fetchImpl.mockResolvedValueOnce(new Response(null, { status: 200 })); + const pending = waitForChatMediaPlayback({ + source: "/media?playback=1", + signal: new AbortController().signal, + fetchImpl, + }); + + await vi.runAllTimersAsync(); + await expect(pending).resolves.toBe("ready"); + expect(fetchImpl).toHaveBeenCalledTimes(8); + } finally { + vi.useRealTimers(); + } + }); + + it("fails a stalled readiness request at its request deadline", async () => { + vi.useFakeTimers(); + try { + const pending = waitForChatMediaPlayback({ + source: "/media?playback=1", + signal: new AbortController().signal, + requestTimeoutMs: 10, + fetchImpl: vi.fn(async () => await new Promise(() => {})), + }); + await vi.advanceTimersByTimeAsync(10); + await expect(pending).resolves.toBe("unavailable"); + } finally { + vi.useRealTimers(); + } + }); + + it("clamps a retry sleep to the remaining overall deadline", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date(0)); + try { + const fetchImpl = vi + .fn() + .mockImplementationOnce(async () => { + vi.setSystemTime(new Date(119_000)); + return new Response(null, { status: 202 }); + }) + .mockResolvedValueOnce(new Response(null, { status: 500 })); + const waitImpl = vi.fn<(delayMs: number, signal: AbortSignal) => Promise>( + async () => undefined, + ); + + await expect( + waitForChatMediaPlayback({ + source: "/media?playback=1", + signal: new AbortController().signal, + fetchImpl, + waitImpl, + }), + ).resolves.toBe("unavailable"); + expect(waitImpl).toHaveBeenCalledWith(1_000, expect.any(AbortSignal)); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/ui/src/pages/chat/components/chat-media-playback.ts b/ui/src/pages/chat/components/chat-media-playback.ts new file mode 100644 index 000000000000..05dc8221b9d3 --- /dev/null +++ b/ui/src/pages/chat/components/chat-media-playback.ts @@ -0,0 +1,144 @@ +import { appendAttachmentUrlSearchParam } from "./chat-message-local-media.ts"; + +export type ChatMediaPlaybackMode = "native" | "transcode"; + +const CHAT_MEDIA_PLAYBACK_RETRY_DELAYS_MS = [ + 2_000, 4_000, 8_000, 16_000, 30_000, 30_000, 20_000, +] as const; +const CHAT_MEDIA_PLAYBACK_REQUEST_TIMEOUT_MS = 30_000; +const CHAT_MEDIA_PLAYBACK_MAX_WAIT_MS = 120_000; + +type ChatMediaPlaybackReadiness = "ready" | "unavailable" | "aborted"; + +function playbackAbortError(signal: AbortSignal): Error { + return signal.reason instanceof Error + ? signal.reason + : new DOMException("playback preparation aborted", "AbortError"); +} + +export function appendChatMediaPlaybackParam(source: string): string { + return appendAttachmentUrlSearchParam(source, "playback", "1"); +} + +export function buildChatMediaFetchHeaders(authToken: string | null | undefined): Headers { + const headers = new Headers(); + const token = authToken?.trim(); + if (token) { + headers.set("Authorization", `Bearer ${token}`); + } + return headers; +} + +function waitForRetry(delayMs: number, signal: AbortSignal): Promise { + if (signal.aborted) { + return Promise.reject(playbackAbortError(signal)); + } + return new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timer); + reject(playbackAbortError(signal)); + }; + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +async function fetchPlaybackHead(params: { + source: string; + headers: Headers; + signal: AbortSignal; + timeoutMs: number; + fetchImpl: typeof fetch; +}): Promise { + const controller = new AbortController(); + let rejectDeadline: ((error: Error) => void) | undefined; + const deadline = new Promise((_resolve, reject) => { + rejectDeadline = reject; + }); + const onAbort = () => { + const error = playbackAbortError(params.signal); + controller.abort(error); + rejectDeadline?.(error); + }; + params.signal.addEventListener("abort", onAbort, { once: true }); + if (params.signal.aborted) { + onAbort(); + } + const timer = setTimeout(() => { + const error = new DOMException("playback readiness request timed out", "TimeoutError"); + controller.abort(error); + rejectDeadline?.(error); + }, params.timeoutMs); + try { + return await Promise.race([ + params.fetchImpl(params.source, { + method: "HEAD", + headers: params.headers, + credentials: "same-origin", + signal: controller.signal, + }), + deadline, + ]); + } finally { + clearTimeout(timer); + params.signal.removeEventListener("abort", onAbort); + } +} + +export async function waitForChatMediaPlayback(params: { + source: string; + authToken?: string | null; + signal: AbortSignal; + onPreparing?: () => void; + fetchImpl?: typeof fetch; + retryDelaysMs?: readonly number[]; + waitImpl?: (delayMs: number, signal: AbortSignal) => Promise; + requestTimeoutMs?: number; +}): Promise { + const fetchImpl = params.fetchImpl ?? fetch; + const retryDelaysMs = params.retryDelaysMs ?? CHAT_MEDIA_PLAYBACK_RETRY_DELAYS_MS; + const waitImpl = params.waitImpl ?? waitForRetry; + const headers = buildChatMediaFetchHeaders(params.authToken); + headers.set("Accept", "audio/*, video/*"); + const deadline = Date.now() + CHAT_MEDIA_PLAYBACK_MAX_WAIT_MS; + + for (let attempt = 0; ; attempt += 1) { + if (params.signal.aborted) { + return "aborted"; + } + try { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + return "unavailable"; + } + const response = await fetchPlaybackHead({ + source: params.source, + headers, + signal: params.signal, + timeoutMs: Math.min( + params.requestTimeoutMs ?? CHAT_MEDIA_PLAYBACK_REQUEST_TIMEOUT_MS, + remainingMs, + ), + fetchImpl, + }); + if (response.status !== 202) { + return response.ok ? "ready" : "unavailable"; + } + params.onPreparing?.(); + const retryDelay = retryDelaysMs[attempt]; + if (retryDelay === undefined) { + return "unavailable"; + } + const remainingAfterResponseMs = deadline - Date.now(); + if (remainingAfterResponseMs <= 0) { + return "unavailable"; + } + await waitImpl(Math.min(retryDelay, remainingAfterResponseMs), params.signal); + } catch { + return params.signal.aborted ? "aborted" : "unavailable"; + } + } +} diff --git a/ui/src/pages/chat/components/chat-media-source.test.ts b/ui/src/pages/chat/components/chat-media-source.test.ts index 9927c1b552be..5eea214b988f 100644 --- a/ui/src/pages/chat/components/chat-media-source.test.ts +++ b/ui/src/pages/chat/components/chat-media-source.test.ts @@ -105,6 +105,36 @@ describe("ChatMediaSourceController", () => { expect(media.getAttribute("src")).toBe("/media?mediaTicket=fresh"); }); + it("applies a queued Blob before a paused player resumes", () => { + const media = document.createElement("audio"); + const state = { currentTime: 18, duration: 80, paused: false }; + mockMediaState(media, state); + const controller = new ChatMediaSourceController(); + controller.updateSource(media, "/media?mediaTicket=old", "/tmp/audio.mp3"); + controller.updateSource(media, "blob:waveform", "/tmp/audio.mp3"); + state.paused = true; + + expect(controller.applyPendingSource(media)).toBe(true); + expect(media.getAttribute("src")).toBe("blob:waveform"); + expect(controller.currentIdentity).toBe("/tmp/audio.mp3"); + }); + + it("resets an applied source across an authentication boundary", () => { + const media = document.createElement("audio"); + const state = { currentTime: 18, duration: 80, paused: true }; + mockMediaState(media, state); + const load = vi.spyOn(media, "load").mockImplementation(() => undefined); + const controller = new ChatMediaSourceController(); + controller.updateSource(media, "blob:protected-audio", "/tmp/audio.mp3"); + + controller.reset(media); + + expect(media.hasAttribute("src")).toBe(false); + expect(load).toHaveBeenCalledOnce(); + expect(controller.currentSource).toBe(""); + expect(controller.currentIdentity).toBe(""); + }); + it("applies a fresh ticket that arrives after the old source has already failed", () => { const media = document.createElement("audio"); const state = { currentTime: 0, duration: 80, paused: true, error: null as MediaError | null }; diff --git a/ui/src/pages/chat/components/chat-media-source.ts b/ui/src/pages/chat/components/chat-media-source.ts index f6b7a57d423e..2fa6374ef9b7 100644 --- a/ui/src/pages/chat/components/chat-media-source.ts +++ b/ui/src/pages/chat/components/chat-media-source.ts @@ -19,6 +19,10 @@ export class ChatMediaSourceController { return this.appliedSource; } + get currentIdentity(): string { + return this.appliedIdentity; + } + get queuedSource(): string { return this.pendingSource; } @@ -82,6 +86,17 @@ export class ChatMediaSourceController { return true; } + applyPendingSource(media: HTMLMediaElement): boolean { + if (!this.pendingSource) { + return false; + } + this.applySource(media, this.pendingSource, this.pendingIdentity, { + currentTime: finiteMediaTime(media.currentTime), + paused: media.paused, + }); + return true; + } + seek(media: HTMLMediaElement, nextTime: number): boolean { const targetTime = Math.max(0, finiteMediaTime(nextTime)); try { @@ -105,6 +120,16 @@ export class ChatMediaSourceController { } } + reset(media: HTMLMediaElement): void { + this.appliedSource = ""; + this.appliedIdentity = ""; + this.pendingSource = ""; + this.pendingIdentity = ""; + this.restore = null; + media.removeAttribute("src"); + media.load(); + } + handleLoadedMetadata(media: HTMLMediaElement, canResume = () => true): void { const restore = this.restore; if (!restore) { @@ -132,14 +157,3 @@ export class ChatMediaSourceController { media.src = source; } } - -const sourceControllers = new WeakMap(); - -export function getChatMediaSourceController(media: HTMLMediaElement): ChatMediaSourceController { - let controller = sourceControllers.get(media); - if (!controller) { - controller = new ChatMediaSourceController(); - sourceControllers.set(media, controller); - } - return controller; -} diff --git a/ui/src/pages/chat/components/chat-message-attachment-availability.ts b/ui/src/pages/chat/components/chat-message-attachment-availability.ts new file mode 100644 index 000000000000..d3a797ecbc57 --- /dev/null +++ b/ui/src/pages/chat/components/chat-message-attachment-availability.ts @@ -0,0 +1,145 @@ +import { buildAssistantAttachmentUrl } from "./chat-message-local-media.ts"; +import { + isChatMediaResourceCurrent, + notifyChatMediaResourceSubscribers, + scheduleChatMediaResourceRefresh, + type ChatMediaResource, +} from "./chat-message-media.ts"; + +export type AssistantAttachmentAvailability = + | { status: "checking" } + | { + status: "available"; + mediaTicket?: string; + mediaTicketExpiresAt?: number; + refreshAfter?: number; + refreshAttempts?: number; + playback?: "native" | "transcode"; + sizeBytes?: number; + durationMs?: number; + width?: number; + height?: number; + } + | { status: "unavailable"; reason: string; checkedAt: number; retryAttempted?: true }; + +export type ManagedAttachmentAvailability = + | { status: "checking"; refreshAfter?: number; refreshAttempts?: number } + | { + status: "available"; + url: string; + expiresAt?: number; + refreshAfter?: number; + refreshAttempts?: number; + } + | { status: "unavailable"; reason: string; checkedAt: number }; + +export const ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS = 5_000; +export const ASSISTANT_ATTACHMENT_METADATA_FETCH_TIMEOUT_MS = 30_000; +export const ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS = 30_000; +export const ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES = 2; + +let assistantAttachmentAvailabilityRenderVersion = 0; + +export function createUnavailableAssistantAttachment( + reason: string, + retryAttempted: boolean, +): Extract { + return { + status: "unavailable", + reason, + checkedAt: Date.now(), + ...(retryAttempted ? { retryAttempted: true } : {}), + }; +} + +export function getAssistantAttachmentAvailabilityRenderVersion(): number { + return assistantAttachmentAvailabilityRenderVersion; +} + +export function bumpAssistantAttachmentAvailabilityRenderVersion(): void { + assistantAttachmentAvailabilityRenderVersion = + (assistantAttachmentAvailabilityRenderVersion + 1) % Number.MAX_SAFE_INTEGER; +} + +export function buildAssistantAttachmentMetaUrl(source: string, basePath?: string): string { + const attachmentUrl = buildAssistantAttachmentUrl(source, basePath); + return `${attachmentUrl}${attachmentUrl.includes("?") ? "&" : "?"}meta=1`; +} + +export function setAssistantAttachmentAvailability( + resource: ChatMediaResource, + availability: AssistantAttachmentAvailability, +): void { + if (!isChatMediaResourceCurrent(resource)) { + return; + } + resource.value = availability; + bumpAssistantAttachmentAvailabilityRenderVersion(); + scheduleAssistantAttachmentRefresh(resource, availability); +} + +export function scheduleAssistantAttachmentRefresh( + resource: ChatMediaResource, + availability: AssistantAttachmentAvailability, +): void { + const refreshAt = + availability.status === "unavailable" && !availability.retryAttempted + ? availability.checkedAt + ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS + : availability.status === "available" && + availability.mediaTicket && + availability.mediaTicketExpiresAt + ? (availability.refreshAfter ?? + availability.mediaTicketExpiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS) + : undefined; + scheduleChatMediaResourceRefresh(resource, refreshAt, () => { + if (resource.value !== availability) { + return; + } + // Keep the failed generation until its retry can inherit the one-attempt + // budget. A ticket refresh keeps the playable generation mounted while + // its replacement is minted, otherwise the checking card resets playback. + if (availability.status === "available") { + // Virtual rows use this version as their media invalidation key. Notify + // alone updates the host but can leave the attachment row memoized. + bumpAssistantAttachmentAvailabilityRenderVersion(); + } else if (availability.status !== "unavailable") { + resource.value = undefined; + bumpAssistantAttachmentAvailabilityRenderVersion(); + } + notifyChatMediaResourceSubscribers(resource); + }); +} + +export function managedAttachmentRefreshDelayMs(refreshAttempts: number): number { + return ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS * 2 ** Math.max(0, refreshAttempts - 1); +} + +export function selectLaterExpiringManagedAttachment( + current: Extract | null, + incoming: Extract, +): Extract { + return current?.expiresAt !== undefined && current.expiresAt >= (incoming.expiresAt ?? 0) + ? current + : incoming; +} + +export function isManagedOutgoingMediaSource(source: string): boolean { + try { + const parsed = new URL(source, window.location.origin); + return ( + parsed.origin === window.location.origin && + parsed.pathname.startsWith("/api/chat/media/outgoing/") + ); + } catch { + return false; + } +} + +export function resolveManagedOutgoingMediaSessionKey(source: string): string | null { + try { + const encodedSessionKey = new URL(source, window.location.origin).pathname.split("/")[5]; + return encodedSessionKey ? decodeURIComponent(encodedSessionKey) : null; + } catch { + return null; + } +} diff --git a/ui/src/pages/chat/components/chat-message-attachment-status.ts b/ui/src/pages/chat/components/chat-message-attachment-status.ts new file mode 100644 index 000000000000..ad5aa54f3797 --- /dev/null +++ b/ui/src/pages/chat/components/chat-message-attachment-status.ts @@ -0,0 +1,33 @@ +import { html, nothing } from "lit"; +import { icons } from "../../../components/icons.ts"; +import type { AttachmentItem } from "./chat-message-media.ts"; + +export function renderAssistantAttachmentStatusCard(params: { + kind: AttachmentItem["attachment"]["kind"]; + label: string; + badge: string; + reason?: string; +}) { + const icon = + params.kind === "image" + ? icons.image + : params.kind === "audio" + ? icons.mic + : params.kind === "video" + ? icons.monitor + : icons.paperclip; + return html` +
+
+ ${icon} + ${params.label} + ${params.badge} +
+ ${params.reason + ? html`
${params.reason}
` + : nothing} +
+ `; +} diff --git a/ui/src/pages/chat/components/chat-message-attachments.ts b/ui/src/pages/chat/components/chat-message-attachments.ts index b8dac0383bc2..908c32c8a9f1 100644 --- a/ui/src/pages/chat/components/chat-message-attachments.ts +++ b/ui/src/pages/chat/components/chat-message-attachments.ts @@ -1,12 +1,29 @@ import { html, nothing } from "lit"; -import { ref } from "lit/directives/ref.js"; -import { styleMap } from "lit/directives/style-map.js"; import { icons } from "../../../components/icons.ts"; import type { ImageLightboxItem } from "../../../components/image-lightbox.ts"; import { t } from "../../../i18n/index.ts"; import "./chat-audio-player.ts"; +import "./chat-video-player.ts"; import { safeAttachmentHref } from "./chat-attachment-href.ts"; -import { getChatMediaSourceController } from "./chat-media-source.ts"; +import { + ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES, + ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS, + ASSISTANT_ATTACHMENT_METADATA_FETCH_TIMEOUT_MS, + ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS, + buildAssistantAttachmentMetaUrl, + bumpAssistantAttachmentAvailabilityRenderVersion, + createUnavailableAssistantAttachment, + getAssistantAttachmentAvailabilityRenderVersion, + isManagedOutgoingMediaSource, + managedAttachmentRefreshDelayMs, + resolveManagedOutgoingMediaSessionKey, + scheduleAssistantAttachmentRefresh, + selectLaterExpiringManagedAttachment, + setAssistantAttachmentAvailability, + type AssistantAttachmentAvailability, + type ManagedAttachmentAvailability, +} from "./chat-message-attachment-availability.ts"; +import { renderAssistantAttachmentStatusCard } from "./chat-message-attachment-status.ts"; import { openResolvedImage } from "./chat-message-image-open.ts"; import { buildAssistantAttachmentUrl, @@ -19,95 +36,11 @@ import { observeChatMediaResource, scheduleChatMediaResourceRefresh, type AttachmentItem, + type ArtifactDownloadResolver, type ChatMediaResource, } from "./chat-message-media.ts"; -type AssistantAttachmentAvailability = - | { status: "checking" } - | { - status: "available"; - mediaTicket?: string; - mediaTicketExpiresAt?: number; - refreshAfter?: number; - refreshAttempts?: number; - } - | { status: "unavailable"; reason: string; checkedAt: number; retryAttempted?: true }; - -const ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS = 5_000; -const ASSISTANT_ATTACHMENT_METADATA_FETCH_TIMEOUT_MS = 30_000; -const ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS = 30_000; -const ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES = 2; -let assistantAttachmentAvailabilityRenderVersion = 0; - -function createUnavailableAssistantAttachment( - reason: string, - retryAttempted: boolean, -): Extract { - return { - status: "unavailable", - reason, - checkedAt: Date.now(), - ...(retryAttempted ? { retryAttempted: true } : {}), - }; -} - -export function getAssistantAttachmentAvailabilityRenderVersion(): number { - return assistantAttachmentAvailabilityRenderVersion; -} - -function bumpAssistantAttachmentAvailabilityRenderVersion() { - assistantAttachmentAvailabilityRenderVersion = - (assistantAttachmentAvailabilityRenderVersion + 1) % Number.MAX_SAFE_INTEGER; -} - -function setAssistantAttachmentAvailability( - resource: ChatMediaResource, - availability: AssistantAttachmentAvailability, -) { - if (!isChatMediaResourceCurrent(resource)) { - return; - } - resource.value = availability; - bumpAssistantAttachmentAvailabilityRenderVersion(); - scheduleAssistantAttachmentRefresh(resource, availability); -} - -function buildAssistantAttachmentMetaUrl(source: string, basePath?: string): string { - const attachmentUrl = buildAssistantAttachmentUrl(source, basePath); - return `${attachmentUrl}${attachmentUrl.includes("?") ? "&" : "?"}meta=1`; -} - -function scheduleAssistantAttachmentRefresh( - resource: ChatMediaResource, - availability: AssistantAttachmentAvailability, -) { - const refreshAt = - availability.status === "unavailable" && !availability.retryAttempted - ? availability.checkedAt + ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS - : availability.status === "available" && - availability.mediaTicket && - availability.mediaTicketExpiresAt - ? (availability.refreshAfter ?? - availability.mediaTicketExpiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS) - : undefined; - scheduleChatMediaResourceRefresh(resource, refreshAt, () => { - if (resource.value !== availability) { - return; - } - // Keep the failed generation until its retry can inherit the one-attempt - // budget. A ticket refresh keeps the playable generation mounted while - // its replacement is minted, otherwise the checking card resets playback. - if (availability.status === "available") { - // Virtual rows use this version as their media invalidation key. Notify - // alone updates the host but can leave the attachment row memoized. - bumpAssistantAttachmentAvailabilityRenderVersion(); - } else if (availability.status !== "unavailable") { - resource.value = undefined; - bumpAssistantAttachmentAvailabilityRenderVersion(); - } - notifyChatMediaResourceSubscribers(resource); - }); -} +export { getAssistantAttachmentAvailabilityRenderVersion }; export function resolveAssistantAttachmentAvailability( source: string, @@ -232,6 +165,11 @@ export function resolveAssistantAttachmentAvailability( available?: boolean; mediaTicket?: string; mediaTicketExpiresAt?: string; + playback?: "native" | "transcode"; + sizeBytes?: number; + durationMs?: number; + width?: number; + height?: number; reason?: string; } | null; if (payload?.available === true) { @@ -252,6 +190,13 @@ export function resolveAssistantAttachmentAvailability( const availability: AssistantAttachmentAvailability = { status: "available", ...(mediaTicket ? { mediaTicket, mediaTicketExpiresAt } : {}), + ...(payload.playback === "native" || payload.playback === "transcode" + ? { playback: payload.playback } + : {}), + ...(typeof payload.sizeBytes === "number" ? { sizeBytes: payload.sizeBytes } : {}), + ...(typeof payload.durationMs === "number" ? { durationMs: payload.durationMs } : {}), + ...(typeof payload.width === "number" ? { width: payload.width } : {}), + ...(typeof payload.height === "number" ? { height: payload.height } : {}), }; resource.retryAttempted = false; setAssistantAttachmentAvailability(resource, availability); @@ -291,59 +236,283 @@ export function resolveAssistantAttachmentAvailability( return refreshingAvailability ?? { status: "checking" }; } -function renderAssistantAttachmentStatusCard(params: { - kind: AttachmentItem["attachment"]["kind"]; - label: string; - badge: string; - reason?: string; -}) { - const icon = - params.kind === "image" - ? icons.image - : params.kind === "audio" - ? icons.mic - : params.kind === "video" - ? icons.monitor - : icons.paperclip; - return html` -
-
- ${icon} - ${params.label} - ${params.badge} -
- ${params.reason - ? html`
${params.reason}
` - : nothing} -
- `; -} - -function videoCardFor(media: HTMLVideoElement): HTMLElement | null { - return media.closest(".chat-assistant-attachment-card--video"); -} - -function markVideoMetadataLoaded(media: HTMLVideoElement, loaded: boolean): void { - videoCardFor(media)?.toggleAttribute("data-metadata-loaded", loaded); -} - -function markVideoUnplayable(media: HTMLVideoElement, unplayable: boolean): void { - videoCardFor(media)?.toggleAttribute("data-unplayable", unplayable); -} - -function syncVideoSource(media: HTMLVideoElement, source: string, sourceIdentity: string): void { - getChatMediaSourceController(media).updateSource(media, source, sourceIdentity); -} - -function recoverVideoSource(media: HTMLVideoElement): boolean { - const recovered = getChatMediaSourceController(media).handleError(media); - if (recovered) { - markVideoMetadataLoaded(media, false); - markVideoUnplayable(media, false); +function retainManagedAttachmentUntilExpiry( + resource: ChatMediaResource, + availability: Extract | null, + refreshAttempts: number, +): Extract | null { + if (!availability?.expiresAt || availability.expiresAt <= Date.now()) { + return null; } - return recovered; + const retained = { + ...availability, + refreshAfter: availability.expiresAt, + refreshAttempts, + }; + setManagedAttachmentAvailability(resource, retained); + return retained; +} + +function setManagedAttachmentAvailability( + resource: ChatMediaResource, + availability: ManagedAttachmentAvailability, + scheduleExpiryOnly = false, +): void { + if (!isChatMediaResourceCurrent(resource)) { + return; + } + resource.value = availability; + bumpAssistantAttachmentAvailabilityRenderVersion(); + const refreshAt = + availability.status === "checking" + ? availability.refreshAfter + : availability.status === "available" && availability.expiresAt !== undefined + ? scheduleExpiryOnly + ? availability.expiresAt + : Math.min( + availability.refreshAfter ?? + availability.expiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS, + availability.expiresAt, + ) + : availability.status === "unavailable" && !resource.retryAttempted + ? availability.checkedAt + ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS + : undefined; + scheduleChatMediaResourceRefresh(resource, refreshAt, () => { + if (resource.value?.status === "unavailable") { + resource.retryAttempted = true; + resource.value = undefined; + } + bumpAssistantAttachmentAvailabilityRenderVersion(); + notifyChatMediaResourceSubscribers(resource); + }); +} + +function resolveManagedAttachmentAvailability( + attachment: AttachmentItem["attachment"], + resolveArtifactDownload: ArtifactDownloadResolver | undefined, + onRequestUpdate: (() => void) | undefined, +): ManagedAttachmentAvailability { + if (!isManagedOutgoingMediaSource(attachment.url)) { + return { status: "available", url: attachment.url }; + } + if (!attachment.artifactId || !resolveArtifactDownload) { + if (new URL(attachment.url, window.location.origin).searchParams.get("mediaTicket")?.trim()) { + return { status: "available", url: attachment.url }; + } + return { + status: "unavailable", + reason: t("chat.attachments.unavailable"), + checkedAt: Date.now(), + }; + } + const sessionKey = resolveManagedOutgoingMediaSessionKey(attachment.url); + if (!sessionKey) { + return { + status: "unavailable", + reason: t("chat.attachments.unavailable"), + checkedAt: Date.now(), + }; + } + const cacheKey = `${attachment.url}::${attachment.artifactId}`; + const resource = observeChatMediaResource( + "managed-media", + cacheKey, + onRequestUpdate, + attachment.url, + ); + const cached = resource.value; + const now = Date.now(); + if (cached?.status === "unavailable") { + setManagedAttachmentAvailability(resource, cached); + return cached; + } + if ( + cached?.status === "checking" && + cached.refreshAfter !== undefined && + cached.refreshAfter > now + ) { + setManagedAttachmentAvailability(resource, cached); + return cached; + } + if (cached?.status === "available") { + if ( + cached.expiresAt !== undefined && + cached.expiresAt <= now && + (cached.refreshAttempts ?? 0) >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES + ) { + resource.retryAttempted = true; + const unavailable: ManagedAttachmentAvailability = { + status: "unavailable", + reason: t("chat.attachments.unavailable"), + checkedAt: now, + }; + setManagedAttachmentAvailability(resource, unavailable); + return unavailable; + } + if ( + cached.expiresAt !== undefined && + cached.expiresAt <= now && + (resource.pending || (cached.refreshAfter !== undefined && cached.refreshAfter > now)) + ) { + const checking: ManagedAttachmentAvailability = { + status: "checking", + ...(!resource.pending && cached.refreshAfter !== undefined + ? { refreshAfter: cached.refreshAfter } + : {}), + refreshAttempts: cached.refreshAttempts, + }; + setManagedAttachmentAvailability(resource, checking); + return checking; + } + const refreshAt = + cached.refreshAfter ?? + (cached.expiresAt === undefined + ? undefined + : cached.expiresAt - ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS); + if (refreshAt === undefined || refreshAt > now) { + setManagedAttachmentAvailability(resource, cached); + return cached; + } + } + if (resource.pending) { + return cached?.status === "available" ? cached : { status: "checking" }; + } + const current = + cached?.status === "available" && (cached.expiresAt === undefined || cached.expiresAt > now) + ? cached + : null; + const keepCurrentForRetry = () => { + if (!current && cached?.status !== "checking") { + return null; + } + const refreshAttempts = current?.refreshAttempts ?? cached?.refreshAttempts ?? 0; + if (refreshAttempts >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES) { + return retainManagedAttachmentUntilExpiry(resource, current, refreshAttempts); + } + const nextRefreshAttempts = refreshAttempts + 1; + const refreshAfter = Date.now() + managedAttachmentRefreshDelayMs(nextRefreshAttempts); + const retryAvailability: ManagedAttachmentAvailability = + !current || (current.expiresAt !== undefined && current.expiresAt <= Date.now()) + ? { status: "checking", refreshAfter, refreshAttempts: nextRefreshAttempts } + : { ...current, refreshAfter, refreshAttempts: nextRefreshAttempts }; + setManagedAttachmentAvailability(resource, retryAvailability); + return retryAvailability; + }; + if (!current) { + setManagedAttachmentAvailability(resource, { status: "checking" }); + } + const pending = Promise.resolve() + .then(() => resolveArtifactDownload({ sessionKey, artifactId: attachment.artifactId! })) + .then((result) => { + if (!isChatMediaResourceCurrent(resource)) { + return null; + } + const url = result?.url.trim(); + if (!url) { + const retryAvailability = keepCurrentForRetry(); + if (retryAvailability) { + return retryAvailability; + } + if ( + (cached?.refreshAttempts ?? 0) >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES + ) { + resource.retryAttempted = true; + } + const unavailable: ManagedAttachmentAvailability = { + status: "unavailable", + reason: t("chat.attachments.unavailable"), + checkedAt: Date.now(), + }; + setManagedAttachmentAvailability(resource, unavailable); + return unavailable; + } + const parsedExpiresAt = Date.parse(result?.expiresAt ?? ""); + const expiresAt = Number.isFinite(parsedExpiresAt) + ? parsedExpiresAt + : Date.now() + 5 * 60_000; + const refreshAttempts = cached?.refreshAttempts ?? 0; + if ( + expiresAt - Date.now() <= ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS && + refreshAttempts >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES + ) { + const incoming: Extract = { + status: "available", + url, + expiresAt, + }; + const retained = retainManagedAttachmentUntilExpiry( + resource, + selectLaterExpiringManagedAttachment(current, incoming), + refreshAttempts, + ); + if (retained) { + return retained; + } + resource.retryAttempted = true; + const unavailable: ManagedAttachmentAvailability = { + status: "unavailable", + reason: t("chat.attachments.unavailable"), + checkedAt: Date.now(), + }; + setManagedAttachmentAvailability(resource, unavailable); + return unavailable; + } + const nextRefreshAttempts = refreshAttempts + 1; + const needsEarlyRefresh = + expiresAt - Date.now() <= ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS; + if (expiresAt <= Date.now()) { + const retryAvailability: ManagedAttachmentAvailability = { + status: "checking", + refreshAfter: Date.now() + managedAttachmentRefreshDelayMs(nextRefreshAttempts), + refreshAttempts: nextRefreshAttempts, + }; + setManagedAttachmentAvailability(resource, retryAvailability); + return retryAvailability; + } + const availability: ManagedAttachmentAvailability = { + status: "available", + url, + expiresAt, + ...(needsEarlyRefresh + ? { + refreshAfter: Date.now() + managedAttachmentRefreshDelayMs(nextRefreshAttempts), + refreshAttempts: nextRefreshAttempts, + } + : {}), + }; + if (!needsEarlyRefresh) { + resource.retryAttempted = false; + } + setManagedAttachmentAvailability(resource, availability); + return availability; + }) + .catch(() => { + const retryAvailability = keepCurrentForRetry(); + if (retryAvailability) { + return retryAvailability; + } + if ((cached?.refreshAttempts ?? 0) >= ASSISTANT_ATTACHMENT_MEDIA_TICKET_MAX_REFRESH_RETRIES) { + resource.retryAttempted = true; + } + const unavailable: ManagedAttachmentAvailability = { + status: "unavailable", + reason: t("chat.attachments.unavailable"), + checkedAt: Date.now(), + }; + setManagedAttachmentAvailability(resource, unavailable); + return unavailable; + }) + .finally(() => { + if (resource.pending === pending) { + resource.pending = undefined; + } + notifyChatMediaResourceSubscribers(resource); + }); + resource.pending = pending; + if (current) { + setManagedAttachmentAvailability(resource, current, true); + } + return current ?? { status: "checking" }; } export function renderAssistantAttachments( @@ -355,6 +524,7 @@ export function renderAssistantAttachments( onAssistantAttachmentLoaded?: () => void, onRequestOpenImage?: () => number, onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void, + resolveArtifactDownload?: ArtifactDownloadResolver, ) { if (attachments.length === 0) { return nothing; @@ -362,17 +532,56 @@ export function renderAssistantAttachments( return html`
${attachments.map(({ attachment }) => { - const availability = resolveAssistantAttachmentAvailability( + const assistantAvailability = resolveAssistantAttachmentAvailability( attachment.url, localMediaPreviewRoots, basePath, authToken, onRequestUpdate, ); - const attachmentUrl = - availability.status === "available" - ? buildAssistantAttachmentUrl(attachment.url, basePath, availability.mediaTicket) + const managedAvailability = + assistantAvailability.status === "available" + ? resolveManagedAttachmentAvailability( + attachment, + resolveArtifactDownload, + onRequestUpdate, + ) : null; + const availability = + assistantAvailability.status !== "available" + ? assistantAvailability + : managedAvailability?.status === "unavailable" + ? managedAvailability + : managedAvailability?.status === "checking" + ? managedAvailability + : assistantAvailability; + const attachmentUrl = + assistantAvailability.status === "available" && + managedAvailability?.status === "available" + ? isLocalAssistantAttachmentSource(attachment.url) + ? buildAssistantAttachmentUrl( + attachment.url, + basePath, + assistantAvailability.mediaTicket, + ) + : managedAvailability.url + : null; + const playback = + assistantAvailability.status === "available" + ? (assistantAvailability.playback ?? attachment.playback ?? "native") + : (attachment.playback ?? "native"); + const sizeBytes = + assistantAvailability.status === "available" + ? (assistantAvailability.sizeBytes ?? attachment.sizeBytes) + : attachment.sizeBytes; + const serverDurationMs = + isLocalAssistantAttachmentSource(attachment.url) && + assistantAvailability.status === "available" + ? assistantAvailability.durationMs + : undefined; + const playbackAuthToken = isLocalAssistantAttachmentSource(attachment.url) + ? (authToken ?? null) + : null; if (attachment.kind === "image") { if (!attachmentUrl) { return renderAssistantAttachmentStatusCard({ @@ -421,6 +630,10 @@ export function renderAssistantAttachments( .src=${attachmentUrl} .sourceIdentity=${attachment.url} .label=${attachment.label} + .playback=${playback} + .authToken=${playbackAuthToken} + .sizeBytes=${sizeBytes} + .serverDurationMs=${serverDurationMs} .voiceNote=${attachment.isVoiceNote === true} .onMediaLoaded=${onAssistantAttachmentLoaded} > @@ -438,85 +651,21 @@ export function renderAssistantAttachments( reason: availability.status === "unavailable" ? availability.reason : undefined, }); } - const dimensions = - attachment.width && attachment.height - ? { "aspect-ratio": `${attachment.width} / ${attachment.height}` } - : {}; - const downloadHref = safeAttachmentHref(attachmentUrl); return html` -
-
- ${attachment.label} - ${downloadHref - ? html`${icons.download}` - : nothing} -
-
- - -
-
-
- ${t("chat.mediaPlayer.videoUnavailable")} -
- ${downloadHref - ? html`${t("chat.mediaPlayer.download", { filename: attachment.label })}` - : nothing} -
-
+ `; } if (!attachmentUrl) { diff --git a/ui/src/pages/chat/components/chat-message-bubble.ts b/ui/src/pages/chat/components/chat-message-bubble.ts index 8f9a870bf96a..062656914b72 100644 --- a/ui/src/pages/chat/components/chat-message-bubble.ts +++ b/ui/src/pages/chat/components/chat-message-bubble.ts @@ -422,6 +422,7 @@ export function renderGroupedMessage( opts.onAssistantAttachmentLoaded, opts.onRequestOpenImage, opts.onOpenImage, + opts.resolveArtifactDownload, )} ${assistantViewContent} ${reasoningMarkdown @@ -488,6 +489,7 @@ export function renderGroupedMessage( opts.onAssistantAttachmentLoaded, opts.onRequestOpenImage, opts.onOpenImage, + opts.resolveArtifactDownload, )} ${reasoningMarkdown ? html`
diff --git a/ui/src/pages/chat/components/chat-message-local-media.ts b/ui/src/pages/chat/components/chat-message-local-media.ts index 63783968cb04..2ee7884d509b 100644 --- a/ui/src/pages/chat/components/chat-message-local-media.ts +++ b/ui/src/pages/chat/components/chat-message-local-media.ts @@ -129,3 +129,22 @@ export function buildAssistantAttachmentUrl( } return `${normalizedBasePath}/__openclaw__/assistant-media?${params.toString()}`; } + +export function appendAttachmentUrlSearchParam( + source: string, + name: string, + value: string, +): string { + const trimmed = source.trim(); + if (!trimmed) { + return trimmed; + } + const hashIndex = trimmed.indexOf("#"); + const hash = hashIndex === -1 ? "" : trimmed.slice(hashIndex); + const withoutHash = hashIndex === -1 ? trimmed : trimmed.slice(0, hashIndex); + const queryIndex = withoutHash.indexOf("?"); + const path = queryIndex === -1 ? withoutHash : withoutHash.slice(0, queryIndex); + const params = new URLSearchParams(queryIndex === -1 ? "" : withoutHash.slice(queryIndex + 1)); + params.set(name, value); + return `${path}?${params.toString()}${hash}`; +} diff --git a/ui/src/pages/chat/components/chat-message-media.ts b/ui/src/pages/chat/components/chat-message-media.ts index a4877083a8b8..94f0eeb06eb9 100644 --- a/ui/src/pages/chat/components/chat-message-media.ts +++ b/ui/src/pages/chat/components/chat-message-media.ts @@ -42,7 +42,11 @@ export type RenderableImageBlock = ImageBlock & { export type AttachmentItem = Extract; -type ChatMediaResourceKind = "assistant-attachment" | "managed-image" | "pairing-qr"; +type ChatMediaResourceKind = + | "assistant-attachment" + | "managed-image" + | "managed-media" + | "pairing-qr"; export type ChatMediaResource = { kind: ChatMediaResourceKind; diff --git a/ui/src/pages/chat/components/chat-message.test.ts b/ui/src/pages/chat/components/chat-message.test.ts index 32c8b57cda6b..1b4b5deb0c44 100644 --- a/ui/src/pages/chat/components/chat-message.test.ts +++ b/ui/src/pages/chat/components/chat-message.test.ts @@ -595,6 +595,16 @@ async function requireAudioPlayer(container: HTMLElement) { return player; } +async function requireVideoPlayer(container: HTMLElement) { + const player = expectElement( + container, + "openclaw-chat-video-player", + HTMLElement, + ) as HTMLElement & { updateComplete: Promise }; + await player.updateComplete; + return player; +} + afterEach(() => { markdownRenderMock.mockClear(); document.querySelectorAll("[data-media-player-test-fixture]").forEach((element) => { @@ -3119,7 +3129,10 @@ describe("grouped chat rendering", () => { const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { expect(url).toContain("meta=1"); expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer session-token"); - return { ok: true, json: async () => mediaTicketPayload("ticket-bootstrap-audio") }; + return { + ok: true, + json: async () => ({ ...mediaTicketPayload("ticket-bootstrap-audio"), durationMs: 2_345 }), + }; }); vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); @@ -3153,6 +3166,373 @@ describe("grouped chat rendering", () => { expect(audio.getAttribute("src")).toBe( `/openclaw/__openclaw__/assistant-media?source=${encodeURIComponent(source)}&mediaTicket=ticket-bootstrap-audio`, ); + expect((audioPlayer as unknown as { serverDurationMs?: number }).serverDurationMs).toBe(2_345); + }); + + it("resolves managed transcode audio through an artifact ticket", async () => { + const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const ticketedUrl = `${source}?mediaTicket=managed-ticket`; + const artifactId = `artifact_managed_media_${crypto.randomUUID()}`; + const resolveArtifactDownload = vi.fn(async () => ({ url: ticketedUrl })); + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + expect(url).toBe(`${ticketedUrl}&playback=1`); + expect(init?.method).toBe("HEAD"); + expect(new Headers(init?.headers).get("Authorization")).toBeNull(); + return new Response(null, { status: 200 }); + }); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + const container = document.body.appendChild(document.createElement("div")); + container.dataset.mediaPlayerTestFixture = ""; + const rerender = () => + renderAssistantMessage( + container, + { + id: "assistant-managed-transcode-audio", + role: "assistant", + content: [ + { + type: "audio", + artifactId, + url: source, + fileName: "voice.caf", + mimeType: "audio/x-caf", + playback: "transcode", + sizeBytes: 4_096, + durationMs: 2_345, + }, + ], + timestamp: Date.now(), + }, + { + showToolCalls: false, + assistantAttachmentAuthToken: "must-not-be-forwarded", + onRequestUpdate: rerender, + resolveArtifactDownload, + }, + ); + + rerender(); + await flushAssistantAttachmentAvailabilityChecks(); + const player = await requireAudioPlayer(container); + await flushAssistantAttachmentAvailabilityChecks(); + await player.updateComplete; + + expect(resolveArtifactDownload).toHaveBeenCalledWith({ + sessionKey: "agent:main:main", + artifactId, + }); + expect(fetchMock).toHaveBeenCalledOnce(); + expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe( + `${ticketedUrl}&playback=1`, + ); + expect((player as unknown as { serverDurationMs?: number }).serverDurationMs).toBeUndefined(); + }); + + it("keeps a valid managed media ticket while refresh retries", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z")); + const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const ticketedUrl = `${source}?mediaTicket=managed-old`; + const artifactId = `artifact_managed_media_${crypto.randomUUID()}`; + const resolveArtifactDownload = vi + .fn<() => Promise<{ url: string; expiresAt: string }>>() + .mockResolvedValueOnce({ + url: ticketedUrl, + expiresAt: new Date(Date.now() + 31_000).toISOString(), + }) + .mockRejectedValueOnce(new Error("refresh unavailable")); + const container = document.body.appendChild(document.createElement("div")); + container.dataset.mediaPlayerTestFixture = ""; + const rerender = () => + renderAssistantMessage( + container, + { + id: "assistant-managed-ticket-refresh", + role: "assistant", + content: [ + { + type: "audio", + artifactId, + url: source, + fileName: "voice.mp3", + mimeType: "audio/mpeg", + playback: "native", + }, + ], + timestamp: Date.now(), + }, + { showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload }, + ); + + rerender(); + await flushAssistantAttachmentAvailabilityChecks(); + const player = await requireAudioPlayer(container); + expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe(ticketedUrl); + + await vi.advanceTimersByTimeAsync(1_001); + await flushAssistantAttachmentAvailabilityChecks(); + await player.updateComplete; + + expect(resolveArtifactDownload).toHaveBeenCalledTimes(2); + expect(container.querySelector(".chat-assistant-attachment-card--blocked")).toBeNull(); + expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe(ticketedUrl); + }); + + it("backs off stale managed ticket refreshes and eventually marks them unavailable", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z")); + const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const artifactId = `artifact_managed_media_${crypto.randomUUID()}`; + let finishSecondRequest: (() => void) | undefined; + const resolveArtifactDownload = vi.fn(() => { + const requestNumber = resolveArtifactDownload.mock.calls.length; + const result = { + url: `${source}?mediaTicket=stale-${requestNumber}`, + expiresAt: new Date(Date.now() + (requestNumber === 1 ? 6_000 : -1_000)).toISOString(), + }; + if (requestNumber !== 2) { + return Promise.resolve(result); + } + return new Promise((resolve) => { + finishSecondRequest = () => resolve(result); + }); + }); + const container = document.body.appendChild(document.createElement("div")); + container.dataset.mediaPlayerTestFixture = ""; + const rerender = () => + renderAssistantMessage( + container, + { + id: "assistant-managed-stale-ticket-refresh", + role: "assistant", + content: [ + { + type: "audio", + artifactId, + url: source, + fileName: "voice.mp3", + mimeType: "audio/mpeg", + playback: "native", + }, + ], + timestamp: Date.now(), + }, + { showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload }, + ); + + rerender(); + await flushAssistantAttachmentAvailabilityChecks(); + expect(resolveArtifactDownload).toHaveBeenCalledTimes(1); + expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull(); + + await vi.advanceTimersByTimeAsync(4_999); + expect(resolveArtifactDownload).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await flushAssistantAttachmentAvailabilityChecks(); + expect(resolveArtifactDownload).toHaveBeenCalledTimes(2); + expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull(); + + await vi.advanceTimersByTimeAsync(1_000); + expect(resolveArtifactDownload).toHaveBeenCalledTimes(2); + expect(container.querySelector("openclaw-chat-audio-player")).toBeNull(); + finishSecondRequest?.(); + await flushAssistantAttachmentAvailabilityChecks(); + await vi.advanceTimersByTimeAsync(9_999); + expect(resolveArtifactDownload).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(1); + await flushAssistantAttachmentAvailabilityChecks(); + + expect(resolveArtifactDownload).toHaveBeenCalledTimes(3); + expect(container.querySelector(".chat-assistant-attachment-card--blocked")).not.toBeNull(); + await vi.advanceTimersByTimeAsync(60_000); + expect(resolveArtifactDownload).toHaveBeenCalledTimes(3); + }); + + it("retains the current managed ticket until expiry after refresh exhaustion", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z")); + const expiresAt = new Date(Date.now() + 20_000).toISOString(); + const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const artifactId = `artifact_managed_media_${crypto.randomUUID()}`; + const resolveArtifactDownload = vi.fn(async () => ({ + url: `${source}?mediaTicket=short-${resolveArtifactDownload.mock.calls.length}`, + expiresAt, + })); + const container = document.body.appendChild(document.createElement("div")); + container.dataset.mediaPlayerTestFixture = ""; + const rerender = () => + renderAssistantMessage( + container, + { + id: "assistant-managed-ticket-exhaustion", + role: "assistant", + content: [ + { + type: "audio", + artifactId, + url: source, + fileName: "voice.mp3", + mimeType: "audio/mpeg", + playback: "native", + }, + ], + timestamp: Date.now(), + }, + { showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload }, + ); + + rerender(); + await flushAssistantAttachmentAvailabilityChecks(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAssistantAttachmentAvailabilityChecks(); + await vi.advanceTimersByTimeAsync(10_000); + await flushAssistantAttachmentAvailabilityChecks(); + + expect(resolveArtifactDownload).toHaveBeenCalledTimes(3); + expect( + container.querySelector("openclaw-chat-audio-player audio")?.getAttribute("src"), + ).toContain("mediaTicket=short-2"); + expect(container.querySelector(".chat-assistant-attachment-card--blocked")).toBeNull(); + + await vi.advanceTimersByTimeAsync(4_999); + expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull(); + await vi.advanceTimersByTimeAsync(1); + await flushAssistantAttachmentAvailabilityChecks(); + + expect(resolveArtifactDownload).toHaveBeenCalledTimes(3); + expect(container.querySelector(".chat-assistant-attachment-card--blocked")).not.toBeNull(); + }); + + it("retains a longer-lived incoming managed ticket after refresh exhaustion", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z")); + const initialExpiry = Date.now() + 20_000; + const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const artifactId = `artifact_managed_media_${crypto.randomUUID()}`; + const resolveArtifactDownload = vi.fn(async () => ({ + url: `${source}?mediaTicket=short-${resolveArtifactDownload.mock.calls.length}`, + expiresAt: new Date( + resolveArtifactDownload.mock.calls.length === 3 ? Date.now() + 20_000 : initialExpiry, + ).toISOString(), + })); + const container = document.body.appendChild(document.createElement("div")); + container.dataset.mediaPlayerTestFixture = ""; + const rerender = () => + renderAssistantMessage( + container, + { + id: "assistant-managed-later-ticket", + role: "assistant", + content: [ + { + type: "audio", + artifactId, + url: source, + fileName: "voice.mp3", + mimeType: "audio/mpeg", + playback: "native", + }, + ], + timestamp: Date.now(), + }, + { showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload }, + ); + + rerender(); + await flushAssistantAttachmentAvailabilityChecks(); + await vi.advanceTimersByTimeAsync(5_000); + await flushAssistantAttachmentAvailabilityChecks(); + await vi.advanceTimersByTimeAsync(10_000); + await flushAssistantAttachmentAvailabilityChecks(); + + expect(resolveArtifactDownload).toHaveBeenCalledTimes(3); + expect( + container.querySelector("openclaw-chat-audio-player audio")?.getAttribute("src"), + ).toContain("mediaTicket=short-3"); + + await vi.advanceTimersByTimeAsync(5_000); + await flushAssistantAttachmentAvailabilityChecks(); + expect(container.querySelector("openclaw-chat-audio-player")).not.toBeNull(); + await vi.advanceTimersByTimeAsync(15_000); + await flushAssistantAttachmentAvailabilityChecks(); + expect(container.querySelector(".chat-assistant-attachment-card--blocked")).not.toBeNull(); + }); + + it("refreshes a managed attachment that arrives with an initial ticket", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z")); + const rawSource = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const source = `${rawSource}?mediaTicket=initial`; + const refreshedSource = `${rawSource}?mediaTicket=refreshed`; + const artifactId = `artifact_managed_media_${crypto.randomUUID()}`; + const resolveArtifactDownload = vi.fn(async () => ({ url: refreshedSource })); + const container = document.body.appendChild(document.createElement("div")); + container.dataset.mediaPlayerTestFixture = ""; + const rerender = () => + renderAssistantMessage( + container, + { + id: "assistant-managed-initial-ticket", + role: "assistant", + content: [ + { + type: "audio", + artifactId, + url: source, + fileName: "voice.mp3", + mimeType: "audio/mpeg", + playback: "native", + }, + ], + timestamp: Date.now(), + }, + { showToolCalls: false, onRequestUpdate: rerender, resolveArtifactDownload }, + ); + + rerender(); + await flushAssistantAttachmentAvailabilityChecks(); + const player = await requireAudioPlayer(container); + + expect(resolveArtifactDownload).toHaveBeenCalledWith({ + sessionKey: "agent:main:main", + artifactId, + }); + expect(expectElement(player, "audio", HTMLAudioElement).getAttribute("src")).toBe( + refreshedSource, + ); + + await vi.advanceTimersByTimeAsync(4 * 60_000 + 30_001); + await flushAssistantAttachmentAvailabilityChecks(); + expect(resolveArtifactDownload).toHaveBeenCalledTimes(2); + }); + + it("does not render an unticketed managed attachment without an artifact resolver", () => { + const source = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const container = document.createElement("div"); + renderAssistantMessage( + container, + { + id: "assistant-managed-media-without-resolver", + role: "assistant", + content: [ + { + type: "audio", + artifactId: `artifact_managed_media_${crypto.randomUUID()}`, + url: source, + fileName: "voice.mp3", + mimeType: "audio/mpeg", + playback: "native", + }, + ], + timestamp: Date.now(), + }, + { showToolCalls: false }, + ); + + expect(container.querySelector("openclaw-chat-audio-player")).toBeNull(); + expect( + container.querySelector(".chat-assistant-attachment-card--blocked")?.textContent, + ).toContain("Unavailable"); }); it("checks local assistant images against server metadata while preview roots load", async () => { @@ -3249,7 +3629,7 @@ describe("grouped chat rendering", () => { }, ); - it("shows the download fallback when the video element emits an error", () => { + it("shows the download fallback when the video element emits an error", async () => { const container = document.body.appendChild(document.createElement("div")); container.dataset.mediaPlayerTestFixture = ""; @@ -3274,8 +3654,10 @@ describe("grouped chat rendering", () => { { showToolCalls: false }, ); - const card = expectElement(container, ".chat-assistant-attachment-card--video", HTMLElement); + const videoPlayer = await requireVideoPlayer(container); + const card = expectElement(videoPlayer, ".chat-assistant-attachment-card--video", HTMLElement); expectElement(card, "video", HTMLVideoElement).dispatchEvent(new Event("error")); + await videoPlayer.updateComplete; expect(card.hasAttribute("data-unplayable")).toBe(true); expect(card.querySelector(".chat-assistant-video-fallback")?.textContent).toContain( "Can't play this format — download instead.", @@ -3756,8 +4138,9 @@ describe("grouped chat rendering", () => { expect(container.querySelector(".chat-text")?.textContent?.trim()).toBe("Blocked\nDone"); }); - it("renders transcript video URLs with encoded extensions", () => { - const container = document.createElement("div"); + it("renders transcript video URLs with encoded extensions", async () => { + const container = document.body.appendChild(document.createElement("div")); + container.dataset.mediaPlayerTestFixture = ""; const mediaUrl = "https://cdn.example/clip%2Emp4?download=1"; renderGroupedMessage( @@ -3773,7 +4156,8 @@ describe("grouped chat rendering", () => { { showToolCalls: false }, ); - expect(expectElement(container, "video", HTMLVideoElement).src).toBe(mediaUrl); + const videoPlayer = await requireVideoPlayer(container); + expect(expectElement(videoPlayer, "video", HTMLVideoElement).src).toBe(mediaUrl); }); it("renders transcript image variants and structured image blocks", async () => { diff --git a/ui/src/pages/chat/components/chat-video-player.test.ts b/ui/src/pages/chat/components/chat-video-player.test.ts new file mode 100644 index 000000000000..024bad0bff68 --- /dev/null +++ b/ui/src/pages/chat/components/chat-video-player.test.ts @@ -0,0 +1,221 @@ +/* @vitest-environment jsdom */ + +import { afterEach, describe, expect, it, vi } from "vitest"; +import "./chat-video-player.ts"; + +afterEach(() => { + document.body.replaceChildren(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("ChatVideoPlayer", () => { + it("keeps one video element mounted across 202 preparation", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 202 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-video-player"); + player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=ticket"; + player.sourceIdentity = "media:clip"; + player.label = "clip.avi"; + player.playback = "transcode"; + document.body.append(player); + await player.updateComplete; + const video = player.querySelector("video"); + await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…")); + expect(player.querySelector("video")).toBe(video); + + await vi.advanceTimersByTimeAsync(2_000); + await player.updateComplete; + + expect(player.querySelector("video")).toBe(video); + expect(video?.getAttribute("src")).toContain("mediaTicket=ticket&playback=1"); + }); + + it("does not preserve a previous attachment when a new rendition fails", async () => { + const player = document.createElement("openclaw-chat-video-player"); + player.src = "https://example.com/first.mp4"; + player.sourceIdentity = "media:first"; + player.label = "first.mp4"; + document.body.append(player); + await player.updateComplete; + expect(player.querySelector("video")?.getAttribute("src")).toBe( + "https://example.com/first.mp4", + ); + + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(null, { status: 500 })), + ); + player.src = "/__openclaw__/assistant-media?source=second.caf&mediaTicket=ticket"; + player.sourceIdentity = "media:second"; + player.label = "second.caf"; + player.playback = "transcode"; + await player.updateComplete; + expect(player.querySelector("video")?.hasAttribute("src")).toBe(false); + await vi.waitFor(() => + expect( + player + .querySelector(".chat-assistant-attachment-card--video") + ?.hasAttribute("data-unplayable"), + ).toBe(true), + ); + + expect(player.querySelector(".chat-assistant-video-fallback")?.textContent).toContain( + "Can't play this format — download instead.", + ); + }); + + it("does not retain an errored source when a refreshed rendition is unavailable", async () => { + let resolveFirstRefresh: ((response: Response) => void) | undefined; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockImplementationOnce( + async () => + await new Promise((resolve) => { + resolveFirstRefresh = resolve; + }), + ) + .mockResolvedValueOnce(new Response(null, { status: 500 })); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-video-player"); + player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=old"; + player.sourceIdentity = "media:clip"; + player.label = "clip.avi"; + player.playback = "transcode"; + document.body.append(player); + await vi.waitFor(() => + expect(player.querySelector("video")?.getAttribute("src")).toContain("mediaTicket=old"), + ); + + player.querySelector("video")?.dispatchEvent(new Event("error")); + await player.updateComplete; + player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=refresh-1"; + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=refresh-2"; + + await vi.waitFor(() => + expect( + player + .querySelector(".chat-assistant-attachment-card--video") + ?.hasAttribute("data-unplayable"), + ).toBe(true), + ); + expect(fetchMock).toHaveBeenCalledTimes(3); + resolveFirstRefresh?.(new Response(null, { status: 200 })); + }); + + it("hides a previous attachment while the replacement HEAD is stalled", async () => { + const player = document.createElement("openclaw-chat-video-player"); + player.src = "https://example.com/first.mp4"; + player.sourceIdentity = "media:first-stalled"; + player.label = "first.mp4"; + document.body.append(player); + await player.updateComplete; + + const fetchMock = vi.fn(async () => await new Promise(() => {})); + vi.stubGlobal("fetch", fetchMock); + player.src = "/__openclaw__/assistant-media?source=second.caf&mediaTicket=ticket"; + player.sourceIdentity = "media:second-stalled"; + player.label = "second.caf"; + player.playback = "transcode"; + await player.updateComplete; + + expect(fetchMock).toHaveBeenCalledOnce(); + await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…")); + expect(player.querySelector("video")?.hasAttribute("src")).toBe(false); + }); + + it("clears authorized video while a new principal is checked", async () => { + let resolveRefresh: ((response: Response) => void) | undefined; + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockImplementationOnce( + async () => + await new Promise((resolve) => { + resolveRefresh = resolve; + }), + ); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-video-player"); + player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=ticket"; + player.sourceIdentity = "media:principal-clip"; + player.authToken = "principal-a"; + player.label = "clip.avi"; + player.playback = "transcode"; + document.body.append(player); + await vi.waitFor(() => + expect(player.querySelector("video")?.getAttribute("src")).toContain("playback=1"), + ); + + player.authToken = "principal-b"; + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + await vi.waitFor(() => expect(player.textContent).toContain("Preparing playback…")); + expect(player.querySelector("video")?.hasAttribute("src")).toBe(false); + resolveRefresh?.(new Response(null, { status: 500 })); + }); + + it("clears a failed rendition after reconnect succeeds", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 500 })) + .mockResolvedValueOnce(new Response(null, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const player = document.createElement("openclaw-chat-video-player"); + player.src = "/__openclaw__/assistant-media?source=clip.avi&mediaTicket=ticket"; + player.sourceIdentity = "media:retry-clip"; + player.label = "clip.avi"; + player.playback = "transcode"; + document.body.append(player); + await vi.waitFor(() => + expect( + player + .querySelector(".chat-assistant-attachment-card--video") + ?.hasAttribute("data-unplayable"), + ).toBe(true), + ); + + player.remove(); + document.body.append(player); + await vi.waitFor(() => + expect(player.querySelector("video")?.getAttribute("src")).toContain("playback=1"), + ); + expect( + player + .querySelector(".chat-assistant-attachment-card--video") + ?.hasAttribute("data-unplayable"), + ).toBe(false); + }); + + it("pauses and clears the source when disconnected while playing", async () => { + const player = document.createElement("openclaw-chat-video-player"); + player.src = "https://example.com/playing.mp4"; + player.sourceIdentity = "media:playing"; + player.label = "playing.mp4"; + document.body.append(player); + await player.updateComplete; + const video = player.querySelector("video")!; + let paused = false; + Object.defineProperty(video, "paused", { configurable: true, get: () => paused }); + const pause = vi.spyOn(video, "pause").mockImplementation(() => { + paused = true; + }); + + player.remove(); + + expect(pause).toHaveBeenCalledOnce(); + expect(paused).toBe(true); + expect(video.hasAttribute("src")).toBe(false); + + document.body.append(player); + await vi.waitFor(() => + expect(video.getAttribute("src")).toBe("https://example.com/playing.mp4"), + ); + }); +}); diff --git a/ui/src/pages/chat/components/chat-video-player.ts b/ui/src/pages/chat/components/chat-video-player.ts new file mode 100644 index 000000000000..33d213fc7438 --- /dev/null +++ b/ui/src/pages/chat/components/chat-video-player.ts @@ -0,0 +1,264 @@ +import { html, type PropertyValues } from "lit"; +import { property, state } from "lit/decorators.js"; +import { ref } from "lit/directives/ref.js"; +import { styleMap } from "lit/directives/style-map.js"; +import { icons } from "../../../components/icons.ts"; +import { t } from "../../../i18n/index.ts"; +import { OpenClawLightDomContentsElement } from "../../../lit/openclaw-element.ts"; +import { safeAttachmentHref } from "./chat-attachment-href.ts"; +import { + appendChatMediaPlaybackParam, + waitForChatMediaPlayback, + type ChatMediaPlaybackMode, +} from "./chat-media-playback.ts"; +import { ChatMediaSourceController } from "./chat-media-source.ts"; + +class ChatVideoPlayer extends OpenClawLightDomContentsElement { + @property() src = ""; + @property() sourceIdentity = ""; + @property() label = ""; + @property() playback: ChatMediaPlaybackMode = "native"; + @property() authToken: string | null = null; + @property({ type: Number }) mediaWidth: number | undefined; + @property({ type: Number }) mediaHeight: number | undefined; + @property({ attribute: false }) onMediaLoaded: (() => void) | undefined; + + @state() private metadataLoaded = false; + @state() private failed = false; + @state() private preparing = false; + + private media: HTMLVideoElement | null = null; + private readonly sourceController = new ChatMediaSourceController(); + private currentSourceFailed = false; + private readinessController: AbortController | null = null; + private readinessKey = ""; + private readySource = ""; + + override connectedCallback(): void { + super.connectedCallback(); + queueMicrotask(() => this.syncSource()); + } + + override disconnectedCallback(): void { + this.readinessController?.abort(); + this.readinessController = null; + this.readinessKey = ""; + this.readySource = ""; + this.sourceController.cancelPendingResume(); + if (this.media) { + if (!this.media.paused) { + this.media.pause(); + } + this.sourceController.reset(this.media); + } + super.disconnectedCallback(); + } + + override updated(changedProperties: PropertyValues): void { + if ( + changedProperties.has("src") || + changedProperties.has("sourceIdentity") || + changedProperties.has("playback") || + changedProperties.has("authToken") + ) { + const authenticationChanged = changedProperties.has("authToken"); + if ( + authenticationChanged || + (changedProperties.has("sourceIdentity") && + this.sourceController.currentIdentity && + this.sourceController.currentIdentity !== this.sourceIdentity.trim()) + ) { + this.metadataLoaded = false; + if (this.media && !this.media.paused) { + this.media.pause(); + } + if (this.media) { + this.sourceController.reset(this.media); + this.currentSourceFailed = false; + } + } + this.syncSource(); + } + } + + private setMedia = (element: Element | undefined) => { + this.media = element instanceof HTMLVideoElement ? element : null; + this.syncSource(); + }; + + private syncSource(): void { + const media = this.media; + const source = this.src.trim(); + if (!media || !source || !this.sourceIdentity.trim() || !this.isConnected) { + return; + } + const playbackSource = + this.playback === "transcode" ? appendChatMediaPlaybackParam(source) : source; + const hasCurrentAttachmentSource = + this.sourceController.currentIdentity === this.sourceIdentity.trim(); + const hasUsableCurrentAttachmentSource = + hasCurrentAttachmentSource && !this.currentSourceFailed; + if (this.playback !== "transcode") { + this.readinessController?.abort(); + this.readinessController = null; + this.readinessKey = ""; + this.readySource = playbackSource; + this.preparing = false; + this.failed = false; + this.currentSourceFailed = false; + this.sourceController.updateSource(media, playbackSource, this.sourceIdentity); + return; + } + + const readinessKey = `${playbackSource}\0${this.authToken?.trim() ?? ""}`; + if (!hasUsableCurrentAttachmentSource) { + this.metadataLoaded = false; + this.preparing = true; + } + if (readinessKey === this.readinessKey) { + if (this.readySource === playbackSource) { + this.sourceController.updateSource(media, playbackSource, this.sourceIdentity); + return; + } + if (this.readinessController) { + return; + } + } + this.readinessController?.abort(); + const controller = new AbortController(); + this.readinessController = controller; + this.readinessKey = readinessKey; + this.readySource = ""; + this.preparing = !hasUsableCurrentAttachmentSource; + this.failed = false; + void waitForChatMediaPlayback({ + source: playbackSource, + authToken: this.authToken, + signal: controller.signal, + onPreparing: () => { + if (this.readinessController === controller && !hasUsableCurrentAttachmentSource) { + this.preparing = true; + } + }, + }).then((result) => { + if (this.readinessController !== controller || result === "aborted") { + return; + } + this.preparing = false; + if (result !== "ready") { + if (hasUsableCurrentAttachmentSource) { + this.readySource = this.sourceController.currentSource; + return; + } + this.failed = true; + return; + } + this.readySource = playbackSource; + this.failed = false; + this.currentSourceFailed = false; + this.sourceController.updateSource(media, playbackSource, this.sourceIdentity); + }); + } + + override render() { + const downloadHref = safeAttachmentHref(this.src); + const dimensions = + this.mediaWidth && this.mediaHeight + ? { "aspect-ratio": `${this.mediaWidth} / ${this.mediaHeight}` } + : {}; + return html` +
+
+ ${this.label} + ${downloadHref + ? html`${icons.download}` + : null} +
+ ${this.preparing + ? html`
+ ${t("chat.mediaPlayer.preparing")} +
` + : null} +
+ + +
+
+
+ ${t("chat.mediaPlayer.videoUnavailable")} +
+ ${downloadHref + ? html`${t("chat.mediaPlayer.download", { filename: this.label })}` + : null} +
+
+ `; + } +} + +if (!customElements.get("openclaw-chat-video-player")) { + customElements.define("openclaw-chat-video-player", ChatVideoPlayer); +} + +declare global { + interface HTMLElementTagNameMap { + "openclaw-chat-video-player": ChatVideoPlayer; + } +} diff --git a/ui/src/styles/chat/layout.css b/ui/src/styles/chat/layout.css index 66118f20d688..33b61baa8274 100644 --- a/ui/src/styles/chat/layout.css +++ b/ui/src/styles/chat/layout.css @@ -1079,6 +1079,11 @@ openclaw-chat-page { display: block; } +openclaw-chat-audio-player, +openclaw-chat-video-player { + display: block; +} + .chat-assistant-attachment-card { display: flex; align-items: center; @@ -1264,6 +1269,39 @@ openclaw-chat-page { outline-offset: 4px; } +.chat-audio-player__waveform { + position: relative; + height: 24px; +} + +.chat-audio-player__waveform svg { + display: block; + width: 100%; + height: 24px; + overflow: visible; +} + +.chat-audio-player__waveform rect { + fill: color-mix(in srgb, var(--muted) 55%, transparent); +} + +.chat-audio-player__waveform rect.is-played { + fill: var(--accent); +} + +.chat-audio-player__seek--waveform { + position: absolute; + inset: 0; + height: 24px; + opacity: 0; +} + +.chat-audio-player__waveform:has(.chat-audio-player__seek:focus-visible) { + border-radius: 4px; + outline: 2px solid var(--focus-ring); + outline-offset: 2px; +} + .chat-audio-player__time { display: flex; justify-content: space-between; @@ -1290,6 +1328,10 @@ openclaw-chat-page { border: 1px solid color-mix(in srgb, var(--border) 82%, transparent); } +.chat-assistant-video-frame[hidden] { + display: none; +} + .chat-assistant-video-frame video { position: relative; z-index: 1; @@ -1351,6 +1393,13 @@ openclaw-chat-page { line-height: 1.4; } +.chat-media-preparing { + padding: 10px 12px; + border-radius: var(--radius-sm); + background: color-mix(in srgb, var(--bg) 78%, transparent); + border: 1px solid color-mix(in srgb, var(--border) 82%, transparent); +} + .chat-assistant-attachment-card__icon { display: inline-flex; width: 16px;