From 180a960ede9e5fb30055e00a45110081e152374e Mon Sep 17 00:00:00 2001 From: Andy Ye <35905412+TurboTheTurtle@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:04:06 -0700 Subject: [PATCH] fix(webchat): keep media loads pinned to bottom (#97575) * fix(webchat): keep media loads pinned to bottom * test(webchat): type message group mock options --- ui/src/ui/app-render.ts | 1 + ui/src/ui/app-view-state.ts | 1 + ui/src/ui/app.ts | 4 +++ ui/src/ui/chat/grouped-render.test.ts | 26 ++++++++++++++ ui/src/ui/chat/grouped-render.ts | 20 +++++++++-- ui/src/ui/views/chat.test.ts | 51 ++++++++++++++++++--------- ui/src/ui/views/chat.ts | 2 ++ 7 files changed, 87 insertions(+), 18 deletions(-) diff --git a/ui/src/ui/app-render.ts b/ui/src/ui/app-render.ts index 25203b534892..1ef7ece49993 100644 --- a/ui/src/ui/app-render.ts +++ b/ui/src/ui/app-render.ts @@ -3902,6 +3902,7 @@ export function renderApp(state: AppViewState) { }, showNewMessages: state.chatNewMessagesBelow && !state.chatManualRefreshInFlight, onScrollToBottom: () => state.scrollToBottom(), + onAssistantAttachmentLoaded: () => state.scheduleChatScroll(), // Sidebar props for tool output viewing sidebarOpen: state.sidebarOpen, sidebarContent: state.sidebarContent, diff --git a/ui/src/ui/app-view-state.ts b/ui/src/ui/app-view-state.ts index 72e1ad42dd99..c397c2354d38 100644 --- a/ui/src/ui/app-view-state.ts +++ b/ui/src/ui/app-view-state.ts @@ -182,6 +182,7 @@ export type AppViewState = { sidebarError: string | null; splitRatio: number; scrollToBottom: (opts?: { smooth?: boolean }) => void; + scheduleChatScroll: () => void; devicesLoading: boolean; devicesError: string | null; devicesList: DevicePairingList | null; diff --git a/ui/src/ui/app.ts b/ui/src/ui/app.ts index b7309a38924b..cdff5e8cdc73 100644 --- a/ui/src/ui/app.ts +++ b/ui/src/ui/app.ts @@ -985,6 +985,10 @@ export class OpenClawApp extends LitElement { ); } + scheduleChatScroll() { + scheduleChatScrollInternal(this as unknown as Parameters[0]); + } + async loadAssistantIdentity(opts?: { sessionKey?: string; expectedSessionKey?: string }) { await loadAssistantIdentityInternal(this, opts); } diff --git a/ui/src/ui/chat/grouped-render.test.ts b/ui/src/ui/chat/grouped-render.test.ts index 2b3421ec2606..d566dcc311ee 100644 --- a/ui/src/ui/chat/grouped-render.test.ts +++ b/ui/src/ui/chat/grouped-render.test.ts @@ -1461,6 +1461,32 @@ describe("grouped chat rendering", () => { ); }); + it("notifies when assistant audio and video attachment metadata loads", () => { + const container = document.createElement("div"); + const onAssistantAttachmentLoaded = vi.fn(); + + renderAssistantMessage( + container, + { + id: "assistant-media-layout", + role: "assistant", + content: + "Audio and video\nMEDIA:https://example.com/voice.ogg\nMEDIA:https://example.com/clip.mp4", + timestamp: Date.now(), + }, + { showToolCalls: false, onAssistantAttachmentLoaded }, + ); + + expectElement(container, "audio", HTMLAudioElement).dispatchEvent( + new Event("loadedmetadata", { bubbles: true }), + ); + expectElement(container, "video", HTMLVideoElement).dispatchEvent( + new Event("loadedmetadata", { bubbles: true }), + ); + + expect(onAssistantAttachmentLoaded).toHaveBeenCalledTimes(2); + }); + it("renders allowed transcript and content image variants", async () => { resetAssistantAttachmentAvailabilityCacheForTest(); const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { diff --git a/ui/src/ui/chat/grouped-render.ts b/ui/src/ui/chat/grouped-render.ts index bc83103bacf6..5aad7c884f1a 100644 --- a/ui/src/ui/chat/grouped-render.ts +++ b/ui/src/ui/chat/grouped-render.ts @@ -421,6 +421,7 @@ type RenderMessageGroupOptions = { isToolExpanded?: (toolCardId: string) => boolean; onToggleToolExpanded?: (toolCardId: string) => void; onRequestUpdate?: () => void; + onAssistantAttachmentLoaded?: () => void; assistantName?: string; assistantAvatar?: string | null; userName?: string | null; @@ -456,6 +457,7 @@ function buildGroupedMessageRenderOptions( isToolExpanded: opts.isToolExpanded, onToggleToolExpanded: opts.onToggleToolExpanded, onRequestUpdate: opts.onRequestUpdate, + onAssistantAttachmentLoaded: opts.onAssistantAttachmentLoaded, canvasPluginSurfaceUrl: opts.canvasPluginSurfaceUrl, basePath: opts.basePath, localMediaPreviewRoots: opts.localMediaPreviewRoots, @@ -1373,6 +1375,7 @@ function renderAssistantAttachments( basePath?: string, authToken?: string | null, onRequestUpdate?: () => void, + onAssistantAttachmentLoaded?: () => void, ) { if (attachments.length === 0) { return nothing; @@ -1424,7 +1427,12 @@ function renderAssistantAttachments( : nothing} ${attachmentUrl - ? html`` + ? html`` : availability.status === "unavailable" ? html`
${availability.reason} @@ -1444,7 +1452,12 @@ function renderAssistantAttachments( } return html`
- + void; embedSandboxMode?: EmbedSandboxMode; allowExternalEmbedUrls?: boolean; }, @@ -1812,6 +1826,7 @@ function renderGroupedMessage( opts.basePath, opts.assistantAttachmentAuthToken, opts.onRequestUpdate, + opts.onAssistantAttachmentLoaded, )} ${reasoningMarkdown ? html`
@@ -1869,6 +1884,7 @@ function renderGroupedMessage( opts.basePath, opts.assistantAttachmentAuthToken, opts.onRequestUpdate, + opts.onAssistantAttachmentLoaded, )} ${reasoningMarkdown ? html`
diff --git a/ui/src/ui/views/chat.test.ts b/ui/src/ui/views/chat.test.ts index db45539610ea..3fe4a5b08537 100644 --- a/ui/src/ui/views/chat.test.ts +++ b/ui/src/ui/views/chat.test.ts @@ -102,23 +102,28 @@ const buildChatItemsMock = vi.hoisted(() => }), ); const renderMessageGroupMock = vi.hoisted(() => - vi.fn((group: { messages: Array<{ message: unknown }> }) => { - const element = document.createElement("div"); - element.className = "chat-group"; - element.textContent = group.messages - .map(({ message }) => { - if (typeof message === "object" && message !== null && "content" in message) { - const content = (message as { content?: unknown }).content; - if (typeof content === "string") { - return content; + vi.fn( + ( + group: { messages: Array<{ message: unknown }> }, + _opts?: { onAssistantAttachmentLoaded?: () => void }, + ) => { + const element = document.createElement("div"); + element.className = "chat-group"; + element.textContent = group.messages + .map(({ message }) => { + if (typeof message === "object" && message !== null && "content" in message) { + const content = (message as { content?: unknown }).content; + if (typeof content === "string") { + return content; + } + return content == null ? "" : JSON.stringify(content); } - return content == null ? "" : JSON.stringify(content); - } - return String(message); - }) - .join("\n"); - return element; - }), + return String(message); + }) + .join("\n"); + return element; + }, + ), ); const assistantAttachmentRenderVersionMock = vi.hoisted(() => ({ value: 0 })); @@ -1145,6 +1150,20 @@ describe("chat transcript rendering cache", () => { expect(renderMessageGroupMock).toHaveBeenCalledTimes(2); }); + it("passes assistant attachment load callbacks to transcript groups", () => { + const onAssistantAttachmentLoaded = vi.fn(); + + renderChatView({ + messages: [{ role: "assistant", content: "MEDIA:https://example.com/voice.ogg" }], + onAssistantAttachmentLoaded, + }); + + expect(renderMessageGroupMock).toHaveBeenCalledTimes(1); + expect(renderMessageGroupMock.mock.calls[0]?.[1]).toMatchObject({ + onAssistantAttachmentLoaded, + }); + }); + it("rebuilds transcript items when the transcript reference changes", () => { const toolMessages: unknown[] = []; const streamSegments: Array<{ text: string; ts: number }> = []; diff --git a/ui/src/ui/views/chat.ts b/ui/src/ui/views/chat.ts index 83fc59708efe..71dc1af9a4a5 100644 --- a/ui/src/ui/views/chat.ts +++ b/ui/src/ui/views/chat.ts @@ -163,6 +163,7 @@ export type ChatProps = { onAttachmentsChange?: (attachments: ChatAttachment[]) => void; showNewMessages?: boolean; onScrollToBottom?: () => void; + onAssistantAttachmentLoaded?: () => void; onRefresh: () => void; onToggleFocusMode?: () => void; getDraft?: () => string; @@ -2282,6 +2283,7 @@ export function renderChat(props: ChatProps) { expandedToolCards.get(toolCardId) ?? false, onToggleToolExpanded: toggleToolCardExpanded, onRequestUpdate: requestUpdate, + onAssistantAttachmentLoaded: props.onAssistantAttachmentLoaded, assistantName: props.assistantName, assistantAvatar: assistantIdentity.avatar, userName: props.userName ?? null,