From 62ed43dfd08851473ce363587e9e6506cb21bdd4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 20 Aug 2026 10:05:21 -0700 Subject: [PATCH] fix(ui): keep ad hoc visualizations inline and readable (#126729) * fix(ui): keep ad hoc visualizations readable * perf(ui): preserve startup bundle budget --- src/agents/tools/dashboard-tool.test.ts | 8 +- src/agents/tools/dashboard-tool.ts | 2 +- src/canvas/widget-tool.prompt.test.ts | 20 ++ src/canvas/widget-tool.test.ts | 9 +- src/canvas/widget-tool.ts | 7 +- .../chat-worked-for-visualization.e2e.test.ts | 181 ++++++++++++++++++ ui/src/pages/chat/chat-thread-grouping.ts | 16 +- ui/src/pages/chat/chat-thread.test.ts | 44 +++++ .../components/chat-transcript-projection.ts | 37 ++-- 9 files changed, 288 insertions(+), 36 deletions(-) create mode 100644 src/canvas/widget-tool.prompt.test.ts create mode 100644 ui/src/e2e/chat-worked-for-visualization.e2e.test.ts diff --git a/src/agents/tools/dashboard-tool.test.ts b/src/agents/tools/dashboard-tool.test.ts index 7985f19aced7..fdce6858dcad 100644 --- a/src/agents/tools/dashboard-tool.test.ts +++ b/src/agents/tools/dashboard-tool.test.ts @@ -34,11 +34,17 @@ function recorder() { } describe("dashboard tool", () => { - it("declares every action, no client capability guard, and stable-name/size guidance", () => { + it("declares every action, no client capability guard, sizing, and the dashboard threshold", () => { const tool = createDashboardTool(); + const directoryDescription = tool.description.slice(0, 177); expect(tool.requiredClientCaps).toBeUndefined(); expect(tool.description).toContain("stable names"); expect(tool.description).toContain("sm=3x3"); + expect(directoryDescription).toMatch( + /(?:single|one[- ]off|ad hoc).{0,40}visualizations?.{0,40}inline/i, + ); + expect(directoryDescription).toContain("explicit dashboard request"); + expect(directoryDescription).toContain("multiple non-code visualizations"); expect(tool.parameters).toMatchObject({ additionalProperties: false, properties: { diff --git a/src/agents/tools/dashboard-tool.ts b/src/agents/tools/dashboard-tool.ts index a9d1bc916678..d80ee0f09e7d 100644 --- a/src/agents/tools/dashboard-tool.ts +++ b/src/agents/tools/dashboard-tool.ts @@ -267,7 +267,7 @@ export function createDashboardTool(opts: DashboardToolOptions = {}): AnyAgentTo label: "Dashboard", name: "dashboard", description: - "Read and arrange this session dashboard: read snapshot; tab_create/tab_update/tab_delete/tabs_reorder; widget_put/widget_move/widget_resize/widget_remove; focus_tab; set_chat_dock moves or hides the chat dock (left/right/bottom/hidden). focus_tab and set_chat_dock require a connected Control UI. Widgets use stable names. Create trusted plugin widgets with widget_put; examples: session:progress props {sessionKey?} renders the session's live progress card (omit sessionKey for the current session), workboard:card props {cardId}, workboard:mini props {boardId, limit}, workboard:board props {boardId}. Sizes: sm=3x3, md=6x4, lg=8x6, xl=12x8, full=12x8 single-widget emphasis.", + "Keep one ad hoc non-code visualization inline. Use this tool only for an explicit dashboard request or multiple non-code visualizations. Read and arrange this session dashboard: read snapshot; tab_create/tab_update/tab_delete/tabs_reorder; widget_put/widget_move/widget_resize/widget_remove; focus_tab; set_chat_dock moves or hides the chat dock (left/right/bottom/hidden). focus_tab and set_chat_dock require a connected Control UI. Widgets use stable names. Create trusted plugin widgets with widget_put; examples: session:progress props {sessionKey?} renders the session's live progress card (omit sessionKey for the current session), workboard:card props {cardId}, workboard:mini props {boardId, limit}, workboard:board props {boardId}. Sizes: sm=3x3, md=6x4, lg=8x6, xl=12x8, full=12x8 single-widget emphasis.", parameters: DashboardToolSchema, execute: async (_toolCallId, rawArgs) => { const params = rawArgs as Record; diff --git a/src/canvas/widget-tool.prompt.test.ts b/src/canvas/widget-tool.prompt.test.ts new file mode 100644 index 000000000000..6155c9794aa8 --- /dev/null +++ b/src/canvas/widget-tool.prompt.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { createShowWidgetTool } from "./widget-tool.js"; + +describe("show_widget prompt", () => { + it("keeps proactive single visualizations inline unless dashboard use meets its threshold", () => { + const tool = createShowWidgetTool(); + const directoryDescription = tool.description.slice(0, 177); + const pinDescription = (tool.parameters as { properties?: { pin?: { description?: string } } }) + .properties?.pin?.description; + + expect(directoryDescription).toMatch(/^Visual helps\? Make widget\. Do not wait for ask\./); + expect(directoryDescription).toMatch( + /(?:single|one[- ]off|ad hoc).{0,40}visualizations?.{0,40}inline/i, + ); + expect(directoryDescription).toContain("explicit dashboard request"); + expect(directoryDescription).toContain("multiple non-code visualizations"); + expect(pinDescription).toContain("explicit dashboard request"); + expect(pinDescription).toContain("multiple non-code visualizations"); + }); +}); diff --git a/src/canvas/widget-tool.test.ts b/src/canvas/widget-tool.test.ts index 0161c55092f9..7b645f5ccfb9 100644 --- a/src/canvas/widget-tool.test.ts +++ b/src/canvas/widget-tool.test.ts @@ -210,6 +210,9 @@ describe("show_widget", () => { callGateway, }); + expect(tool.description).toContain( + "Inline hosting is disabled; set pin=true to place it on this session's dashboard", + ); await expect( tool.execute("unpinned", { title: "Diagram", @@ -242,12 +245,6 @@ describe("show_widget", () => { await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); }); - it("tells the agent to use widgets proactively", () => { - expect(createShowWidgetTool().description).toMatch( - /^Visual helps\? Make widget\. Do not wait for ask\./, - ); - }); - it("keeps widget documents from duplicating host-owned metadata and controls", () => { const description = createShowWidgetTool().description; diff --git a/src/canvas/widget-tool.ts b/src/canvas/widget-tool.ts index d7580837c672..9c6552cc1e34 100644 --- a/src/canvas/widget-tool.ts +++ b/src/canvas/widget-tool.ts @@ -65,7 +65,10 @@ function createShowWidgetToolSchema( }), ), pin: Type.Optional( - Type.Boolean({ description: "Also pin this widget to the session dashboard" }), + Type.Boolean({ + description: + "Pin only for an explicit dashboard request or multiple non-code visualizations", + }), ), tab: Type.Optional( Type.String({ pattern: "^[a-z0-9-]{1,40}$", description: "Dashboard tab slug" }), @@ -270,7 +273,7 @@ export function createShowWidgetTool(options: ShowWidgetToolOptions = {}): AnyAg return { label: "Show Widget", name: "show_widget", - description: `Visual helps? Make widget. Do not wait for ask. Use for comparisons, trends, timelines, flows, hierarchies, dashboards, status, progress, layouts, and choices. Text clearer? Skip. Show a widget on the user's current surface; kind defaults to html${advertisedRegisteredKinds.length ? ` and registered kinds are ${advertisedRegisteredKinds.join(", ")}` : ""}. ${inlineHostEnabled ? "Set pin=true to also place it on this session's dashboard" : "Inline hosting is disabled; set pin=true to place it on this session's dashboard"}; use name for a stable widget id, tab for a tab slug, size sm|md|lg|xl|full, presentation.frame card|full-bleed|frameless, and after for a sibling widget anchor. Pinned widgets may declare capabilities.netOrigins and capabilities.tools for operator approval. HTML widgets are self-contained HTML or SVG. Dashboard host APIs: openclaw.prompt.send(text), openclaw.state.emit(payload), openclaw.data.read(bindingId, params?), and openclaw.cron.trigger(jobId). \`title\` is host metadata. Start directly with content; do not repeat the title or recreate dashboard chrome. HTML is pre-themed with --surface --card --elevated --text --text-strong --muted --border --border-strong --accent --accent-fill --accent-fg --ok --warn --danger --info --radius --font-body --font-mono.${presenterPrompt}`, + description: `Visual helps? Make widget. Do not wait for ask. Keep one ad hoc non-code visualization inline. Pin only for an explicit dashboard request or multiple non-code visualizations. Use for comparisons, trends, timelines, flows, hierarchies, dashboards, status, progress, layouts, and choices. Text clearer? Skip. Show a widget on the user's current surface; kind defaults to html${advertisedRegisteredKinds.length ? ` and registered kinds are ${advertisedRegisteredKinds.join(", ")}` : ""}. ${inlineHostEnabled ? "Set pin=true to also place it on this session's dashboard" : "Inline hosting is disabled; set pin=true to place it on this session's dashboard"}; use name for a stable widget id, tab for a tab slug, size sm|md|lg|xl|full, presentation.frame card|full-bleed|frameless, and after for a sibling widget anchor. Pinned widgets may declare capabilities.netOrigins and capabilities.tools for operator approval. HTML widgets are self-contained HTML or SVG. Dashboard host APIs: openclaw.prompt.send(text), openclaw.state.emit(payload), openclaw.data.read(bindingId, params?), and openclaw.cron.trigger(jobId). \`title\` is host metadata. Start directly with content; do not repeat the title or recreate dashboard chrome. HTML is pre-themed with --surface --card --elevated --text --text-strong --muted --border --border-strong --accent --accent-fill --accent-fg --ok --warn --danger --info --radius --font-body --font-mono.${presenterPrompt}`, parameters: createShowWidgetToolSchema(kinds, explicitPresenters), ...(currentChannelPresenter ? {} : { requiredClientCaps: SHOW_WIDGET_REQUIRED_CLIENT_CAPS }), execute: async (_toolCallId, args) => { diff --git a/ui/src/e2e/chat-worked-for-visualization.e2e.test.ts b/ui/src/e2e/chat-worked-for-visualization.e2e.test.ts new file mode 100644 index 000000000000..a331fc2bec0b --- /dev/null +++ b/ui/src/e2e/chat-worked-for-visualization.e2e.test.ts @@ -0,0 +1,181 @@ +// Control UI E2E covers completed-work expansion and persistent visual outcomes. +import fs from "node:fs/promises"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { controlUiSessionUrl, installMockGateway } from "../test-helpers/control-ui-e2e.ts"; +import { waitForChatScrollIdle } from "./chat-flow.test-support.ts"; +import { createControlUiE2eSuite } from "./control-ui-e2e-suite.test-support.ts"; + +declare global { + interface Window { + pauseTranscriptResizeObservers: () => void; + resumeTranscriptResizeObservers: () => void; + } +} + +const suite = createControlUiE2eSuite({ + name: "Control UI completed work visualizations", + startServerBeforeBrowser: true, +}); + +async function captureProof(page: import("playwright").Page, name: string) { + const artifactDir = process.env.OPENCLAW_CONTROL_UI_E2E_ARTIFACT_DIR?.trim(); + if (!artifactDir) { + return; + } + await fs.mkdir(artifactDir, { recursive: true }); + await page.screenshot({ path: path.join(artifactDir, `${name}.png`), fullPage: true }); +} + +suite.define(() => { + it("keeps visual output visible and virtual rows apart when completed work expands", async () => { + await suite.withPage({ viewport: { height: 900, width: 1200 } }, async ({ page }) => { + const sessionKey = "agent:main:dashboard:worked-for-geometry"; + await page.addInitScript(() => { + const NativeResizeObserver = window.ResizeObserver; + let paused = false; + Object.defineProperties(window, { + pauseTranscriptResizeObservers: { + configurable: true, + value: () => { + paused = true; + }, + }, + resumeTranscriptResizeObservers: { + configurable: true, + value: () => { + paused = false; + }, + }, + }); + window.ResizeObserver = class PausableResizeObserver extends NativeResizeObserver { + constructor(callback: ResizeObserverCallback) { + super((entries, observer) => { + if (!paused) { + callback(entries, observer); + } + }); + } + }; + }); + await page.route("**/cv_worked_for_visual/index.html", async (route) => { + await route.fulfill({ + contentType: "text/html", + body: ` + +
+
Release confidence
+
Gateway
92%
+
Control UI
84%
+
Mobile
68%
+
`, + }); + }); + await installMockGateway(page, { + sessionKey, + historyMessages: [ + { role: "user", content: "Check it.", timestamp: 1_000 }, + { + role: "assistant", + content: [ + { type: "text", text: "Here is the visualization." }, + { + type: "canvas", + preview: { + kind: "canvas", + surface: "assistant_message", + render: "url", + title: "Release status", + viewId: "cv_worked_for_visual", + url: "/__openclaw__/canvas/documents/cv_worked_for_visual/index.html", + preferredHeight: 180, + sandbox: "scripts", + }, + }, + ], + timestamp: 1_500, + }, + { + role: "assistant", + content: [{ type: "text", text: "Checking." }], + openclawStreamFallback: { + itemId: "worked-for-checking", + replacementText: "Checking.", + source: "segment", + }, + timestamp: 2_000, + }, + { + role: "toolResult", + toolCallId: "worked-for-tool", + toolName: "bash", + content: "ok", + timestamp: 3_000, + }, + { + role: "assistant", + content: [{ type: "text", text: "Done." }], + timestamp: 4_000, + }, + ], + }); + + await page.goto(controlUiSessionUrl(suite.server.baseUrl, sessionKey)); + await page.getByText("Done.", { exact: true }).waitFor(); + await page.locator('.chat-tool-card__preview[data-kind="canvas"]').waitFor(); + const workedFor = page.locator(".chat-work-group > .chat-activity-group__summary"); + await workedFor.waitFor(); + await waitForChatScrollIdle(page); + await captureProof(page, "worked-for-geometry-collapsed"); + + const geometry = await workedFor.evaluate(async (element) => { + const owner = element.closest("openclaw-chat-pane") as + | (HTMLElement & { updateComplete: Promise }) + | null; + const thread = element.closest(".chat-thread"); + if (!owner || !thread) { + throw new Error("Worked-for disclosure is missing its chat pane or thread"); + } + + window.pauseTranscriptResizeObservers(); + (element as HTMLButtonElement).click(); + await owner.updateComplete; + + const viewport = thread.getBoundingClientRect(); + const rows = Array.from(thread.querySelectorAll(".chat-virtual-row")) + .map((row) => { + const rect = row.getBoundingClientRect(); + return { key: row.dataset.virtualRowKey, top: rect.top, bottom: rect.bottom }; + }) + .filter((row) => row.bottom > viewport.top && row.top < viewport.bottom); + const overlaps = rows.flatMap((row, index) => { + const next = rows[index + 1]; + return next && row.bottom > next.top ? [{ row, next }] : []; + }); + + return { expanded: element.getAttribute("aria-expanded"), rows, overlaps }; + }); + + await captureProof(page, "worked-for-geometry-expanded"); + expect(geometry.expanded).toBe("true"); + expect(geometry.rows.length).toBeGreaterThanOrEqual(3); + expect(geometry.overlaps).toEqual([]); + + await page.evaluate(() => window.resumeTranscriptResizeObservers()); + await workedFor.click(); + await workedFor.click(); + await waitForChatScrollIdle(page); + await captureProof(page, "worked-for-geometry-settled"); + }); + }); +}); diff --git a/ui/src/pages/chat/chat-thread-grouping.ts b/ui/src/pages/chat/chat-thread-grouping.ts index 7c59a2b97576..f324606251cc 100644 --- a/ui/src/pages/chat/chat-thread-grouping.ts +++ b/ui/src/pages/chat/chat-thread-grouping.ts @@ -511,7 +511,7 @@ type ActivityRunRenderItem = { type TurnRenderItem = RenderChatItem | StreamRunRenderItem; function isCollapsibleWorkGroup(item: TurnRenderItem): item is MessageGroup { - if (item.kind !== "group" || item.isStreaming) { + if (item.kind !== "group" || item.isStreaming || groupHasVisibleReplyContent(item, false)) { return false; } const role = item.role.toLowerCase(); @@ -522,15 +522,15 @@ function isCollapsibleWorkGroup(item: TurnRenderItem): item is MessageGroup { // visible outcome; they must never fold into the work rollup. Normalized // content passes unknown block types through (e.g. raw image blocks), so // anything that is not a tool block counts as visible reply content. -function assistantGroupHasVisibleReplyContent(group: MessageGroup): boolean { +function groupHasVisibleReplyContent(group: MessageGroup, includeText = true): boolean { return group.messages.some(({ message }) => { - if (extractTextCached(message)?.trim()) { + if (includeText && extractTextCached(message)?.trim()) { return true; } const content = safeNormalizeMessage(message)?.content ?? []; return content.some((block) => { if (block.type === "text") { - return Boolean(block.text?.trim()); + return includeText && Boolean(block.text?.trim()); } return !isToolCallContentType(block.type) && !isToolResultContentType(block.type); }); @@ -541,7 +541,7 @@ export function assistantGroupCanOwnActiveRunStatus(group: MessageGroup): boolea return ( group.role.toLowerCase() === "assistant" && !assistantGroupIsForwardedBoundary(group) && - assistantGroupHasVisibleReplyContent(group) + groupHasVisibleReplyContent(group) ); } @@ -550,11 +550,7 @@ export function assistantGroupCanOwnActiveRunStatus(group: MessageGroup): boolea // stands in for the final reply. Turns whose last content is commentary // merely collapse less; the visible reply is never folded away. function isFinalReplyGroup(item: TurnRenderItem): boolean { - return ( - isCollapsibleWorkGroup(item) && - item.role.toLowerCase() === "assistant" && - assistantGroupHasVisibleReplyContent(item) - ); + return item.kind === "group" && !item.isStreaming && assistantGroupCanOwnActiveRunStatus(item); } /** diff --git a/ui/src/pages/chat/chat-thread.test.ts b/ui/src/pages/chat/chat-thread.test.ts index 0e4ecb432101..93632bdc1ad1 100644 --- a/ui/src/pages/chat/chat-thread.test.ts +++ b/ui/src/pages/chat/chat-thread.test.ts @@ -757,6 +757,50 @@ describe("collapseCompletedTurnWork", () => { expect(requireGroup(items[2]).role).toBe("assistant"); }); + it.each([ + { + role: "assistant", + visualization: assistantMessage( + [ + { type: "text", text: "Here is the chart." }, + createAssistantCanvasBlock({ suffix: "completed_turn_assistant_visual" }), + ], + 2_000, + ), + commentary: [], + workRoles: ["tool"], + }, + { + role: "tool", + visualization: toolResultMessage( + "visualization", + "show_widget", + [createAssistantCanvasBlock({ suffix: "completed_turn_tool_visual" })], + 2_000, + ), + commentary: [assistantMessage("Checking the details.", 2_500)], + workRoles: ["assistant", "tool"], + }, + ])( + "keeps an earlier visualization visible while collapsing only tool work ($role)", + ({ visualization, commentary, workRoles }) => { + const items = collapsedItems({ + messages: [ + userMessage("show the result", 1_000), + visualization, + ...commentary, + toolResult("call-1", 3_000), + assistantMessage("All done.", 4_000), + ], + }); + + expect(items.map((item) => item.kind)).toEqual(["group", "group", "work-group", "group"]); + expect(canvasBlocksIn(requireGroup(items[1]))).toHaveLength(1); + expect(requireWorkGroup(items[2]).groups.map((group) => group.role)).toEqual(workRoles); + expect(messageRecord(requireGroup(items[3])).content).toBe("All done."); + }, + ); + it.each([ "agent:main:main", "agent:main:telegram:direct:42", diff --git a/ui/src/pages/chat/components/chat-transcript-projection.ts b/ui/src/pages/chat/components/chat-transcript-projection.ts index 01f68bce986c..85445cd44006 100644 --- a/ui/src/pages/chat/components/chat-transcript-projection.ts +++ b/ui/src/pages/chat/components/chat-transcript-projection.ts @@ -1,6 +1,6 @@ // Chat-item projection, expansion, reply hydration, and guarded row rendering. import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; -import { html, nothing, type TemplateResult } from "lit"; +import { nothing, type TemplateResult } from "lit"; import { guard } from "lit/directives/guard.js"; import { classifySessionKind } from "../../../../../src/sessions/classify-session-kind.js"; import { i18n } from "../../../i18n/index.ts"; @@ -463,16 +463,13 @@ export function projectChatTranscript( } if (item.kind === "work-group") { const workExpanded = expandedToolCards.get(item.key) ?? false; - return html` - ${renderWorkGroupSummary(item, { - expanded: workExpanded, - onToggle: () => { - setExpansionState(expandedToolCards, item.key, !workExpanded); - requestUpdate(); - }, - })} - ${workExpanded ? item.groups.map((group) => renderGroupItem(group)) : nothing} - `; + return renderWorkGroupSummary(item, { + expanded: workExpanded, + onToggle: () => { + setExpansionState(expandedToolCards, item.key, !workExpanded); + requestUpdate(); + }, + }); } if (item.kind === "activity-run") { const firstGroup = item.groups[0]; @@ -577,11 +574,19 @@ export function projectChatTranscript( turnRecapOwnerKey = lastItem.key; } } - const transcriptRows: TranscriptRow[] = transcriptItems.map((item) => ({ - kind: "item", - key: item.key, - item, - })); + // New row keys measure expanded work immediately; existing keys keep their + // cached height until ResizeObserver reports the changed layout. + const transcriptRows = transcriptItems.flatMap((item): TranscriptRow[] => + [{ kind: "item" as const, key: item.key, item }].concat( + item.kind === "work-group" && expandedToolCards.get(item.key) + ? item.groups.map((group) => ({ + kind: "item" as const, + key: `${item.key}:${group.key}`, + item: group, + })) + : [], + ), + ); const realtimeConversation = renderRealtimeTalkConversation(props); if (realtimeConversation !== nothing) { transcriptRows.push({