From 01a7f4fb3fc05616cbf9995df0c8b2303db09abe Mon Sep 17 00:00:00 2001 From: "Jason (Json)" <263060202+fuller-stack-dev@users.noreply.github.com> Date: Tue, 14 Jul 2026 04:14:01 -0600 Subject: [PATCH] feat(ui): add images and model selection to new sessions (#107358) Merged via squash. Prepared head SHA: cf99b5bab5911a7f25774385e2d68a54be81ef76 Co-authored-by: fuller-stack-dev <263060202+fuller-stack-dev@users.noreply.github.com> Co-authored-by: fuller-stack-dev <263060202+fuller-stack-dev@users.noreply.github.com> Reviewed-by: @fuller-stack-dev --- .../OpenClawProtocol/GatewayModels.swift | 4 + .../gateway-protocol/src/schema/logs-chat.ts | 5 +- .../src/schema/sessions-create.ts | 48 ++ .../gateway-protocol/src/schema/sessions.ts | 46 +- packages/sdk/src/types.ts | 1 + .../server-methods/attachment-normalize.ts | 2 +- .../session-create-initial-turn.ts | 51 ++ src/gateway/server-methods/sessions.ts | 55 +- src/gateway/server.sessions.create.test.ts | 47 ++ ui/src/e2e/chat-flow.e2e.test.ts | 96 ++++ ui/src/e2e/new-session-page.e2e.test.ts | 377 +++++++++++++- ui/src/i18n/.i18n/catalog-fallbacks.json | 27 +- ui/src/i18n/locales/en.ts | 1 + ui/src/lib/sessions/create.ts | 2 + ui/src/pages/chat/attachment-api.ts | 33 ++ ui/src/pages/chat/chat-gateway.ts | 20 +- ui/src/pages/chat/chat-history-retry.ts | 44 ++ ui/src/pages/chat/chat-history.ts | 63 +-- ui/src/pages/chat/chat-pane-state.ts | 19 + ui/src/pages/chat/chat-pane.ts | 5 +- ui/src/pages/chat/chat-queue.ts | 8 +- ui/src/pages/chat/chat-send.ts | 34 +- ui/src/pages/chat/chat-state.ts | 21 +- ui/src/pages/chat/chat-view.ts | 2 +- .../pages/chat/components/chat-attachments.ts | 484 ++++++++++++++++++ ui/src/pages/chat/components/chat-composer.ts | 473 +---------------- .../chat/components/chat-model-controls.ts | 35 +- .../components/chat-model-provider-menu.ts | 19 + ui/src/pages/chat/initial-turn-handoff.ts | 64 +++ .../pages/chat/terminal-message-identity.ts | 122 +++++ ui/src/pages/new-session/attachment-draft.ts | 51 ++ ui/src/pages/new-session/composer.ts | 142 ++++- .../pages/new-session/create-params.test.ts | 45 ++ ui/src/pages/new-session/create-params.ts | 12 + ui/src/pages/new-session/model-control.ts | 92 ++++ ui/src/pages/new-session/new-session-page.ts | 86 ++-- .../new-session/rejected-initial-turn.ts | 52 ++ ui/src/styles/new-session.css | 7 + 38 files changed, 1960 insertions(+), 735 deletions(-) create mode 100644 packages/gateway-protocol/src/schema/sessions-create.ts create mode 100644 src/gateway/server-methods/session-create-initial-turn.ts create mode 100644 ui/src/pages/chat/attachment-api.ts create mode 100644 ui/src/pages/chat/chat-history-retry.ts create mode 100644 ui/src/pages/chat/chat-pane-state.ts create mode 100644 ui/src/pages/chat/components/chat-attachments.ts create mode 100644 ui/src/pages/chat/components/chat-model-provider-menu.ts create mode 100644 ui/src/pages/chat/initial-turn-handoff.ts create mode 100644 ui/src/pages/chat/terminal-message-identity.ts create mode 100644 ui/src/pages/new-session/attachment-draft.ts create mode 100644 ui/src/pages/new-session/model-control.ts create mode 100644 ui/src/pages/new-session/rejected-initial-turn.ts diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 5cd9b4dc2b91..764bf4cac961 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -4524,6 +4524,7 @@ public struct SessionsCreateParams: Codable, Sendable { public let emitcommandhooks: Bool? public let task: String? public let message: String? + public let attachments: [AnyCodable]? public let worktree: Bool? public let worktreebaseref: String? public let worktreename: String? @@ -4541,6 +4542,7 @@ public struct SessionsCreateParams: Codable, Sendable { emitcommandhooks: Bool? = nil, task: String? = nil, message: String? = nil, + attachments: [AnyCodable]? = nil, worktree: Bool? = nil, worktreebaseref: String? = nil, worktreename: String? = nil, @@ -4557,6 +4559,7 @@ public struct SessionsCreateParams: Codable, Sendable { self.emitcommandhooks = emitcommandhooks self.task = task self.message = message + self.attachments = attachments self.worktree = worktree self.worktreebaseref = worktreebaseref self.worktreename = worktreename @@ -4575,6 +4578,7 @@ public struct SessionsCreateParams: Codable, Sendable { case emitcommandhooks = "emitCommandHooks" case task case message + case attachments case worktree case worktreebaseref = "worktreeBaseRef" case worktreename = "worktreeName" diff --git a/packages/gateway-protocol/src/schema/logs-chat.ts b/packages/gateway-protocol/src/schema/logs-chat.ts index 0a74810b7590..df8645d5c20a 100644 --- a/packages/gateway-protocol/src/schema/logs-chat.ts +++ b/packages/gateway-protocol/src/schema/logs-chat.ts @@ -82,6 +82,9 @@ export const ChatMessageGetResultSchema = closedObject({ /** Typed result shape for callers that branch on message availability. */ export type ChatMessageGetResult = Static; +/** Attachment envelope shared by chat.send and session creation's initial turn. */ +export const ChatAttachmentsSchema = Type.Array(Type.Unknown()); + /** User-to-agent send request; idempotency key lets clients safely retry transport failures. */ export const ChatSendParamsSchema = closedObject({ sessionKey: ChatSendSessionKeyString, @@ -97,7 +100,7 @@ export const ChatSendParamsSchema = closedObject({ originatingTo: Type.Optional(Type.String()), originatingAccountId: Type.Optional(Type.String()), originatingThreadId: Type.Optional(Type.String()), - attachments: Type.Optional(Type.Array(Type.Unknown())), + attachments: Type.Optional(ChatAttachmentsSchema), timeoutMs: Type.Optional(Type.Integer({ minimum: 0 })), systemInputProvenance: Type.Optional(InputProvenanceSchema), systemProvenanceReceipt: Type.Optional(Type.String()), diff --git a/packages/gateway-protocol/src/schema/sessions-create.ts b/packages/gateway-protocol/src/schema/sessions-create.ts new file mode 100644 index 000000000000..33ce22106b54 --- /dev/null +++ b/packages/gateway-protocol/src/schema/sessions-create.ts @@ -0,0 +1,48 @@ +import { Type } from "typebox"; +import { closedObject } from "./closed-object.js"; +import { ChatAttachmentsSchema } from "./logs-chat.js"; +import { NonEmptyString, SessionLabelString } from "./primitives.js"; + +/** Creates or adopts a session with optional model, label, and parent linkage. */ +export const SessionsCreateParamsSchema = closedObject({ + key: Type.Optional(NonEmptyString), + agentId: Type.Optional(NonEmptyString), + label: Type.Optional(SessionLabelString), + model: Type.Optional(NonEmptyString), + catalogId: Type.Optional(NonEmptyString), + parentSessionKey: Type.Optional(NonEmptyString), + fork: Type.Optional( + Type.Boolean({ description: "Fork the parent transcript; requires parentSessionKey." }), + ), + emitCommandHooks: Type.Optional(Type.Boolean()), + task: Type.Optional(Type.String()), + message: Type.Optional(Type.String()), + attachments: Type.Optional(ChatAttachmentsSchema), + worktree: Type.Optional(Type.Boolean()), + worktreeBaseRef: Type.Optional( + Type.String({ + minLength: 1, + description: "Base ref for the new managed worktree branch. Requires worktree=true.", + }), + ), + worktreeName: Type.Optional( + Type.String({ + pattern: "^[a-z0-9][a-z0-9-]{0,63}$", + description: "Managed worktree name; becomes branch openclaw/. Requires worktree=true.", + }), + ), + execNode: Type.Optional( + Type.String({ + minLength: 1, + description: + "Bind session exec to host=node with this node id/name. Requires operator.admin.", + }), + ), + cwd: Type.Optional( + Type.String({ + minLength: 1, + description: + "Absolute source directory for a managed worktree, or the working directory on execNode. Requires operator.admin.", + }), + ), +}); diff --git a/packages/gateway-protocol/src/schema/sessions.ts b/packages/gateway-protocol/src/schema/sessions.ts index 44c3ebd7d25f..3caf9236d931 100644 --- a/packages/gateway-protocol/src/schema/sessions.ts +++ b/packages/gateway-protocol/src/schema/sessions.ts @@ -5,6 +5,9 @@ import { closedObject } from "./closed-object.js"; import { ErrorShapeSchema } from "./frames.js"; import { PluginJsonValueSchema } from "./plugins.js"; import { NonEmptyString, SessionLabelString } from "./primitives.js"; +import { SessionsCreateParamsSchema } from "./sessions-create.js"; + +export { SessionsCreateParamsSchema }; /** * Session protocol schemas. @@ -292,49 +295,6 @@ export const SessionsResolveParamsSchema = closedObject({ allowMissing: Type.Optional(Type.Boolean()), }); -/** Creates or adopts a session with optional model, label, and parent linkage. */ -export const SessionsCreateParamsSchema = closedObject({ - key: Type.Optional(NonEmptyString), - agentId: Type.Optional(NonEmptyString), - label: Type.Optional(SessionLabelString), - model: Type.Optional(NonEmptyString), - catalogId: Type.Optional(NonEmptyString), - parentSessionKey: Type.Optional(NonEmptyString), - fork: Type.Optional( - Type.Boolean({ description: "Fork the parent transcript; requires parentSessionKey." }), - ), - emitCommandHooks: Type.Optional(Type.Boolean()), - task: Type.Optional(Type.String()), - message: Type.Optional(Type.String()), - worktree: Type.Optional(Type.Boolean()), - worktreeBaseRef: Type.Optional( - Type.String({ - minLength: 1, - description: "Base ref for the new managed worktree branch. Requires worktree=true.", - }), - ), - worktreeName: Type.Optional( - Type.String({ - pattern: "^[a-z0-9][a-z0-9-]{0,63}$", - description: "Managed worktree name; becomes branch openclaw/. Requires worktree=true.", - }), - ), - execNode: Type.Optional( - Type.String({ - minLength: 1, - description: - "Bind session exec to host=node with this node id/name. Requires operator.admin.", - }), - ), - cwd: Type.Optional( - Type.String({ - minLength: 1, - description: - "Absolute source directory for a managed worktree, or the working directory on execNode. Requires operator.admin.", - }), - ), -}); - export const SessionWorktreeInfoSchema = closedObject({ id: NonEmptyString, path: NonEmptyString, diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 1c831f6c07b9..8ef7e1c379ec 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -344,6 +344,7 @@ export type SessionCreateParams = { parentSessionKey?: string; task?: string; message?: string; + attachments?: unknown[]; }; /** Parameters for sending a message to an existing session. */ diff --git a/src/gateway/server-methods/attachment-normalize.ts b/src/gateway/server-methods/attachment-normalize.ts index 4b49c3eaa133..6cd790c33d85 100644 --- a/src/gateway/server-methods/attachment-normalize.ts +++ b/src/gateway/server-methods/attachment-normalize.ts @@ -3,7 +3,7 @@ import type { ChatAttachment } from "../chat-attachments.js"; /** RPC attachment payload shape accepted by chat-like gateway methods. */ -type RpcAttachmentInput = { +export type RpcAttachmentInput = { type?: unknown; mimeType?: unknown; fileName?: unknown; diff --git a/src/gateway/server-methods/session-create-initial-turn.ts b/src/gateway/server-methods/session-create-initial-turn.ts new file mode 100644 index 000000000000..f09f86fe05f0 --- /dev/null +++ b/src/gateway/server-methods/session-create-initial-turn.ts @@ -0,0 +1,51 @@ +import { + normalizeRpcAttachmentsToChatAttachments, + type RpcAttachmentInput, +} from "./attachment-normalize.js"; + +function resolveOptionalInitialSessionMessage(params: { + task?: unknown; + message?: unknown; +}): string | undefined { + if (typeof params.task === "string" && params.task.trim()) { + return params.task; + } + if (typeof params.message === "string" && params.message.trim()) { + return params.message; + } + return undefined; +} + +export function resolveSessionCreateInitialTurn(params: { + attachments?: unknown[]; + message?: unknown; + task?: unknown; +}) { + const message = resolveOptionalInitialSessionMessage(params); + const normalizedAttachments = normalizeRpcAttachmentsToChatAttachments( + params.attachments as RpcAttachmentInput[] | undefined, + ); + if (params.attachments?.length && !message && normalizedAttachments.length === 0) { + return null; + } + const attachments = normalizedAttachments.length ? normalizedAttachments : undefined; + return { + attachments, + hasInitialTurn: message !== undefined || attachments !== undefined, + message, + }; +} + +export function shouldAttachPendingMessageSeq(params: { + cached?: boolean; + payload: unknown; +}): boolean { + if (params.cached) { + return false; + } + const status = + params.payload && typeof params.payload === "object" + ? (params.payload as { status?: unknown }).status + : undefined; + return status === "started"; +} diff --git a/src/gateway/server-methods/sessions.ts b/src/gateway/server-methods/sessions.ts index 585da20a9ec5..022c6a58ab8a 100644 --- a/src/gateway/server-methods/sessions.ts +++ b/src/gateway/server-methods/sessions.ts @@ -172,6 +172,10 @@ import { } from "./session-active-runs.js"; import { resolveSessionCatalogCreateTarget } from "./session-catalog.js"; import { emitSessionsChanged } from "./session-change-event.js"; +import { + resolveSessionCreateInitialTurn, + shouldAttachPendingMessageSeq, +} from "./session-create-initial-turn.js"; import type { GatewayClient, GatewayRequestContext, @@ -407,30 +411,6 @@ function loadSessionEntriesForTarget(params: { return { target, storePath: target.storePath, store, entry }; } -function resolveOptionalInitialSessionMessage(params: { - task?: unknown; - message?: unknown; -}): string | undefined { - if (typeof params.task === "string" && params.task.trim()) { - return params.task; - } - if (typeof params.message === "string" && params.message.trim()) { - return params.message; - } - return undefined; -} - -function shouldAttachPendingMessageSeq(params: { payload: unknown; cached?: boolean }): boolean { - if (params.cached) { - return false; - } - const status = - params.payload && typeof params.payload === "object" - ? (params.payload as { status?: unknown }).status - : undefined; - return status === "started"; -} - function emitSessionOperation( context: Pick, payload: Omit, @@ -1557,7 +1537,23 @@ export const sessionsHandlers: GatewayRequestHandlers = { ); return; } - const initialMessage = resolveOptionalInitialSessionMessage(p); + const initialTurn = resolveSessionCreateInitialTurn(p); + if (!initialTurn) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + "sessions.create attachments require usable content", + ), + ); + return; + } + const { + attachments: initialAttachments, + hasInitialTurn, + message: initialMessage, + } = initialTurn; const requestedCwd = normalizeOptionalString(p.cwd); const requestedExecNode = normalizeOptionalString(p.execNode); if (requestedCwd && p.worktree !== true && !requestedExecNode) { @@ -1634,7 +1630,7 @@ export const sessionsHandlers: GatewayRequestHandlers = { !targetKey && parentSessionKey && p.emitCommandHooks === true && - !initialMessage && + !hasInitialTurn && cfg.session?.dmScope === "main" ) { const parent = loadSessionEntry( @@ -1769,10 +1765,10 @@ export const sessionsHandlers: GatewayRequestHandlers = { clearSpawnedCwd: p.worktree !== true, fork: p.fork, emitCommandHooks: p.emitCommandHooks, - resetMainWhenUnspecified: !initialMessage, + resetMainWhenUnspecified: !hasInitialTurn, commandSource: "webchat", loadGatewayModelCatalog: context.loadGatewayModelCatalog, - afterCreate: initialMessage + afterCreate: hasInitialTurn ? async ({ key, agentId, entry, storePath }) => { messageSeq = (await readSessionMessageCountAsync({ @@ -1790,8 +1786,9 @@ export const sessionsHandlers: GatewayRequestHandlers = { params: { sessionKey: key, ...(key === "global" ? { agentId } : {}), - message: initialMessage, + message: initialMessage ?? "", idempotencyKey: randomUUID(), + ...(initialAttachments ? { attachments: initialAttachments } : {}), }, respond: (ok, payload, error, meta) => { if (ok && payload && typeof payload === "object") { diff --git a/src/gateway/server.sessions.create.test.ts b/src/gateway/server.sessions.create.test.ts index b567f9925a5a..716d7a6030a1 100644 --- a/src/gateway/server.sessions.create.test.ts +++ b/src/gateway/server.sessions.create.test.ts @@ -1813,6 +1813,53 @@ test("sessions.create can start the first agent turn from an initial task", asyn ws.close(); }); +test("sessions.create forwards an attachment-only first turn", async () => { + await createSessionStoreDir(); + testState.agentsConfig = { list: [{ id: "main", default: true }] }; + const { chatHandlers } = await import("./server-methods/chat.js"); + const chatSend = vi.spyOn(chatHandlers, "chat.send").mockImplementation(async ({ respond }) => { + respond(true, { runId: "attachment-run", status: "started" }); + }); + const attachment = { + type: "image", + mimeType: "image/png", + fileName: "pixel.png", + content: + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII=", + }; + + try { + const created = await directSessionReq<{ runStarted?: boolean; runId?: string }>( + "sessions.create", + { agentId: "main", message: "", attachments: [attachment] }, + ); + + expect(created.ok).toBe(true); + expect(created.payload).toMatchObject({ runStarted: true, runId: "attachment-run" }); + expect(chatSend.mock.calls[0]?.[0].params).toMatchObject({ + message: "", + attachments: [attachment], + }); + } finally { + chatSend.mockRestore(); + } +}); + +test("sessions.create rejects unusable attachment-only input before creating a session", async () => { + await createSessionStoreDir(); + testState.agentsConfig = { list: [{ id: "main", default: true }] }; + + const created = await directSessionReq("sessions.create", { + agentId: "main", + attachments: [null], + }); + + expect(created.ok).toBe(false); + expect(created.error?.message).toContain("attachments require usable content"); + const listed = await directSessionReq<{ sessions?: unknown[] }>("sessions.list", {}); + expect(listed.payload?.sessions).toEqual([]); +}); + test("sessions.create rejects replacing its parent key", async () => { await createSessionStoreDir(); testState.agentsConfig = { list: [{ id: "main", default: true }] }; diff --git a/ui/src/e2e/chat-flow.e2e.test.ts b/ui/src/e2e/chat-flow.e2e.test.ts index d47113ccbc44..4dc23ef303cf 100644 --- a/ui/src/e2e/chat-flow.e2e.test.ts +++ b/ui/src/e2e/chat-flow.e2e.test.ts @@ -427,6 +427,102 @@ describeControlUiE2e("Control UI mocked Gateway E2E", () => { } }); + it("reconciles authoritative history before a trailing final by run identity", async () => { + const context = await newBrowserContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page, { historyMessages: [] }); + try { + await page.goto(`${server.baseUrl}chat`); + const composer = page.locator(".agent-chat__composer-combobox textarea"); + await composer.fill("reconcile the terminal event ordering"); + await page.getByRole("button", { name: "Send message" }).click(); + const send = await gateway.waitForRequest("chat.send"); + const runId = requireString( + requireRecord(send.params).idempotencyKey, + "chat send idempotency key", + ); + const finalText = "One authoritative final response."; + const messageId = "assistant-authoritative-final"; + const authoritative = { + __openclaw: { id: messageId, seq: 2 }, + content: [{ text: finalText, type: "text" }], + role: "assistant", + timestamp: Date.now(), + }; + await gateway.emitGatewayEvent("chat", { + deltaText: finalText, + message: { + content: [{ text: finalText, type: "text" }], + role: "assistant", + timestamp: Date.now(), + }, + runId, + sessionKey: "main", + state: "delta", + }); + await page.locator(".chat-bubble.streaming", { hasText: finalText }).waitFor(); + await gateway.setHistoryMessages([ + { + __openclaw: { id: "user-reconcile", seq: 1 }, + content: [{ text: "reconcile the terminal event ordering", type: "text" }], + role: "user", + timestamp: Date.now() - 1, + }, + authoritative, + ]); + const historyRequestsBefore = (await gateway.getRequests("chat.history")).length; + await gateway.emitGatewayEvent("session.message", { + activeRunIds: [], + clientRunId: runId, + hasActiveRun: false, + message: authoritative, + messageId, + messageSeq: 2, + session: { + activeRunIds: [], + hasActiveRun: false, + key: "main", + kind: "direct", + status: "done", + updatedAt: Date.now(), + }, + sessionKey: "main", + }); + await expect + .poll(async () => (await gateway.getRequests("chat.history")).length) + .toBeGreaterThan(historyRequestsBefore); + await page.getByText(finalText, { exact: true }).waitFor(); + await expect + .poll(() => + page.locator(".chat-group.assistant .chat-text", { hasText: finalText }).count(), + ) + .toBe(1); + + await gateway.emitChatFinal({ runId, text: finalText }); + await expect + .poll(() => page.locator(".chat-group.assistant .chat-duplicate-count").count()) + .toBe(0); + await expect + .poll(() => + page.locator(".chat-group.assistant .chat-text", { hasText: finalText }).count(), + ) + .toBe(1); + + await gateway.emitChatFinal({ runId: "a-different-legitimate-run", text: finalText }); + await expect + .poll(() => + page.locator(".chat-group.assistant .chat-text", { hasText: finalText }).count(), + ) + .toBe(2); + } finally { + await closeBrowserContext(context); + } + }); + it("restores the selected session transcript after a hard reload", async () => { const context = await newBrowserContext({ locale: "en-US", diff --git a/ui/src/e2e/new-session-page.e2e.test.ts b/ui/src/e2e/new-session-page.e2e.test.ts index 273e09b6d1ed..a51ddac7e78c 100644 --- a/ui/src/e2e/new-session-page.e2e.test.ts +++ b/ui/src/e2e/new-session-page.e2e.test.ts @@ -27,6 +27,26 @@ const NODE_PICKED = "/Users/peter/Projects"; const NODE_UNC = "\\\\server\\share\\repo"; const EXEC_ONLY_PICKED = "C:\\Users\\peter\\repo"; +const ONE_PIXEL_PNG_B64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII="; + +async function pastePng(target: Locator, count = 1) { + await target.evaluate( + (element, { base64, fileCount }) => { + const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)); + const clipboard = new DataTransfer(); + for (let index = 0; index < fileCount; index += 1) { + const fileName = fileCount === 1 ? "pixel.png" : `pixel-${index + 1}.png`; + clipboard.items.add(new File([bytes], fileName, { type: "image/png" })); + } + element.dispatchEvent( + new ClipboardEvent("paste", { bubbles: true, cancelable: true, clipboardData: clipboard }), + ); + }, + { base64: ONE_PIXEL_PNG_B64, fileCount: count }, + ); +} + function installRepositorySwitchGateway(page: Page, sessionKey: string) { return installMockGateway(page, { workspaceGit: true, @@ -143,6 +163,285 @@ describeControlUiE2e("Control UI new-session page mocked Gateway E2E", () => { await server?.close(); }); + it("pastes an image into the draft and forwards it with the initial turn", async () => { + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page, { + methodResponses: { + "sessions.create": { key: "agent:main:image-draft", runStarted: true }, + }, + }); + try { + await page.goto(`${server.baseUrl}new`); + const message = page.locator(".new-session-page__message"); + await message.waitFor(); + await pastePng(message); + + await page.locator('.chat-attachment-thumb img[alt="Attachment preview"]').waitFor(); + await page.getByRole("button", { name: "Start session" }).click(); + + const create = await gateway.waitForRequest("sessions.create"); + expect(create.params).toMatchObject({ + agentId: "main", + message: "", + attachments: [ + { + type: "image", + mimeType: "image/png", + fileName: "pixel.png", + content: ONE_PIXEL_PNG_B64, + }, + ], + }); + } finally { + await context.close(); + } + }); + + it("waits for pasted image reads before enabling session creation", async () => { + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + await page.addInitScript(() => { + const readAsDataUrl = Object.getOwnPropertyDescriptor(FileReader.prototype, "readAsDataURL") + ?.value as FileReader["readAsDataURL"]; + FileReader.prototype.readAsDataURL = function (blob: Blob) { + (globalThis as unknown as { finishPastedImageRead?: () => void }).finishPastedImageRead = + () => readAsDataUrl.call(this, blob); + }; + }); + const gateway = await installMockGateway(page, { + methodResponses: { + "sessions.create": { key: "agent:main:delayed-image-draft", runStarted: true }, + }, + }); + try { + await page.goto(`${server.baseUrl}new`); + const composer = page.locator(".new-session-page__message"); + const submit = page.getByRole("button", { name: "Start session" }); + await composer.fill("include the image that is still loading"); + await pastePng(composer); + + await expect.poll(() => submit.isDisabled()).toBe(true); + expect(await gateway.getRequests("sessions.create")).toHaveLength(0); + await page.evaluate(() => { + const finish = (globalThis as unknown as { finishPastedImageRead?: () => void }) + .finishPastedImageRead; + if (!finish) { + throw new Error("Pasted image read was not started"); + } + finish(); + }); + + await page.locator('.chat-attachment-thumb img[alt="Attachment preview"]').waitFor(); + await expect.poll(() => submit.isEnabled()).toBe(true); + await submit.click(); + const create = await gateway.waitForRequest("sessions.create"); + expect(create.params).toMatchObject({ + message: "include the image that is still loading", + attachments: [{ fileName: "pixel.png", content: ONE_PIXEL_PNG_B64 }], + }); + } finally { + await context.close(); + } + }); + + it("releases a completed file when the rest of its pasted batch is aborted", async () => { + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + await page.addInitScript(() => { + const readAsDataUrl = Object.getOwnPropertyDescriptor(FileReader.prototype, "readAsDataURL") + ?.value as FileReader["readAsDataURL"]; + let readCount = 0; + FileReader.prototype.readAsDataURL = function (blob: Blob) { + readCount += 1; + if (readCount === 1) { + readAsDataUrl.call(this, blob); + } + }; + const createObjectURL = URL.createObjectURL.bind(URL); + const revokeObjectURL = URL.revokeObjectURL.bind(URL); + const proof = { created: 0, revoked: 0 }; + (globalThis as unknown as { attachmentUrlProof: typeof proof }).attachmentUrlProof = proof; + URL.createObjectURL = (blob: Blob) => { + proof.created += 1; + return createObjectURL(blob); + }; + URL.revokeObjectURL = (url: string) => { + proof.revoked += 1; + revokeObjectURL(url); + }; + }); + await installMockGateway(page); + try { + await page.goto(`${server.baseUrl}new`); + const composer = page.locator(".new-session-page__message"); + await pastePng(composer, 2); + await expect + .poll(() => + page.evaluate( + () => + (globalThis as unknown as { attachmentUrlProof: { created: number } }) + .attachmentUrlProof.created, + ), + ) + .toBe(1); + + await page.evaluate(() => { + const app = document.querySelector("openclaw-app") as HTMLElement & { + runtime?: { context: { navigate: (routeId: string) => void } }; + }; + app.runtime?.context.navigate("chat"); + }); + await page.waitForURL((url) => url.pathname.endsWith("/chat")); + await expect + .poll(() => + page.evaluate( + () => + (globalThis as unknown as { attachmentUrlProof: { revoked: number } }) + .attachmentUrlProof.revoked, + ), + ) + .toBe(1); + } finally { + await context.close(); + } + }); + + it("releases pasted image previews after remove, reset, disconnect, and success", async () => { + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + await page.addInitScript(() => { + const createObjectURL = URL.createObjectURL.bind(URL); + const revokeObjectURL = URL.revokeObjectURL.bind(URL); + const proof = { created: 0, revoked: 0 }; + (globalThis as unknown as { attachmentUrlProof: typeof proof }).attachmentUrlProof = proof; + URL.createObjectURL = (blob: Blob) => { + proof.created += 1; + return createObjectURL(blob); + }; + URL.revokeObjectURL = (url: string) => { + proof.revoked += 1; + revokeObjectURL(url); + }; + }); + await installMockGateway(page, { + methodResponses: { + "sessions.create": { key: "agent:main:preview-cleanup", runStarted: true }, + }, + }); + const proof = () => + page.evaluate( + () => + (globalThis as unknown as { attachmentUrlProof: { created: number; revoked: number } }) + .attachmentUrlProof, + ); + const navigate = (routeId: string, search = "") => + page.evaluate( + ({ targetRouteId, targetSearch }) => { + const app = document.querySelector("openclaw-app") as HTMLElement & { + runtime?: { + context: { + navigate: (routeId: string, options?: { search?: string }) => void; + }; + }; + }; + if (!app.runtime) { + throw new Error("OpenClaw application runtime is unavailable"); + } + app.runtime.context.navigate(targetRouteId, { search: targetSearch }); + }, + { targetRouteId: routeId, targetSearch: search }, + ); + + try { + await page.goto(`${server.baseUrl}new`); + const composer = page.locator(".new-session-page__message"); + + await pastePng(composer); + await page.locator('.chat-attachment-thumb img[alt="Attachment preview"]').waitFor(); + await page.getByRole("button", { name: "Remove attachment" }).click(); + await expect.poll(async () => (await proof()).revoked).toBe(1); + + await pastePng(composer); + await page.locator('.chat-attachment-thumb img[alt="Attachment preview"]').waitFor(); + await navigate("new-session", "?agent=main&catalog=missing"); + await expect.poll(() => page.locator(".chat-attachment-thumb").count()).toBe(0); + await expect.poll(async () => (await proof()).revoked).toBe(2); + + await navigate("new-session"); + await composer.waitFor(); + await pastePng(composer); + await page.locator('.chat-attachment-thumb img[alt="Attachment preview"]').waitFor(); + await navigate("chat"); + await page.waitForURL((url) => url.pathname.endsWith("/chat")); + await expect.poll(async () => (await proof()).revoked).toBe(3); + + await navigate("new-session"); + await composer.waitFor(); + await pastePng(composer); + await page.locator('.chat-attachment-thumb img[alt="Attachment preview"]').waitFor(); + await page.getByRole("button", { name: "Start session" }).click(); + await page.waitForURL( + (url) => url.searchParams.get("session") === "agent:main:preview-cleanup", + ); + await expect.poll(async () => await proof()).toEqual({ created: 4, revoked: 4 }); + } finally { + await context.close(); + } + }); + + it("selects the model for a plain new session", async () => { + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page, { + models: [ + { id: "gpt-5.5", name: "GPT 5.5", provider: "openai" }, + { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6", provider: "anthropic" }, + ], + methodResponses: { + "sessions.create": { key: "agent:main:model-draft", runStarted: true }, + }, + }); + try { + await page.goto(`${server.baseUrl}new`); + const modelSelect = page.locator('[data-chat-model-select="true"]'); + await modelSelect.waitFor(); + await modelSelect.click(); + await page.locator('[data-chat-model-provider="anthropic"]').click(); + await page.locator('[data-chat-model-option="anthropic/claude-sonnet-4-6"]').click(); + await page.locator(".new-session-page__message").fill("use this model"); + await page.getByRole("button", { name: "Start session" }).click(); + + const create = await gateway.waitForRequest("sessions.create"); + expect(create.params).toMatchObject({ + message: "use this model", + model: "anthropic/claude-sonnet-4-6", + }); + } finally { + await context.close(); + } + }); + it("drafts a session with a browsed folder and creates it on first message", async () => { const context = await browser.newContext({ locale: "en-US", @@ -1336,10 +1635,15 @@ describeControlUiE2e("Control UI new-session page mocked Gateway E2E", () => { try { await page.goto(`${server.baseUrl}new`); - await page.locator(".new-session-page__message").fill(message); + const composer = page.locator(".new-session-page__message"); + await composer.fill(message); + await pastePng(composer); await page.getByRole("button", { name: "Start session" }).click(); const create = await gateway.waitForRequest("sessions.create"); - expect(create.params).toMatchObject({ message }); + expect(create.params).toMatchObject({ + message, + attachments: [{ fileName: "pixel.png", content: ONE_PIXEL_PNG_B64 }], + }); await page.waitForURL((url) => url.searchParams.get("session") === sessionKey, { timeout: 30_000, @@ -1357,13 +1661,80 @@ describeControlUiE2e("Control UI new-session page mocked Gateway E2E", () => { await page.getByRole("button", { name: "Retry queued message" }).click(); const retry = await gateway.waitForRequest("chat.send"); - expect(retry.params).toMatchObject({ sessionKey, message }); + expect(retry.params).toMatchObject({ + sessionKey, + message, + attachments: [{ fileName: "pixel.png", content: ONE_PIXEL_PNG_B64 }], + }); expect(await gateway.getRequests("sessions.create")).toHaveLength(0); } finally { await context.close(); } }); + it("adopts a created session when rejected-turn persistence exceeds browser storage", async () => { + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + await page.addInitScript(() => { + const setItem = Object.getOwnPropertyDescriptor(Storage.prototype, "setItem") + ?.value as Storage["setItem"]; + Storage.prototype.setItem = function (key: string, value: string) { + if (key.startsWith("openclaw.control.chatComposer.v2:")) { + throw new DOMException("Quota exceeded", "QuotaExceededError"); + } + return setItem.call(this, key, value); + }; + }); + const sessionKey = "agent:main:storage-failed-initial-turn"; + const message = "retry this in the session that already exists"; + const runError = "initial send rejected"; + const gateway = await installMockGateway(page, { + methodResponses: { + "sessions.create": { + key: sessionKey, + runStarted: false, + runError: { code: "INVALID_REQUEST", message: runError }, + }, + "chat.history": { + messages: [], + sessionId: "storage-failed-initial-turn", + sessionInfo: { hasActiveRun: false, key: sessionKey, status: "done" }, + }, + "chat.send": { runId: "storage-failure-retry", status: "started" }, + }, + }); + + try { + await page.goto(`${server.baseUrl}new`); + const composer = page.locator(".new-session-page__message"); + await composer.fill(message); + await pastePng(composer); + await page.getByRole("button", { name: "Start session" }).click(); + + await page.waitForURL((url) => url.searchParams.get("session") === sessionKey, { + timeout: 30_000, + }); + await expect.poll(() => page.locator(".chat-queue__text").allInnerTexts()).toContain(message); + await expect + .poll(() => page.locator(".chat-queue__error").allInnerTexts()) + .toContain(runError); + await page.getByRole("button", { name: "Retry queued message" }).click(); + const retry = await gateway.waitForRequest("chat.send"); + expect(retry.params).toMatchObject({ + sessionKey, + message, + attachments: [{ fileName: "pixel.png", content: ONE_PIXEL_PNG_B64 }], + }); + expect(await gateway.getRequests("sessions.create")).toHaveLength(1); + } finally { + await context.close(); + } + }); + it("browses capable nodes and accepts manual paths for exec-only nodes", async () => { const context = await browser.newContext({ locale: "en-US", diff --git a/ui/src/i18n/.i18n/catalog-fallbacks.json b/ui/src/i18n/.i18n/catalog-fallbacks.json index ba6a75f84753..625010c52c15 100644 --- a/ui/src/i18n/.i18n/catalog-fallbacks.json +++ b/ui/src/i18n/.i18n/catalog-fallbacks.json @@ -1,5 +1,28 @@ { - "fallbacks": {}, - "sourceHash": "0c13460dca0cb1cfcf4d17d97316218cb003189bf09681a53dd54cedebb4225b", + "fallbacks": { + "newSession.readingAttachment": [ + "ar", + "de", + "es", + "fa", + "fr", + "hi", + "id", + "it", + "ja-JP", + "ko", + "nl", + "pl", + "pt-BR", + "ru", + "th", + "tr", + "uk", + "vi", + "zh-CN", + "zh-TW" + ] + }, + "sourceHash": "b61326c5179cddcc954999eae56871241d84cc9626f7013688e820db779258a5", "version": 1 } diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index b99f246d5162..730103cc9389 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -487,6 +487,7 @@ export const en: TranslationMap = { worktreeNamePlaceholder: "auto", worktreeNameInvalid: "Worktree names use lowercase letters, digits, and dashes.", messagePlaceholder: "What should this session work on?", + readingAttachment: "Reading attachment", start: "Start session", starting: "Starting…", createFailed: "Couldn't create the session.", diff --git a/ui/src/lib/sessions/create.ts b/ui/src/lib/sessions/create.ts index 7c969553d697..fbbe257c3192 100644 --- a/ui/src/lib/sessions/create.ts +++ b/ui/src/lib/sessions/create.ts @@ -24,6 +24,8 @@ export type SessionCreateParams = { cwd?: string; /** First message; the gateway creates the session and starts the run in one call. */ message?: string; + /** Attachments for the first message, using the chat.send wire format. */ + attachments?: unknown[]; task?: string; }; diff --git a/ui/src/pages/chat/attachment-api.ts b/ui/src/pages/chat/attachment-api.ts new file mode 100644 index 000000000000..dc647c607013 --- /dev/null +++ b/ui/src/pages/chat/attachment-api.ts @@ -0,0 +1,33 @@ +import type { ChatAttachment } from "../../lib/chat/chat-types.ts"; +import { getChatAttachmentDataUrl } from "./attachment-payload-store.ts"; + +function dataUrlToBase64(dataUrl: string): { content: string; mimeType: string } | null { + const match = /^data:([^;]+);base64,(.+)$/.exec(dataUrl); + if (!match) { + return null; + } + const mimeType = match[1]; + const content = match[2]; + return mimeType && content ? { mimeType, content } : null; +} + +/** Converts composer attachments into the base64 payload accepted by chat.send. */ +export function buildChatApiAttachments(attachments?: readonly ChatAttachment[]) { + return attachments?.length + ? attachments + .map((attachment) => { + const dataUrl = getChatAttachmentDataUrl(attachment); + const parsed = dataUrl ? dataUrlToBase64(dataUrl) : null; + if (!parsed) { + return null; + } + return { + type: parsed.mimeType.startsWith("image/") ? "image" : "file", + mimeType: parsed.mimeType, + fileName: attachment.fileName, + content: parsed.content, + }; + }) + .filter((attachment): attachment is NonNullable => attachment !== null) + : undefined; +} diff --git a/ui/src/pages/chat/chat-gateway.ts b/ui/src/pages/chat/chat-gateway.ts index eda0e787eb37..b90fe91dd7cb 100644 --- a/ui/src/pages/chat/chat-gateway.ts +++ b/ui/src/pages/chat/chat-gateway.ts @@ -21,6 +21,10 @@ import { clearToolStreamSegments, hasVisibleStreamParts, } from "./stream-reconciliation.ts"; +import { + authoritativeHistoryAppliedForRun, + rememberLiveTerminalRun, +} from "./terminal-message-identity.ts"; export type { ChatEventPayload } from "./chat-history.ts"; @@ -162,6 +166,11 @@ function handleChatEvent(state: ChatState, payload?: ChatEventPayload) { state.chatRunId !== null && typeof payload.runId === "string" && payload.runId === state.chatRunId; + const authoritativeTerminalMatches = Boolean( + payload.runId && + authoritativeHistoryAppliedForRun(state, payload.runId) && + chatEventSessionMatches(state, payload), + ); if (!sessionMatches && !activeRunMatches) { if (payload.state === "final") { const finalMessage = normalizeFinalAssistantMessage(payload.message); @@ -221,7 +230,11 @@ function handleChatEvent(state: ChatState, payload?: ChatEventPayload) { } } else if (payload.state === "final") { const finalMessage = normalizeFinalAssistantMessage(payload.message); - if (finalMessage && !shouldHideAssistantChatMessage(finalMessage)) { + if (authoritativeTerminalMatches) { + // History already owns this run's terminal message. Discard the live + // projection; reconcileTerminalRun below clears its remaining stream. + clearToolStreamSegments(state); + } else if (finalMessage && !shouldHideAssistantChatMessage(finalMessage)) { if ( hasVisibleStreamParts(state, { includeCurrent: false, @@ -233,7 +246,10 @@ function handleChatEvent(state: ChatState, payload?: ChatEventPayload) { }); clearToolStreamSegments(state); } - state.chatMessages = appendTerminalAssistantMessage(state.chatMessages, finalMessage); + state.chatMessages = appendTerminalAssistantMessage( + state.chatMessages, + rememberLiveTerminalRun(finalMessage, terminalRunId), + ); } else { state.chatMessages = materializeVisibleAssistantStreamMessages(state.chatMessages, state); } diff --git a/ui/src/pages/chat/chat-history-retry.ts b/ui/src/pages/chat/chat-history-retry.ts new file mode 100644 index 000000000000..f85b43855868 --- /dev/null +++ b/ui/src/pages/chat/chat-history-retry.ts @@ -0,0 +1,44 @@ +import { GatewayRequestError } from "../../api/gateway.ts"; + +const DEFAULT_RETRY_MS = 500; +const MAX_RETRY_MS = 5_000; + +export function isRetryableStartupUnavailable( + err: unknown, + method: string, +): err is GatewayRequestError { + if (!(err instanceof GatewayRequestError)) { + return false; + } + if (err.gatewayCode !== "UNAVAILABLE" || !err.retryable) { + return false; + } + const details = err.details; + if (!details || typeof details !== "object") { + return true; + } + const detailMethod = (details as { method?: unknown }).method; + return typeof detailMethod !== "string" || detailMethod === method; +} + +export function isUnknownGatewayMethodError( + err: unknown, + method: string, +): err is GatewayRequestError { + return ( + err instanceof GatewayRequestError && + err.gatewayCode === "INVALID_REQUEST" && + err.message.includes(`unknown method: ${method}`) + ); +} + +export function resolveStartupRetryDelayMs(err: GatewayRequestError): number { + const retryAfterMs = typeof err.retryAfterMs === "number" ? err.retryAfterMs : DEFAULT_RETRY_MS; + return Math.min(Math.max(retryAfterMs, 100), MAX_RETRY_MS); +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} diff --git a/ui/src/pages/chat/chat-history.ts b/ui/src/pages/chat/chat-history.ts index 0a49accba1f1..fbcdf82bfc93 100644 --- a/ui/src/pages/chat/chat-history.ts +++ b/ui/src/pages/chat/chat-history.ts @@ -1,10 +1,6 @@ // Control UI page module owns Chat transcript loading and selected-session message subscription. import type { CommandsListResult } from "../../../../packages/gateway-protocol/src/index.js"; -import { - GatewayRequestError, - type GatewayBrowserClient, - type GatewayHelloOk, -} from "../../api/gateway.ts"; +import type { GatewayBrowserClient, GatewayHelloOk } from "../../api/gateway.ts"; import type { AgentsListResult, GatewaySessionRow, @@ -47,6 +43,12 @@ import { resolveUiSelectedSessionAgentId, } from "../../lib/sessions/session-key.ts"; import { normalizeLowercaseStringOrEmpty } from "../../lib/string-coerce.ts"; +import { + isRetryableStartupUnavailable, + isUnknownGatewayMethodError, + resolveStartupRetryDelayMs, + sleep, +} from "./chat-history-retry.ts"; import { isLocallyOptimisticHistoryMessage, messageDisplaySignature, @@ -76,14 +78,13 @@ import { prunePersistedToolStreamMessages, visibleCurrentAssistantStreamTail, } from "./stream-reconciliation.ts"; +import { reconcileAuthoritativeTerminalHistory } from "./terminal-message-identity.ts"; const SILENT_REPLY_PATTERN = /^\s*NO_REPLY\s*$/; const SYNTHETIC_TRANSCRIPT_REPAIR_RESULT = "[openclaw] missing tool result in session history; inserted synthetic error result for transcript repair."; const CHAT_HISTORY_REQUEST_LIMIT = 100; const STARTUP_CHAT_HISTORY_RETRY_TIMEOUT_MS = 60_000; -const STARTUP_CHAT_HISTORY_DEFAULT_RETRY_MS = 500; -const STARTUP_CHAT_HISTORY_MAX_RETRY_MS = 5_000; const chatHistoryRequestVersions = new WeakMap(); const selectedSessionMessageSubscriptionGenerations = new WeakMap(); @@ -306,41 +307,6 @@ function collectLateOptimisticTailMessages( return lateTail; } -function isRetryableStartupUnavailable(err: unknown, method: string): err is GatewayRequestError { - if (!(err instanceof GatewayRequestError)) { - return false; - } - if (err.gatewayCode !== "UNAVAILABLE" || !err.retryable) { - return false; - } - const details = err.details; - if (!details || typeof details !== "object") { - return true; - } - const detailMethod = (details as { method?: unknown }).method; - return typeof detailMethod !== "string" || detailMethod === method; -} - -function isUnknownGatewayMethodError(err: unknown, method: string): err is GatewayRequestError { - return ( - err instanceof GatewayRequestError && - err.gatewayCode === "INVALID_REQUEST" && - err.message.includes(`unknown method: ${method}`) - ); -} - -function resolveStartupRetryDelayMs(err: GatewayRequestError): number { - const retryAfterMs = - typeof err.retryAfterMs === "number" ? err.retryAfterMs : STARTUP_CHAT_HISTORY_DEFAULT_RETRY_MS; - return Math.min(Math.max(retryAfterMs, 100), STARTUP_CHAT_HISTORY_MAX_RETRY_MS); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - export type ChatState = { client: GatewayBrowserClient | null; connected: boolean; @@ -1008,14 +974,21 @@ async function loadChatHistoryUncached( state.chatHistoryPagination = resolveChatHistoryPagination(res); applyChatAgentsList(state, res.agentsList, client); const visibleMessages = messages.filter((message) => !shouldHideHistoryMessage(message)); - const lateOptimisticTail = collectLateOptimisticTailMessages( + const reconciledTerminal = reconcileAuthoritativeTerminalHistory({ + currentMessages: state.chatMessages, + host: state, previousMessages, - state.chatMessages, + sessionKey, + visibleMessages, + }); + const lateOptimisticTail = collectLateOptimisticTailMessages( + reconciledTerminal.previousMessages, + reconciledTerminal.currentMessages, visibleMessages, ); state.chatMessages = preserveOptimisticTailMessages( visibleMessages, - previousMessages, + reconciledTerminal.previousMessages, shouldHideHistoryMessage, ); if (lateOptimisticTail.length > 0) { diff --git a/ui/src/pages/chat/chat-pane-state.ts b/ui/src/pages/chat/chat-pane-state.ts new file mode 100644 index 000000000000..89873bdbe421 --- /dev/null +++ b/ui/src/pages/chat/chat-pane-state.ts @@ -0,0 +1,19 @@ +import { resolveControlUiAuthToken } from "../../app/control-ui-auth.ts"; + +export function resolveAssistantAttachmentAuthToken(state: { + hello?: { auth?: { deviceToken?: string | null } | null } | null; + password?: string | null; + settings?: { token?: string | null } | null; +}) { + return resolveControlUiAuthToken(state); +} + +export function dismissChatError(state: { + chatError?: string | null; + lastError: string | null; + lastErrorCode?: string | null; +}) { + state.lastError = null; + state.lastErrorCode = null; + state.chatError = null; +} diff --git a/ui/src/pages/chat/chat-pane.ts b/ui/src/pages/chat/chat-pane.ts index f7ecb859ca74..0d377579d52e 100644 --- a/ui/src/pages/chat/chat-pane.ts +++ b/ui/src/pages/chat/chat-pane.ts @@ -72,6 +72,7 @@ import { syncSelectedSessionMessageSubscription, type ChatHistoryPagination, } from "./chat-history.ts"; +import { dismissChatError, resolveAssistantAttachmentAuthToken } from "./chat-pane-state.ts"; import { markQueuedChatSendsWaitingForReconnect } from "./chat-queue.ts"; import { dismissRealtimeTalkError } from "./chat-realtime.ts"; import { flushChatQueueForEvent, retryReconnectableQueuedChatSends } from "./chat-send.ts"; @@ -85,7 +86,6 @@ import { canCreateChatSession, ChatStateController, createPageState, - dismissChatError, handlePageGatewayEvent, refreshChatCommands, refreshChatMetadata, @@ -94,7 +94,6 @@ import { refreshRouteSessionOptions, resetChatStateForRouteSession, retryChatComposerMemoryFallback, - resolveAssistantAttachmentAuthToken, resolveChatAgentId, resolveChatAvatarUrl, saveRouteSessionSettings, @@ -102,12 +101,12 @@ import { } from "./chat-state.ts"; import { renderChat, resetChatViewState, type ChatProps } from "./chat-view.ts"; import { renderCatalogTerminalButton } from "./components/catalog-terminal-button.ts"; +import { chatAttachmentFromDataUrl } from "./components/chat-attachments.ts"; import { createBackgroundTasksProps, renderBackgroundTasksToggle, type BackgroundTasksProps, } from "./components/chat-background-tasks.ts"; -import { chatAttachmentFromDataUrl } from "./components/chat-composer.ts"; import { renderChatControls } from "./components/chat-controls.ts"; import { chatPullRequestId, diff --git a/ui/src/pages/chat/chat-queue.ts b/ui/src/pages/chat/chat-queue.ts index 4d31d461a164..19018cbc3b45 100644 --- a/ui/src/pages/chat/chat-queue.ts +++ b/ui/src/pages/chat/chat-queue.ts @@ -35,7 +35,7 @@ type ChatQueueSessionHost = ChatQueueStoreHost & sessionKey: string; }; -type ChatQueueScopedSessionHost = ChatQueueSessionHost & SessionScopeHost; +export type ChatQueueScopedSessionHost = ChatQueueSessionHost & SessionScopeHost; const chatOutboxProjectionHosts = new Set(); // Durable rows use crash-safe states. Overlay live work process-wide so panes @@ -48,7 +48,7 @@ const localRecoveryItemIds = new WeakMap // quota fallback may bypass durable admission on an explicit retry. const volatileQueueItemIds = new WeakMap>(); -function markLocalRecoveryItem(host: ChatQueueScopedSessionHost, id: string): void { +export function markLocalRecoveryItem(host: ChatQueueScopedSessionHost, id: string): void { const ids = localRecoveryItemIds.get(host) ?? new Set(); ids.add(id); localRecoveryItemIds.set(host, ids); @@ -66,7 +66,7 @@ export function isVolatileQueuedMessage(host: ChatQueueScopedSessionHost, id: st return volatileQueueItemIds.get(host)?.has(id) === true; } -function markVolatileQueuedMessage(host: ChatQueueScopedSessionHost, id: string): void { +export function markVolatileQueuedMessage(host: ChatQueueScopedSessionHost, id: string): void { const ids = volatileQueueItemIds.get(host) ?? new Set(); ids.add(id); volatileQueueItemIds.set(host, ids); @@ -384,7 +384,7 @@ export function replacePendingQueuedMessageProjection( return true; } -function writeChatQueueForScope( +export function writeChatQueueForScope( host: ChatQueueScopedSessionHost, sessionKey: string, queue: ChatQueueItem[], diff --git a/ui/src/pages/chat/chat-send.ts b/ui/src/pages/chat/chat-send.ts index 0147bd0df0e1..f2318c46d4a4 100644 --- a/ui/src/pages/chat/chat-send.ts +++ b/ui/src/pages/chat/chat-send.ts @@ -33,6 +33,7 @@ import { } from "../../lib/sessions/session-key.ts"; import { normalizeLowercaseStringOrEmpty } from "../../lib/string-coerce.ts"; import { generateUUID } from "../../lib/uuid.ts"; +import { buildChatApiAttachments } from "./attachment-api.ts"; import { discardChatAttachmentDataUrls, getChatAttachmentDataUrl, @@ -201,37 +202,6 @@ type ChatSendOptions = { onSideQuestionSendRejected?: () => void; }; -function dataUrlToBase64(dataUrl: string): { content: string; mimeType: string } | null { - const match = /^data:([^;]+);base64,(.+)$/.exec(dataUrl); - if (!match) { - return null; - } - const mimeType = match[1]; - const content = match[2]; - return mimeType && content ? { mimeType, content } : null; -} - -function buildApiAttachments(attachments?: ChatAttachment[]) { - const hasAttachments = attachments && attachments.length > 0; - return hasAttachments - ? attachments - .map((att) => { - const dataUrl = getChatAttachmentDataUrl(att); - const parsed = dataUrl ? dataUrlToBase64(dataUrl) : null; - if (!parsed) { - return null; - } - return { - type: parsed.mimeType.startsWith("image/") ? "image" : "file", - mimeType: parsed.mimeType, - fileName: att.fileName, - content: parsed.content, - }; - }) - .filter((a): a is NonNullable => a !== null) - : undefined; -} - function normalizeAckTimingValue(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; } @@ -295,7 +265,7 @@ async function requestChatSend( message: params.message, deliver: false, idempotencyKey: params.runId, - attachments: buildApiAttachments(params.attachments), + attachments: buildChatApiAttachments(params.attachments), }); if (controlUiReconnectResume) { state.reconnectResumeSessionId = null; diff --git a/ui/src/pages/chat/chat-state.ts b/ui/src/pages/chat/chat-state.ts index 87de96d4028a..ca4fdbab02cf 100644 --- a/ui/src/pages/chat/chat-state.ts +++ b/ui/src/pages/chat/chat-state.ts @@ -12,7 +12,6 @@ import { loadLocalAssistantIdentity, } from "../../app/assistant-identity.ts"; import type { ApplicationContext } from "../../app/context.ts"; -import { resolveControlUiAuthToken } from "../../app/control-ui-auth.ts"; import { loadLocalUserIdentity, loadSettings, @@ -111,6 +110,7 @@ import { storedChatOutboxScopeKey, type StoredChatOutboxScope, } from "./composer-persistence.ts"; +import { admitInitialTurnHandoff } from "./initial-turn-handoff.ts"; import { handleChatDraftChange, handleChatInputHistoryKey, @@ -135,6 +135,10 @@ import { scheduleCommittedChatScroll, } from "./scroll.ts"; import { cacheChatMessages, readChatMessagesFromCache } from "./session-message-cache.ts"; +import { + clearAuthoritativeTerminal, + rememberAuthoritativeTerminal, +} from "./terminal-message-identity.ts"; import { handleAgentEvent, handleSessionOperationEvent, @@ -302,16 +306,6 @@ export function canCreateChatSession( ); } -export function resolveAssistantAttachmentAuthToken(state: ChatPageHost) { - return resolveControlUiAuthToken(state); -} - -export function dismissChatError(state: ChatPageHost) { - state.lastError = null; - state.lastErrorCode = null; - state.chatError = null; -} - function saveChatQueueForSession(state: ChatPageHost, sessionKey: string) { const scope = resolveStoredChatOutboxScope(state, sessionKey); const scopeKey = storedChatOutboxScopeKey(scope); @@ -505,6 +499,7 @@ export function resetChatStateForRouteSession( state.chatAvatarSource = null; state.chatAvatarStatus = null; state.chatAvatarReason = null; + clearAuthoritativeTerminal(state); resetChatRealtimeConversation(state); state.chatQueue = restoreChatQueueForSession(state, sessionKey); restoreChatComposerState(state); @@ -512,12 +507,13 @@ export function resetChatStateForRouteSession( // projection without rendering through the old route's persistence owner. // switchPaneSession requests an update only after adopting the new baseline. syncVisibleChatQueueProjection(state, { requestUpdate: false }); + const initialTurn = admitInitialTurnHandoff(state, sessionKey); const { fallback } = resolveChatComposerMemoryFallback(state, sessionKey); if (fallback) { state.chatMessage = fallback.message; state.chatAttachments = [...fallback.attachments]; } - const restoredStorageFailure = fallback?.storageFailed === true; + const restoredStorageFailure = fallback?.storageFailed === true || initialTurn; if (options.previousDraftRetry || restoredStorageFailure) { state.lastError = CHAT_COMPOSER_DRAFT_STORAGE_ERROR; state.chatError = CHAT_COMPOSER_DRAFT_STORAGE_ERROR; @@ -1050,6 +1046,7 @@ function handleSessionMessageEvent(state: ChatPageHost, payload: unknown) { state.selectedChatSessionArchived = event.archived; } const runIdBeforeApply = state.chatRunId; + rememberAuthoritativeTerminal({ event, host: state, matchesChat, payload, runIdBeforeApply }); const result = reconcileSessionEvent(state, payload); if (runIdBeforeApply && matchesChat) { const runId = event.clientRunId ?? event.runId ?? runIdBeforeApply; diff --git a/ui/src/pages/chat/chat-view.ts b/ui/src/pages/chat/chat-view.ts index c32fb15be702..e31e08a88eb3 100644 --- a/ui/src/pages/chat/chat-view.ts +++ b/ui/src/pages/chat/chat-view.ts @@ -21,13 +21,13 @@ import type { ChatSideResult, ChatSideResultPending } from "../../lib/chat/side- import type { EmbedSandboxMode } from "../../lib/chat/tool-display.ts"; import type { ProviderUsageDisplayProps } from "../../lib/provider-quota-summary.ts"; import type { UiSessionDefaultsHost } from "../../lib/sessions/session-key.ts"; +import { handleChatAttachmentDrop } from "./components/chat-attachments.ts"; import { renderBackgroundTasksRail, renderBackgroundTasksToggle, type BackgroundTasksProps, } from "./components/chat-background-tasks.ts"; import { - handleChatAttachmentDrop, isChatRunWorking, renderChatComposer, resetChatComposerState, diff --git a/ui/src/pages/chat/components/chat-attachments.ts b/ui/src/pages/chat/components/chat-attachments.ts new file mode 100644 index 000000000000..32ebe22bed92 --- /dev/null +++ b/ui/src/pages/chat/components/chat-attachments.ts @@ -0,0 +1,484 @@ +// Shared attachment controls for chat and new-session composers. +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { html, nothing } from "lit"; +import { icons } from "../../../components/icons.ts"; +import "../../../components/tooltip.ts"; +import "../../../components/web-awesome.ts"; +import { t } from "../../../i18n/index.ts"; +import type { ChatAttachment } from "../../../lib/chat/chat-types.ts"; +import { + getChatAttachmentDataUrl, + getChatAttachmentPreviewUrl, + registerChatAttachmentPayload, + releaseChatAttachmentPayload, +} from "../attachment-payload-store.ts"; + +const CHAT_ATTACHMENT_ACCEPT = + "image/*,audio/*,application/pdf,text/*,.csv,.json,.md,.txt,.zip," + + ".doc,.docx,.xls,.xlsx,.ppt,.pptx"; +const LARGE_PASTE_TEXT_THRESHOLD = 1000; +const LARGE_PASTE_TEXT_MIME_TYPE = "text/plain"; +const LARGE_PASTE_TEXT_FILE_PREFIX = "pasted-text-"; +const PASTED_TEXT_PREVIEW_MAX_LENGTH = 20; +const largePastedTextAttachments = new WeakSet(); +const pastedTextPreviews = new WeakMap(); + +export type ChatAttachmentControlsProps = { + attachments?: ChatAttachment[]; + disabled?: boolean; + getAttachments?: () => ChatAttachment[]; + draft?: string; + getDraft?: () => string; + onAttachmentsChange?: (attachments: ChatAttachment[]) => void; + onDraftChange?: (next: string) => void; + onPendingReadsChange?: (delta: 1 | -1) => void; + onRequestUpdate?: () => void; + readSignal?: AbortSignal; +}; + +function currentAttachments(props: ChatAttachmentControlsProps): ChatAttachment[] { + return props.getAttachments?.() ?? props.attachments ?? []; +} + +function isSupportedChatAttachmentFile(file: Pick): boolean { + if (file.type.startsWith("video/")) { + return false; + } + return !/\.(?:avi|m4v|mov|mp4|mpeg|mpg|webm)$/i.test(file.name); +} + +function clickComposerInput(target: HTMLElement, selector: string) { + target.closest("details")?.removeAttribute("open"); + target + .closest(".agent-chat__composer-shell, .new-session-page__composer") + ?.querySelector(selector) + ?.click(); +} + +function generateAttachmentId(): string { + return `att-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; +} + +function chatAttachmentFromFile(file: File, dataUrl: string): ChatAttachment { + const attachment = { + id: generateAttachmentId(), + mimeType: file.type || "application/octet-stream", + fileName: file.name || undefined, + sizeBytes: file.size, + }; + return registerChatAttachmentPayload({ attachment, dataUrl, file }); +} + +export function isLargePastedTextAttachment(attachment: ChatAttachment): boolean { + return largePastedTextAttachments.has(attachment); +} + +function encodeTextAsDataUrl(text: string): string { + const bytes = new TextEncoder().encode(text); + const chunks: string[] = []; + const chunkSize = 0x8000; + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + chunks.push(String.fromCharCode(...bytes.subarray(offset, offset + chunkSize))); + } + return `data:${LARGE_PASTE_TEXT_MIME_TYPE};base64,${btoa(chunks.join(""))}`; +} + +function createLargePastedTextAttachment(text: string): ChatAttachment { + const file = new File([text], `${LARGE_PASTE_TEXT_FILE_PREFIX}${Date.now()}.txt`, { + type: LARGE_PASTE_TEXT_MIME_TYPE, + }); + const attachment = chatAttachmentFromFile(file, encodeTextAsDataUrl(text)); + largePastedTextAttachments.add(attachment); + const preview = compactPastedTextPreview(text); + if (preview) { + pastedTextPreviews.set(attachment, preview); + } + return attachment; +} + +function readTextFromDataUrl(dataUrl: string): string | null { + const match = /^data:([^,]*),(.*)$/s.exec(dataUrl); + if (!match) { + return null; + } + const metadata = match[1]; + const payload = match[2]; + if (metadata === undefined || payload === undefined) { + return null; + } + if (metadata.toLowerCase().includes(";base64")) { + try { + const binary = atob(payload); + const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0)); + return new TextDecoder().decode(bytes); + } catch { + return null; + } + } + try { + return decodeURIComponent(payload.replace(/\+/g, "%20")); + } catch { + return null; + } +} + +function compactPastedTextPreview(text: string): string | null { + const normalized = text.replace(/\s+/gu, " ").trim(); + if (!normalized) { + return null; + } + if (normalized.length <= PASTED_TEXT_PREVIEW_MAX_LENGTH) { + return normalized; + } + return `${truncateUtf16Safe(normalized, PASTED_TEXT_PREVIEW_MAX_LENGTH).trimEnd()}...`; +} + +function pastedTextPreview(attachment: ChatAttachment): string { + return pastedTextPreviews.get(attachment) ?? attachment.fileName ?? "Attached file"; +} + +function appendPastedTextToDraft(draft: string, text: string): string { + if (!draft.trim()) { + return text; + } + return `${draft.replace(/\s+$/u, "")}\n\n${text}`; +} + +function handleLargeTextPaste(e: ClipboardEvent, props: ChatAttachmentControlsProps): boolean { + if (!props.onAttachmentsChange) { + return false; + } + const text = e.clipboardData?.getData("text/plain"); + if (!text || text.length <= LARGE_PASTE_TEXT_THRESHOLD) { + return false; + } + e.preventDefault(); + const attachment = createLargePastedTextAttachment(text); + props.onAttachmentsChange([...currentAttachments(props), attachment]); + return true; +} + +function dataImageClipboardFile( + dataUrl: string, + baseName = "pasted-image", +): { file: File; dataUrl: string } | null { + const match = /^\s*data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)\s*$/i.exec(dataUrl); + if (!match) { + return null; + } + const mimeType = match[1]?.toLowerCase(); + const base64Source = match[2]; + if (!mimeType || !base64Source) { + return null; + } + if (!isSupportedChatAttachmentFile({ name: baseName, type: mimeType })) { + return null; + } + const base64 = base64Source.replace(/\s+/g, ""); + try { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + const extension = mimeType.split("/")[1]?.replace(/[^a-z0-9.+-]/gi, "") || "png"; + return { + file: new File([bytes], `${baseName}.${extension}`, { type: mimeType }), + dataUrl: `data:${mimeType};base64,${base64}`, + }; + } catch { + return null; + } +} + +/** Builds a registered chat attachment from a base64 image data URL. */ +export function chatAttachmentFromDataUrl( + dataUrl: string, + fileName: string, +): ChatAttachment | null { + const baseName = fileName.replace(/\.[a-z0-9]+$/i, "") || "image"; + const parsed = dataImageClipboardFile(dataUrl, baseName); + return parsed ? chatAttachmentFromFile(parsed.file, parsed.dataUrl) : null; +} + +function readAttachmentFile( + file: File, + props: ChatAttachmentControlsProps, +): Promise { + if (props.readSignal?.aborted) { + return Promise.resolve(null); + } + props.onPendingReadsChange?.(1); + return new Promise((resolve) => { + const reader = new FileReader(); + let settled = false; + const finish = (attachment: ChatAttachment | null) => { + if (settled) { + return; + } + settled = true; + props.readSignal?.removeEventListener("abort", abort); + props.onPendingReadsChange?.(-1); + resolve(attachment); + }; + const abort = () => { + reader.abort(); + finish(null); + }; + props.readSignal?.addEventListener("abort", abort, { once: true }); + reader.addEventListener("error", () => finish(null), { once: true }); + reader.addEventListener("abort", () => finish(null), { once: true }); + reader.addEventListener( + "load", + () => { + const dataUrl = typeof reader.result === "string" ? reader.result : null; + finish( + dataUrl && !props.readSignal?.aborted ? chatAttachmentFromFile(file, dataUrl) : null, + ); + }, + { once: true }, + ); + reader.readAsDataURL(file); + }); +} + +async function appendAttachmentFiles(files: readonly File[], props: ChatAttachmentControlsProps) { + const supported = files.filter(isSupportedChatAttachmentFile); + if (!props.onAttachmentsChange || supported.length === 0) { + return; + } + const additions = ( + await Promise.all(supported.map((file) => readAttachmentFile(file, props))) + ).filter((attachment): attachment is ChatAttachment => attachment !== null); + if (props.readSignal?.aborted) { + for (const attachment of additions) { + releaseChatAttachmentPayload(attachment.id); + } + return; + } + if (additions.length === 0) { + return; + } + props.onAttachmentsChange([...currentAttachments(props), ...additions]); +} + +export function handleChatAttachmentPaste(e: ClipboardEvent, props: ChatAttachmentControlsProps) { + const items = e.clipboardData?.items; + if (!items || !props.onAttachmentsChange) { + return; + } + const imageFiles = Array.from(items) + .filter((item) => item.type.startsWith("image/")) + .map((item) => item.getAsFile()) + .filter((file): file is File => file !== null); + if (imageFiles.length === 0) { + const text = e.clipboardData?.getData("text/plain"); + const pasted = text ? dataImageClipboardFile(text) : null; + if (!pasted) { + handleLargeTextPaste(e, props); + return; + } + e.preventDefault(); + props.onAttachmentsChange([ + ...currentAttachments(props), + chatAttachmentFromFile(pasted.file, pasted.dataUrl), + ]); + return; + } + e.preventDefault(); + void appendAttachmentFiles(imageFiles, props); +} + +function showPastedTextInComposer(att: ChatAttachment, props: ChatAttachmentControlsProps): void { + const dataUrl = getChatAttachmentDataUrl(att); + const text = dataUrl ? readTextFromDataUrl(dataUrl) : null; + if (!text || !props.onDraftChange) { + return; + } + const nextAttachments = currentAttachments(props).filter( + (attachment) => attachment.id !== att.id, + ); + releaseChatAttachmentPayload(att.id); + props.onAttachmentsChange?.(nextAttachments); + props.onDraftChange(appendPastedTextToDraft(props.getDraft?.() ?? props.draft ?? "", text)); + props.onRequestUpdate?.(); +} + +function handleChatAttachmentFileSelect(e: Event, props: ChatAttachmentControlsProps) { + const input = e.target as HTMLInputElement; + const files = [...(input.files ?? [])]; + input.value = ""; + void appendAttachmentFiles(files, props); +} + +export function handleChatAttachmentDrop(e: DragEvent, props: ChatAttachmentControlsProps) { + e.preventDefault(); + void appendAttachmentFiles([...(e.dataTransfer?.files ?? [])], props); +} + +export function renderChatAttachmentInputs(props: ChatAttachmentControlsProps) { + return html` + { + if (!props.disabled) { + handleChatAttachmentFileSelect(event, props); + } + }} + /> + { + if (!props.disabled) { + handleChatAttachmentFileSelect(event, props); + } + }} + /> + { + if (!props.disabled) { + handleChatAttachmentFileSelect(event, props); + } + }} + /> + `; +} + +export function renderChatAttachmentMenu(props: ChatAttachmentControlsProps) { + return html` + ) => { + const menu = event.currentTarget as HTMLElement; + const selector = + event.detail.item.value === "camera" + ? ".agent-chat__camera-input" + : event.detail.item.value === "photo" + ? ".agent-chat__photo-input" + : event.detail.item.value === "file" + ? ".agent-chat__file-input" + : null; + if (selector) { + clickComposerInput(menu, selector); + } + }} + > + + + + ${t("chat.composer.takePhoto")} + + + + ${t("chat.composer.attachPhoto")} + + + + ${t("chat.composer.attachFileOption")} + + + `; +} + +export function renderAttachmentPreview(props: ChatAttachmentControlsProps) { + const attachments = props.attachments ?? []; + if (attachments.length === 0) { + return nothing; + } + return html` +
+ ${attachments.map( + (att) => html` +
+ ${att.mimeType.startsWith("image/") && getChatAttachmentPreviewUrl(att) + ? html`Attachment preview` + : isLargePastedTextAttachment(att) + ? html` +
+ ${icons.fileText} + + ${pastedTextPreview(att)} + + +
+ ` + : html` + +
+ ${icons.paperclip} + ${att.fileName ?? "Attached file"} +
+ + `} + + + +
+ `, + )} +
+ `; +} diff --git a/ui/src/pages/chat/components/chat-composer.ts b/ui/src/pages/chat/components/chat-composer.ts index b03a1c29c797..260f1f94d555 100644 --- a/ui/src/pages/chat/components/chat-composer.ts +++ b/ui/src/pages/chat/components/chat-composer.ts @@ -38,12 +38,6 @@ import { goalElapsedMs, } from "../../../lib/session-goal.ts"; import { detectTextDirection } from "../../../lib/text-direction.ts"; -import { - getChatAttachmentDataUrl, - getChatAttachmentPreviewUrl, - registerChatAttachmentPayload, - releaseChatAttachmentPayload, -} from "../attachment-payload-store.ts"; import { exportChatMarkdown } from "../export.ts"; import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "../input-history.ts"; import type { RealtimeTalkConversationEntry } from "../realtime-talk-conversation.ts"; @@ -51,6 +45,13 @@ import type { RealtimeTalkLevelSignal } from "../realtime-talk-level.ts"; import type { RealtimeTalkStatus } from "../realtime-talk.ts"; import { CHAT_RUN_STATUS_TOAST_DURATION_MS, type ChatRunUiStatus } from "../run-lifecycle.ts"; import type { CompactionStatus, FallbackStatus } from "../tool-stream.ts"; +import { + handleChatAttachmentPaste, + isLargePastedTextAttachment, + renderAttachmentPreview, + renderChatAttachmentInputs, + renderChatAttachmentMenu, +} from "./chat-attachments.ts"; import { renderChatVoiceError, renderMicrophoneActivity, @@ -74,15 +75,6 @@ const COMPOSER_CHROME_INTERACTIVE_SELECTOR = [ "[role='listbox']", "[role='option']", ].join(","); -const CHAT_ATTACHMENT_ACCEPT = - "image/*,audio/*,application/pdf,text/*,.csv,.json,.md,.txt,.zip," + - ".doc,.docx,.xls,.xlsx,.ppt,.pptx"; -const LARGE_PASTE_TEXT_THRESHOLD = 1000; -const LARGE_PASTE_TEXT_MIME_TYPE = "text/plain"; -const LARGE_PASTE_TEXT_FILE_PREFIX = "pasted-text-"; -const PASTED_TEXT_PREVIEW_MAX_LENGTH = 20; -const largePastedTextAttachments = new WeakSet(); -const pastedTextPreviews = new WeakMap(); type ChatComposerProps = { paneId: string; @@ -946,20 +938,6 @@ function renderSlashMenu( `; } -type ChatAttachmentControlsProps = { - attachments?: ChatAttachment[]; - getAttachments?: () => ChatAttachment[]; - draft?: string; - getDraft?: () => string; - onAttachmentsChange?: (attachments: ChatAttachment[]) => void; - onDraftChange?: (next: string) => void; - onRequestUpdate?: () => void; -}; - -function currentAttachments(props: ChatAttachmentControlsProps): ChatAttachment[] { - return props.getAttachments?.() ?? props.attachments ?? []; -} - type ChatQueueProps = { queue: ChatQueueItem[]; canAbort?: boolean; @@ -1077,355 +1055,6 @@ function renderChatQueueItem(item: ChatQueueItem, props: ChatQueueProps) { `; } -function isSupportedChatAttachmentFile(file: Pick): boolean { - if (file.type.startsWith("video/")) { - return false; - } - return !/\.(?:avi|m4v|mov|mp4|mpeg|mpg|webm)$/i.test(file.name); -} - -function clickComposerInput(target: HTMLElement, selector: string) { - target.closest("details")?.removeAttribute("open"); - target.closest(".agent-chat__composer-shell")?.querySelector(selector)?.click(); -} - -function clickComposerFileInput(target: HTMLElement) { - clickComposerInput(target, ".agent-chat__file-input"); -} - -function clickComposerPhotoInput(target: HTMLElement) { - clickComposerInput(target, ".agent-chat__photo-input"); -} - -function clickComposerCameraInput(target: HTMLElement) { - clickComposerInput(target, ".agent-chat__camera-input"); -} - -function generateAttachmentId(): string { - return `att-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; -} - -function chatAttachmentFromFile(file: File, dataUrl: string): ChatAttachment { - const attachment = { - id: generateAttachmentId(), - mimeType: file.type || "application/octet-stream", - fileName: file.name || undefined, - sizeBytes: file.size, - }; - return registerChatAttachmentPayload({ attachment, dataUrl, file }); -} - -function isLargePastedTextAttachment(attachment: ChatAttachment): boolean { - return largePastedTextAttachments.has(attachment); -} - -function encodeTextAsDataUrl(text: string): string { - const bytes = new TextEncoder().encode(text); - const chunks: string[] = []; - const chunkSize = 0x8000; - for (let offset = 0; offset < bytes.length; offset += chunkSize) { - chunks.push(String.fromCharCode(...bytes.subarray(offset, offset + chunkSize))); - } - return `data:${LARGE_PASTE_TEXT_MIME_TYPE};base64,${btoa(chunks.join(""))}`; -} - -function createLargePastedTextAttachment(text: string): ChatAttachment { - const file = new File([text], `${LARGE_PASTE_TEXT_FILE_PREFIX}${Date.now()}.txt`, { - type: LARGE_PASTE_TEXT_MIME_TYPE, - }); - const attachment = chatAttachmentFromFile(file, encodeTextAsDataUrl(text)); - largePastedTextAttachments.add(attachment); - const preview = compactPastedTextPreview(text); - if (preview) { - pastedTextPreviews.set(attachment, preview); - } - return attachment; -} - -function readTextFromDataUrl(dataUrl: string): string | null { - const match = /^data:([^,]*),(.*)$/s.exec(dataUrl); - if (!match) { - return null; - } - const metadata = match[1]; - const payload = match[2]; - if (metadata === undefined || payload === undefined) { - return null; - } - if (metadata.toLowerCase().includes(";base64")) { - try { - const binary = atob(payload); - const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0)); - return new TextDecoder().decode(bytes); - } catch { - return null; - } - } - try { - return decodeURIComponent(payload.replace(/\+/g, "%20")); - } catch { - return null; - } -} - -function compactPastedTextPreview(text: string): string | null { - const normalized = text.replace(/\s+/gu, " ").trim(); - if (!normalized) { - return null; - } - if (normalized.length <= PASTED_TEXT_PREVIEW_MAX_LENGTH) { - return normalized; - } - return `${truncateUtf16Safe(normalized, PASTED_TEXT_PREVIEW_MAX_LENGTH).trimEnd()}...`; -} - -function pastedTextPreview(attachment: ChatAttachment): string { - return pastedTextPreviews.get(attachment) ?? attachment.fileName ?? "Attached file"; -} - -function appendPastedTextToDraft(draft: string, text: string): string { - if (!draft.trim()) { - return text; - } - return `${draft.replace(/\s+$/u, "")}\n\n${text}`; -} - -function handleLargeTextPaste(e: ClipboardEvent, props: ChatAttachmentControlsProps): boolean { - if (!props.onAttachmentsChange) { - return false; - } - const text = e.clipboardData?.getData("text/plain"); - if (!text || text.length <= LARGE_PASTE_TEXT_THRESHOLD) { - return false; - } - e.preventDefault(); - const attachment = createLargePastedTextAttachment(text); - props.onAttachmentsChange([...currentAttachments(props), attachment]); - return true; -} - -function dataImageClipboardFile( - dataUrl: string, - baseName = "pasted-image", -): { file: File; dataUrl: string } | null { - const match = /^\s*data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)\s*$/i.exec(dataUrl); - if (!match) { - return null; - } - const mimeType = match[1]?.toLowerCase(); - const base64Source = match[2]; - if (!mimeType || !base64Source) { - return null; - } - if (!isSupportedChatAttachmentFile({ name: baseName, type: mimeType })) { - return null; - } - const base64 = base64Source.replace(/\s+/g, ""); - try { - const binary = atob(base64); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) { - bytes[i] = binary.charCodeAt(i); - } - const extension = mimeType.split("/")[1]?.replace(/[^a-z0-9.+-]/gi, "") || "png"; - return { - file: new File([bytes], `${baseName}.${extension}`, { type: mimeType }), - dataUrl: `data:${mimeType};base64,${base64}`, - }; - } catch { - return null; - } -} - -/** Builds a registered chat attachment from a base64 image data URL (e.g. a browser-panel annotation). */ -export function chatAttachmentFromDataUrl( - dataUrl: string, - fileName: string, -): ChatAttachment | null { - const baseName = fileName.replace(/\.[a-z0-9]+$/i, "") || "image"; - const parsed = dataImageClipboardFile(dataUrl, baseName); - return parsed ? chatAttachmentFromFile(parsed.file, parsed.dataUrl) : null; -} - -function isImageAttachment(att: ChatAttachment): boolean { - return att.mimeType.startsWith("image/"); -} - -function handleChatAttachmentPaste(e: ClipboardEvent, props: ChatAttachmentControlsProps) { - const items = e.clipboardData?.items; - if (!items || !props.onAttachmentsChange) { - return; - } - const imageItems: DataTransferItem[] = []; - for (const item of Array.from(items)) { - if (item.type.startsWith("image/")) { - imageItems.push(item); - } - } - if (imageItems.length === 0) { - const text = e.clipboardData?.getData("text/plain"); - const pasted = text ? dataImageClipboardFile(text) : null; - if (!pasted) { - handleLargeTextPaste(e, props); - return; - } - e.preventDefault(); - props.onAttachmentsChange([ - ...currentAttachments(props), - chatAttachmentFromFile(pasted.file, pasted.dataUrl), - ]); - return; - } - e.preventDefault(); - for (const item of imageItems) { - const file = item.getAsFile(); - if (!file) { - continue; - } - const reader = new FileReader(); - reader.addEventListener("load", () => { - const dataUrl = reader.result as string; - const newAttachment = chatAttachmentFromFile(file, dataUrl); - props.onAttachmentsChange?.([...currentAttachments(props), newAttachment]); - }); - reader.readAsDataURL(file); - } -} - -function showPastedTextInComposer(att: ChatAttachment, props: ChatAttachmentControlsProps): void { - const dataUrl = getChatAttachmentDataUrl(att); - const text = dataUrl ? readTextFromDataUrl(dataUrl) : null; - if (!text || !props.onDraftChange) { - return; - } - const nextAttachments = currentAttachments(props).filter( - (attachment) => attachment.id !== att.id, - ); - releaseChatAttachmentPayload(att.id); - props.onAttachmentsChange?.(nextAttachments); - props.onDraftChange(appendPastedTextToDraft(props.getDraft?.() ?? props.draft ?? "", text)); - props.onRequestUpdate?.(); -} - -function handleChatAttachmentFileSelect(e: Event, props: ChatAttachmentControlsProps) { - const input = e.target as HTMLInputElement; - if (!input.files || !props.onAttachmentsChange) { - return; - } - const additions: ChatAttachment[] = []; - let pending = 0; - for (const file of input.files) { - if (!isSupportedChatAttachmentFile(file)) { - continue; - } - pending++; - const reader = new FileReader(); - reader.addEventListener("load", () => { - additions.push(chatAttachmentFromFile(file, reader.result as string)); - pending--; - if (pending === 0) { - props.onAttachmentsChange?.([...currentAttachments(props), ...additions]); - } - }); - reader.readAsDataURL(file); - } - input.value = ""; -} - -export function handleChatAttachmentDrop(e: DragEvent, props: ChatAttachmentControlsProps) { - e.preventDefault(); - const files = e.dataTransfer?.files; - if (!files || !props.onAttachmentsChange) { - return; - } - const additions: ChatAttachment[] = []; - let pending = 0; - for (const file of files) { - if (!isSupportedChatAttachmentFile(file)) { - continue; - } - pending++; - const reader = new FileReader(); - reader.addEventListener("load", () => { - additions.push(chatAttachmentFromFile(file, reader.result as string)); - pending--; - if (pending === 0) { - props.onAttachmentsChange?.([...currentAttachments(props), ...additions]); - } - }); - reader.readAsDataURL(file); - } -} - -function renderAttachmentPreview(props: ChatAttachmentControlsProps) { - const attachments = props.attachments ?? []; - if (attachments.length === 0) { - return nothing; - } - return html` -
- ${attachments.map( - (att) => html` -
- ${isImageAttachment(att) && getChatAttachmentPreviewUrl(att) - ? html`Attachment preview` - : isLargePastedTextAttachment(att) - ? html` -
- ${icons.fileText} - - ${pastedTextPreview(att)} - - -
- ` - : html` - -
- ${icons.paperclip} - ${att.fileName ?? "Attached file"} -
- - `} - - - -
- `, - )} -
- `; -} - type ComposerRunStatus = | ChatRunUiStatus | { @@ -2564,43 +2193,7 @@ export function renderChatComposer(props: ChatComposerProps) { })} - { - if (canCompose) { - handleChatAttachmentFileSelect(event, props); - } - }} - /> - { - if (canCompose) { - handleChatAttachmentFileSelect(event, props); - } - }} - /> - { - if (canCompose) { - handleChatAttachmentFileSelect(event, props); - } - }} - /> - + ${renderChatAttachmentInputs({ ...props, disabled: !canCompose })} ${renderChatVoiceError({ status: props.realtimeTalkStatus, detail: props.realtimeTalkDetail, @@ -2608,55 +2201,7 @@ export function renderChatComposer(props: ChatComposerProps) { })}
- ) => { - const menu = event.currentTarget as HTMLElement; - switch (event.detail.item.value) { - case "camera": - clickComposerCameraInput(menu); - break; - case "photo": - clickComposerPhotoInput(menu); - break; - case "file": - clickComposerFileInput(menu); - break; - case undefined: - break; - } - }} - > - - - - ${t("chat.composer.takePhoto")} - - - - ${t("chat.composer.attachPhoto")} - - - - ${t("chat.composer.attachFileOption")} - - + ${renderChatAttachmentMenu({ ...props, disabled: !canCompose })}
-
-
- - - +
+
+ ${renderChatAttachmentInputs(attachmentProps)} ${renderAttachmentPreview(attachmentProps)} +
+ ${renderChatAttachmentMenu(attachmentProps)} +
+ +
+
+ + + +
+ ${options.modelControl && options.modelControl !== nothing + ? html`
+
${options.modelControl}
+
` + : nothing} + ${options.pendingAttachmentReads > 0 + ? html`${t("newSession.readingAttachment")}` + : nothing}
`; } + +export function renderNewSessionDraftComposer(options: { + agentDefaultModel?: string; + agentId: string; + attachmentDraft: NewSessionAttachmentDraft; + canSubmit: boolean; + context: import("../../app/context.ts").ApplicationContext | undefined; + isCatalogTarget: boolean; + message: string; + modelControl: NewSessionModelControl; + requiresModifier: boolean; + submitting: boolean; + onInput: (message: string) => void; + onSubmit: () => void; +}) { + const readSignal = options.attachmentDraft.readSignal; + return renderNewSessionComposer({ + attachments: options.attachmentDraft.attachments, + canSubmit: options.canSubmit, + getAttachments: () => options.attachmentDraft.attachments, + message: options.message, + modelControl: options.isCatalogTarget + ? nothing + : options.modelControl.render({ + agentDefaultModel: options.agentDefaultModel, + agentId: options.agentId, + context: options.context, + sending: options.submitting, + }), + pendingAttachmentReads: options.attachmentDraft.pendingReads, + readSignal, + requiresModifier: options.requiresModifier, + submitting: options.submitting, + onAttachmentsChange: (attachments) => { + if (!options.submitting) { + options.attachmentDraft.replace(attachments); + } + }, + onPendingReadsChange: (delta) => options.attachmentDraft.updatePending(readSignal, delta), + onInput: options.onInput, + onSubmit: options.onSubmit, + }); +} diff --git a/ui/src/pages/new-session/create-params.test.ts b/ui/src/pages/new-session/create-params.test.ts index 56352ce45fb9..0553d72f36a9 100644 --- a/ui/src/pages/new-session/create-params.test.ts +++ b/ui/src/pages/new-session/create-params.test.ts @@ -16,6 +16,51 @@ describe("buildDraftSessionCreateParams", () => { ).toEqual({ agentId: "main", message: "hello" }); }); + it("includes initial-message attachments", () => { + const attachments = [ + { type: "image", mimeType: "image/png", fileName: "pixel.png", content: "aGVsbG8=" }, + ]; + expect( + buildDraftSessionCreateParams({ + agentId: "main", + message: "", + attachments, + worktree: false, + }), + ).toEqual({ agentId: "main", message: "", attachments }); + }); + + it("includes a selected model for a plain session", () => { + expect( + buildDraftSessionCreateParams({ + agentId: "main", + message: "use the selected model", + model: "anthropic/claude-sonnet-4-6", + worktree: false, + }), + ).toEqual({ + agentId: "main", + message: "use the selected model", + model: "anthropic/claude-sonnet-4-6", + }); + }); + + it("does not combine a catalog target with a draft model override", () => { + expect( + buildDraftSessionCreateParams({ + agentId: "main", + message: "start coding", + model: "openai/gpt-5.5", + worktree: false, + catalogId: "claude", + }), + ).toEqual({ + agentId: "main", + message: "start coding", + catalogId: "claude", + }); + }); + it("submits the catalog target for server-side resolution", () => { expect( buildDraftSessionCreateParams({ diff --git a/ui/src/pages/new-session/create-params.ts b/ui/src/pages/new-session/create-params.ts index 5052b4940851..38d854edfa38 100644 --- a/ui/src/pages/new-session/create-params.ts +++ b/ui/src/pages/new-session/create-params.ts @@ -1,10 +1,19 @@ import { normalizeAgentId } from "../../lib/sessions/session-key.ts"; import { normalizeOptionalString } from "../../lib/string-coerce.ts"; +const WORKTREE_NAME_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/; + +export function isWorktreeNameValid(value: string): boolean { + const name = value.trim(); + return !name || WORKTREE_NAME_PATTERN.test(name); +} + /** Maps the new-session draft selections onto additive sessions.create params. */ export function buildDraftSessionCreateParams(draft: { agentId: string; message: string; + model?: string; + attachments?: unknown[]; worktree: boolean; baseRef?: string; worktreeName?: string; @@ -17,11 +26,14 @@ export function buildDraftSessionCreateParams(draft: { const workspace = normalizeOptionalString(draft.workspace); const execNode = normalizeOptionalString(draft.execNode); const catalogId = normalizeOptionalString(draft.catalogId); + const model = normalizeOptionalString(draft.model); const customFolder = cwd && cwd !== workspace ? cwd : undefined; return { agentId: normalizeAgentId(draft.agentId), message: draft.message, + ...(draft.attachments?.length ? { attachments: draft.attachments } : {}), ...(catalogId ? { catalogId } : {}), + ...(!catalogId && model ? { model } : {}), ...(draft.worktree ? { worktree: true, diff --git a/ui/src/pages/new-session/model-control.ts b/ui/src/pages/new-session/model-control.ts new file mode 100644 index 000000000000..0386515308eb --- /dev/null +++ b/ui/src/pages/new-session/model-control.ts @@ -0,0 +1,92 @@ +import type { ModelCatalogEntry } from "../../api/types.ts"; +import type { ApplicationContext } from "../../app/context.ts"; +import { normalizeAgentId } from "../../lib/sessions/session-key.ts"; +import { renderChatModelControls } from "../chat/components/chat-model-controls.ts"; + +export class NewSessionModelControl { + private requestToken = 0; + private catalog: ModelCatalogEntry[] = []; + private loading = false; + selected = ""; + + constructor(private readonly notify: () => void) {} + + invalidate(resetSelection = false) { + this.requestToken += 1; + this.loading = false; + this.catalog = []; + if (resetSelection) { + this.selected = ""; + } + } + + reset() { + this.invalidate(true); + this.notify(); + } + + load(context: ApplicationContext | undefined, agentId: string, enabled: boolean) { + const snapshot = context?.gateway.snapshot; + const client = snapshot?.client; + const normalizedAgentId = normalizeAgentId(agentId); + const requestId = ++this.requestToken; + this.catalog = []; + if (!snapshot?.connected || !client || !normalizedAgentId || !enabled) { + this.loading = false; + this.notify(); + return; + } + this.loading = true; + this.notify(); + void client + .request<{ models?: ModelCatalogEntry[] }>("chat.metadata", { + agentId: normalizedAgentId, + }) + .then((result) => { + if (requestId === this.requestToken) { + this.catalog = Array.isArray(result.models) ? result.models : []; + } + }) + .catch(() => { + if (requestId === this.requestToken) { + this.catalog = []; + } + }) + .finally(() => { + if (requestId === this.requestToken) { + this.loading = false; + this.notify(); + } + }); + } + + render(options: { + agentDefaultModel?: string; + agentId: string; + context: ApplicationContext | undefined; + sending: boolean; + }) { + const snapshot = options.context?.gateway.snapshot; + const sessionKey = `new-session:${normalizeAgentId(options.agentId)}`; + return renderChatModelControls({ + activeRunId: null, + agentDefaultModel: options.agentDefaultModel, + connected: snapshot?.connected === true, + gatewayAvailable: Boolean(snapshot?.client), + loading: false, + modelCatalog: this.catalog, + modelOverrides: { [sessionKey]: this.selected }, + modelSwitching: false, + modelsLoading: this.loading, + mode: "model", + sending: options.sending, + sessionKey, + sessionsResult: options.context?.sessions.state.result ?? null, + stream: null, + onModelSelect: (value) => { + this.selected = value; + }, + onRequestUpdate: this.notify, + }); + } +} diff --git a/ui/src/pages/new-session/new-session-page.ts b/ui/src/pages/new-session/new-session-page.ts index 311b7e44e730..efcb5556251d 100644 --- a/ui/src/pages/new-session/new-session-page.ts +++ b/ui/src/pages/new-session/new-session-page.ts @@ -15,16 +15,16 @@ import { t } from "../../i18n/index.ts"; import { searchForSession } from "../../lib/sessions/index.ts"; import { buildAgentMainSessionKey, normalizeAgentId } from "../../lib/sessions/session-key.ts"; import { normalizeOptionalString } from "../../lib/string-coerce.ts"; -import { generateUUID } from "../../lib/uuid.ts"; import { OpenClawLightDomElement } from "../../lit/openclaw-element.ts"; import { SubscriptionsController } from "../../lit/subscriptions-controller.ts"; import "../../styles/chat.css"; import "../../styles/new-session.css"; +import { buildChatApiAttachments } from "../chat/attachment-api.ts"; import { renderWelcomeState } from "../chat/components/chat-welcome.ts"; -import { admitStoredChatComposerQueueItem } from "../chat/composer-persistence.ts"; +import { NewSessionAttachmentDraft } from "./attachment-draft.ts"; import * as catalog from "./catalog-target.ts"; -import { renderNewSessionComposer } from "./composer.ts"; -import { buildDraftSessionCreateParams } from "./create-params.ts"; +import { renderNewSessionDraftComposer } from "./composer.ts"; +import { buildDraftSessionCreateParams, isWorktreeNameValid } from "./create-params.ts"; import { type BrowserTarget, type DraftBranches, @@ -32,9 +32,10 @@ import { readDraftNodes, } from "./discovery.ts"; import type { NewSessionRouteData } from "./location.ts"; +import { NewSessionModelControl } from "./model-control.ts"; import { folderDisplayName, isAbsolutePath } from "./path.ts"; +import { retainRejectedInitialTurn } from "./rejected-initial-turn.ts"; -const WORKTREE_NAME_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/; const CATALOG_RETRY_DELAYS_MS = [0, 1_000, 3_000] as const; class NewSessionPage extends OpenClawLightDomElement { @@ -79,6 +80,8 @@ class NewSessionPage extends OpenClawLightDomElement { private branchesRequestToken = 0; private baseRefEditGeneration = 0; private browserRequestToken = 0; + private readonly attachmentDraft = new NewSessionAttachmentDraft(() => this.requestUpdate()); + private readonly modelControl = new NewSessionModelControl(() => this.requestUpdate()); private gatewaySource: ApplicationContext["gateway"] | null = null; private gatewayClient: ApplicationContext["gateway"]["snapshot"]["client"] = null; private gatewayConnected = false; @@ -131,6 +134,8 @@ class NewSessionPage extends OpenClawLightDomElement { this.branches = null; this.baseRef = ""; // Never carry a derived ref across a transport epoch. this.agentsHydrated = false; + this.modelControl.invalidate(resetHostSelection); + this.attachmentDraft.abortReads(); this.closeBrowser(); this.invalidateSubmission(true); // Transport loss makes an in-flight create outcome unknowable. if (!resetHostSelection) { @@ -200,6 +205,8 @@ class NewSessionPage extends OpenClawLightDomElement { override disconnectedCallback() { this.subscriptions.clear(); + // This invalidates submitRequestToken before payload release below, so a + // late sessions.create result cannot navigate with attachments we no longer own. this.invalidateGatewayDiscovery(true); this.gatewaySource = null; this.gatewayClient = null; @@ -209,6 +216,7 @@ class NewSessionPage extends OpenClawLightDomElement { this.catalogRetryAttempt = 0; globalThis.clearTimeout(this.catalogRetryTimer); this.catalogRetryTimer = undefined; + this.attachmentDraft.reset({ release: true }); super.disconnectedCallback(); } @@ -307,6 +315,7 @@ class NewSessionPage extends OpenClawLightDomElement { this.folderSelectedByUser = false; } void this.loadNodes(); + this.modelControl.load(this.context, this.agentId, !catalog.isTarget(this.data)); this.maybeLoadBranches(); } @@ -323,6 +332,8 @@ class NewSessionPage extends OpenClawLightDomElement { this.branchesLoading = false; this.execNode = ""; this.message = ""; + this.modelControl.reset(); + this.attachmentDraft.reset({ release: true }); this.error = null; this.wherePopoverHiding = false; this.folderPopoverHiding = false; @@ -442,7 +453,8 @@ class NewSessionPage extends OpenClawLightDomElement { if ( this.submitting || this.submissionOutcomeUnknown || - !this.message.trim() || + this.attachmentDraft.pendingReads > 0 || + (!this.message.trim() && this.attachmentDraft.attachments.length === 0) || !this.context?.gateway.snapshot.connected ) { return false; @@ -470,8 +482,7 @@ class NewSessionPage extends OpenClawLightDomElement { if (this.worktree && !this.worktreeAvailable()) { return false; } - const name = this.worktreeName.trim(); - if (this.worktree && name && !WORKTREE_NAME_PATTERN.test(name)) { + if (this.worktree && !isWorktreeNameValid(this.worktreeName)) { return false; } return true; @@ -483,6 +494,7 @@ class NewSessionPage extends OpenClawLightDomElement { return; } const message = this.message.trim(); + const attachments = this.attachmentDraft.attachments; const requestId = ++this.submitRequestToken; this.submitting = true; this.error = null; @@ -500,6 +512,8 @@ class NewSessionPage extends OpenClawLightDomElement { buildDraftSessionCreateParams({ agentId: this.agentId, message, + model: this.modelControl.selected, + attachments: buildChatApiAttachments(attachments), worktree: this.worktree, baseRef: this.baseRef, worktreeName: this.worktreeName, @@ -516,36 +530,17 @@ class NewSessionPage extends OpenClawLightDomElement { this.error = context.sessions.state.error ?? t("newSession.createFailed"); return; } - if (result.initialRun.status === "rejected") { - const gateway = context.gateway.snapshot; - const persisted = admitStoredChatComposerQueueItem( - { - settings: loadSettings(), - assistantAgentId: gateway.assistantAgentId, - agentsList: context.agents.state.agentsList, - hello: gateway.hello, - }, - result.key, - { - id: generateUUID(), - text: message, - createdAt: Date.now(), - kind: "queued", - refreshSessions: true, - sendAttempts: 1, - sendError: result.initialRun.error, - sendState: "failed", - sessionKey: result.key, - agentId: normalizeAgentId(this.agentId), - }, - ); - if (!persisted) { - // Stay on the draft when browser storage is unavailable: preserving - // the typed task takes priority over navigating to the partial session. - this.error = result.initialRun.error; - return; - } - } + const handedOffAttachments = + result.initialRun.status === "rejected" && + retainRejectedInitialTurn({ + agentId: this.agentId, + attachments, + context, + error: result.initialRun.error, + message, + sessionKey: result.key, + }); + this.attachmentDraft.clearAfterSubmit(!handedOffAttachments); context.gateway.setSessionKey(result.key); context.navigate("chat", { search: searchForSession(result.key) }); } finally { @@ -565,12 +560,14 @@ class NewSessionPage extends OpenClawLightDomElement { return; } this.agentId = normalizeAgentId(agentId); + this.modelControl.reset(); this.agentSelectedByUser = true; this.folder = this.execNode ? "" : this.workspacePath(); this.folderSelectedByUser = false; this.worktree = false; this.worktreeName = ""; this.closeBrowser(); + this.modelControl.load(this.context, this.agentId, true); this.maybeLoadBranches(); } @@ -1191,10 +1188,7 @@ class NewSessionPage extends OpenClawLightDomElement { /** Target row + composer, rendered mid-screen between the hero and recents. */ private renderDraftBlock() { - const worktreeNameInvalid = - this.worktree && - this.worktreeName.trim() !== "" && - !WORKTREE_NAME_PATTERN.test(this.worktreeName.trim()); + const worktreeNameInvalid = this.worktree && !isWorktreeNameValid(this.worktreeName); return html`
${this.renderTargetBar()} @@ -1205,9 +1199,15 @@ class NewSessionPage extends OpenClawLightDomElement { ${this.submissionOutcomeUnknown ? html`
${t("newSession.createOutcomeUnknown")}
` : nothing} - ${renderNewSessionComposer({ + ${renderNewSessionDraftComposer({ + agentDefaultModel: this.selectedAgent()?.model?.primary, + agentId: this.agentId, + attachmentDraft: this.attachmentDraft, canSubmit: this.canSubmit(), + context: this.context, + isCatalogTarget: catalog.isTarget(this.data), message: this.message, + modelControl: this.modelControl, requiresModifier: loadSettings().chatSendShortcut === "modifier-enter", submitting: this.submitting, onInput: (message) => { diff --git a/ui/src/pages/new-session/rejected-initial-turn.ts b/ui/src/pages/new-session/rejected-initial-turn.ts new file mode 100644 index 000000000000..545975f54113 --- /dev/null +++ b/ui/src/pages/new-session/rejected-initial-turn.ts @@ -0,0 +1,52 @@ +import type { ApplicationContext } from "../../app/context.ts"; +import { loadSettings } from "../../app/settings.ts"; +import type { ChatAttachment } from "../../lib/chat/chat-types.ts"; +import { normalizeAgentId } from "../../lib/sessions/session-key.ts"; +import { generateUUID } from "../../lib/uuid.ts"; +import { admitStoredChatComposerQueueItem } from "../chat/composer-persistence.ts"; +import { prepareInitialTurnHandoff } from "../chat/initial-turn-handoff.ts"; + +/** Returns true when attachment payload ownership moved to the volatile handoff. */ +export function retainRejectedInitialTurn(options: { + agentId: string; + attachments: ChatAttachment[]; + context: ApplicationContext; + error: string; + message: string; + sessionKey: string; +}): boolean { + const gateway = options.context.gateway.snapshot; + const rejectedItem = { + id: generateUUID(), + text: options.message, + attachments: options.attachments, + createdAt: Date.now(), + kind: "queued" as const, + refreshSessions: true, + sendAttempts: 1, + sendError: options.error, + sendState: "failed" as const, + sessionKey: options.sessionKey, + agentId: normalizeAgentId(options.agentId), + }; + const persisted = admitStoredChatComposerQueueItem( + { + settings: loadSettings(), + assistantAgentId: gateway.assistantAgentId, + agentsList: options.context.agents.state.agentsList, + hello: gateway.hello, + }, + options.sessionKey, + rejectedItem, + ); + if (persisted) { + return false; + } + // The server already created this key. A volatile handoff prevents retry + // from creating a duplicate when large attachments exceed browser storage. + prepareInitialTurnHandoff(options.sessionKey, { + ...rejectedItem, + sendRunId: generateUUID(), + }); + return true; +} diff --git a/ui/src/styles/new-session.css b/ui/src/styles/new-session.css index 22457ab17e84..021eeda7a1d9 100644 --- a/ui/src/styles/new-session.css +++ b/ui/src/styles/new-session.css @@ -50,6 +50,13 @@ openclaw-new-session-page { text-align: left; } +/* The welcome column starts beside the sidebar. Open the shared model menu + into that column instead of letting its default right edge cover navigation. */ +.new-session-page__composer .chat-controls__inline-select-menu--combined { + right: auto; + left: 0; +} + /* Cursor-style quiet trigger row. Web Awesome owns popup/listbox behavior. */ .new-session-page__triggers { display: flex;