diff --git a/ui/src/api/gateway.node.test.ts b/ui/src/api/gateway.node.test.ts index 757d6438b8e9..ccafd843fb2a 100644 --- a/ui/src/api/gateway.node.test.ts +++ b/ui/src/api/gateway.node.test.ts @@ -460,6 +460,7 @@ describe("GatewayBrowserClient", () => { GATEWAY_CLIENT_CAPS.TOOL_EVENTS, GATEWAY_CLIENT_CAPS.INLINE_WIDGETS, GATEWAY_CLIENT_CAPS.UI_COMMANDS, + GATEWAY_CLIENT_CAPS.SYSTEM_AGENT_QR_CODE, ]); expect(connectFrame.params?.scopes).toEqual([...CONTROL_UI_OPERATOR_SCOPES]); }); @@ -1209,6 +1210,7 @@ describe("GatewayBrowserClient", () => { }); await vi.waitFor(() => expect(onRecoveryScopeChange).toHaveBeenCalledOnce()); + expect(client.authenticatedDeviceId).toBe("device-1"); expect(client.recoveryScopeReady).toBe(true); expect(client.recoveryScope).toBe("device-recovery-scope"); expect(client.recoveryScope).not.toContain("stored-device-token"); diff --git a/ui/src/api/gateway.ts b/ui/src/api/gateway.ts index 11cac8e26778..4ecc7bd36bd3 100644 --- a/ui/src/api/gateway.ts +++ b/ui/src/api/gateway.ts @@ -309,6 +309,7 @@ async function buildGatewayConnectDevice(params: { export class GatewayBrowserClient { private readonly client: GatewayProtocolClient; + private authenticatedDeviceIdValue: string | null = null; inboundActivitySeq = 0; private lastInboundActivityAtMs: number | null = null; private tickWatchTimer: ReturnType | null = null; @@ -390,6 +391,10 @@ export class GatewayBrowserClient { return this.opts.url; } + get authenticatedDeviceId(): string | null { + return this.authenticatedDeviceIdValue; + } + start() { this.client.start(); } @@ -500,6 +505,7 @@ export class GatewayBrowserClient { GATEWAY_CLIENT_CAPS.TOOL_EVENTS, GATEWAY_CLIENT_CAPS.INLINE_WIDGETS, GATEWAY_CLIENT_CAPS.UI_COMMANDS, + GATEWAY_CLIENT_CAPS.SYSTEM_AGENT_QR_CODE, ], auth: buildGatewayConnectAuth(selectedAuth), userAgent: navigator.userAgent, @@ -516,6 +522,7 @@ export class GatewayBrowserClient { } private handleConnectHello(hello: GatewayHelloOk, plan: ConnectPlan) { + this.authenticatedDeviceIdValue = plan.deviceIdentity?.deviceId ?? null; this.startTickWatch(hello); this.pendingDeviceTokenRetry = false; this.deviceTokenRetryBudgetUsed = false; diff --git a/ui/src/components/wizard-step-controls.ts b/ui/src/components/wizard-step-controls.ts index 4e60370c0c9d..7f1f23ee104a 100644 --- a/ui/src/components/wizard-step-controls.ts +++ b/ui/src/components/wizard-step-controls.ts @@ -6,9 +6,13 @@ import { renderSensitiveInput } from "./sensitive-input.ts"; import "../styles/wizard-step-controls.css"; type WizardStepOption = NonNullable[number]; +type WizardQrStep = Extract; +export type WizardStepPresentation = + | WizardStep + | (Omit & { qrDataUrl?: undefined; expiresInMs: 0 }); type WizardStepControlsProps = { - step: WizardStep; + step: WizardStepPresentation; /** Current draft answer; owned by the caller so it survives re-renders. */ value: unknown; /** Disables every control while an answer is in flight. */ @@ -55,7 +59,7 @@ function renderOptionBody(option: WizardStepOption, presentation?: "channels", s `; } -function renderDeviceCode(step: WizardStep) { +function renderDeviceCode(step: WizardStepPresentation) { const deviceCode = step.deviceCode; if (!deviceCode) { return nothing; @@ -170,6 +174,44 @@ function renderContinueStep(props: WizardStepControlsProps) { `; } +function renderQrStep(props: WizardStepControlsProps) { + const dataUrl = props.step.qrDataUrl; + const active = + typeof dataUrl === "string" && + dataUrl.startsWith("data:image/png;base64,") && + typeof props.step.expiresInMs === "number" && + props.step.expiresInMs > 0; + return html` + ${renderMessage(props)} + ${active + ? html`${t("custodian.setupQrCodeAlt")} +
+ + ${renderAnswerButton(props, t("modelSetup.wizard.continue"), () => + props.onAnswer(true), + )} +
` + : html`
${t("custodian.setupQrCodeExpired")}
+
+ +
`} + `; +} + function renderProgressStep(props: WizardStepControlsProps) { return html`
@@ -326,10 +368,8 @@ export function renderWizardStepControls( return props.step.executor === "gateway" ? renderProgressStep(props) : renderContinueStep(props); - // QR steps need a client-owned presentation; clients without one must not - // turn them into an answerable generic step. case "qr": - return nothing; + return renderQrStep(props); // These show whatever the step supplies behind a single Continue. case "note": case "action": diff --git a/ui/src/e2e/custodian-qr-code.e2e.test.ts b/ui/src/e2e/custodian-qr-code.e2e.test.ts new file mode 100644 index 000000000000..8845f9974f1d --- /dev/null +++ b/ui/src/e2e/custodian-qr-code.e2e.test.ts @@ -0,0 +1,131 @@ +// Control UI proves the generic system-agent QR wizard step through a mocked Gateway. +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { chromium, type Browser } from "playwright"; +import qrcode from "qrcode"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + canRunPlaywrightChromium, + installMockGateway, + resolvePlaywrightChromiumExecutablePath, + startControlUiE2eServer, + type ControlUiE2eServer, +} from "../test-helpers/control-ui-e2e.ts"; + +const executablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); +const chromiumAvailable = canRunPlaywrightChromium(executablePath); +const allowMissingChromium = process.env.OPENCLAW_UI_E2E_ALLOW_MISSING_CHROMIUM === "1"; +const describeE2e = chromiumAvailable || !allowMissingChromium ? describe : describe.skip; +const captureProof = process.env.OPENCLAW_CAPTURE_UI_PROOF === "1"; +const proofDir = path.join(process.cwd(), ".artifacts", "control-ui-e2e", "custodian-qr-code"); + +let browser: Browser; +let server: ControlUiE2eServer; + +describeE2e("Custodian QR wizard step", () => { + beforeAll(async () => { + if (!chromiumAvailable) { + throw new Error(`Playwright Chromium is unavailable at ${executablePath}`); + } + server = await startControlUiE2eServer(); + browser = await chromium.launch({ executablePath }); + }); + + afterAll(async () => { + await browser?.close(); + await server?.close(); + }); + + it("advertises, renders, and acknowledges the shared QR step", async () => { + const qrDataUrl = await qrcode.toDataURL("https://openclaw.ai/qr-proof", { + margin: 2, + width: 560, + }); + const qrResponse = { + sessionId: "e2e-system-agent-qr", + reply: "Scan this code to continue setup.", + action: "none", + wizardInputPending: true, + step: { + id: "setup-qr", + type: "qr", + title: "Link Signal", + message: "Scan the code, then continue.", + qrDataUrl, + expiresInMs: 30 * 60 * 1000, + executor: "client", + }, + }; + if (captureProof) { + await mkdir(proofDir, { recursive: true }); + } + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 844, width: 390 }, + ...(captureProof + ? { recordVideo: { dir: proofDir, size: { height: 844, width: 390 } } } + : {}), + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page, { + featureMethods: ["chat.metadata", "chat.startup", "openclaw.chat"], + methodResponses: { + "openclaw.chat": { + cases: [ + { + match: { wizardAnswer: { stepId: "setup-qr", value: true } }, + response: { + sessionId: "e2e-system-agent-qr", + reply: "Signal is configured.", + action: "none", + }, + }, + { match: { pollStepId: "setup-qr" }, response: qrResponse }, + { response: qrResponse }, + ], + }, + }, + }); + + try { + expect( + ( + await page.goto(`${server.baseUrl}custodian?onboarding=1`, { + timeout: 60_000, + waitUntil: "domcontentloaded", + }) + )?.status(), + ).toBe(200); + const image = page.getByAltText("QR code for setup"); + await image.waitFor(); + + expect(await image.getAttribute("src")).toBe(qrDataUrl); + await expect + .poll(() => image.evaluate((node: HTMLImageElement) => node.naturalWidth)) + .toBeGreaterThan(0); + await expect + .poll(() => page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)) + .toBe(true); + + if (captureProof) { + await page.screenshot({ + animations: "disabled", + fullPage: true, + path: path.join(proofDir, "qr-step.png"), + }); + } + + await page.getByRole("button", { name: "Continue" }).click(); + await page.getByText("Signal is configured.").waitFor(); + const requests = await gateway.getRequests("openclaw.chat"); + expect(requests.at(-1)?.params).toMatchObject({ + sessionId: "e2e-system-agent-qr", + wizardAnswer: { stepId: "setup-qr", value: true }, + }); + expect(await image.count()).toBe(0); + } finally { + await context.close(); + } + }); +}); diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index 65effff06005..8ade24bf52a5 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -2304,6 +2304,8 @@ export const en: TranslationMap = { cancel: "Cancel", send: "Send", thinking: "OpenClaw is thinking", + setupQrCodeAlt: "QR code for setup", + setupQrCodeExpired: "This QR code expired. Waiting for the setup result…", earlier: "Earlier", requestFailed: "OpenClaw could not reply. Try again.", connectionChanged: "The Gateway connection changed. Retry to continue this setup.", diff --git a/ui/src/pages/custodian/custodian-page.qr.test.ts b/ui/src/pages/custodian/custodian-page.qr.test.ts new file mode 100644 index 000000000000..655678b9b13c --- /dev/null +++ b/ui/src/pages/custodian/custodian-page.qr.test.ts @@ -0,0 +1,608 @@ +/* @vitest-environment jsdom */ + +import { GatewayProtocolRequestError } from "@openclaw/gateway-client/browser"; +import { buildSystemAgentSessionInvalidatedErrorDetails } from "@openclaw/gateway-protocol"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import { waitForFast } from "../../test-helpers/wait-for.ts"; +import { createContext, mountPage } from "./custodian-page.test-harness.ts"; + +const QR_DATA_URL = "data:image/png;base64,AAAA"; +const SESSION_ID = "qr-session"; + +function qrResult(expiresInMs = 60_000, stepId = "qr-step") { + return { + sessionId: SESSION_ID, + reply: "Scan this code, then continue.", + action: "none" as const, + wizardInputPending: true, + step: { + id: stepId, + type: "qr" as const, + title: "Link a device", + message: "Scan this QR code, then continue.", + qrDataUrl: QR_DATA_URL, + expiresInMs, + executor: "client" as const, + }, + }; +} + +function terminalResult(reply = "Signal is configured.", sessionId = SESSION_ID) { + return { sessionId, reply, action: "none" as const }; +} + +describe("custodian QR wizard step", () => { + beforeEach(() => { + vi.spyOn(crypto, "randomUUID").mockReturnValue("00000000-0000-4000-8000-000000000001"); + }); + + afterEach(() => { + vi.useRealTimers(); + document.body.replaceChildren(); + vi.restoreAllMocks(); + }); + + it("renders and acknowledges QR through the generic wizard answer", async () => { + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(terminalResult("Device linked.")); + const { page } = await mountPage(createContext(request).context); + + await waitForFast(() => expect(page.querySelector(".wizard-step__qr")).not.toBeNull()); + const image = page.querySelector(".wizard-step__qr"); + expect(image?.getAttribute("src")).toBe(QR_DATA_URL); + expect(page.textContent).not.toContain(QR_DATA_URL); + + page.querySelector(".custodian__wizard-step .btn.primary")?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); + + expect(request.mock.calls[1]?.[1]).toEqual({ + sessionId: SESSION_ID, + wizardAnswer: { stepId: "qr-step", value: true }, + }); + await waitForFast(() => + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false), + ); + }); + + it("cancels QR setup through the typed wizard answer", async () => { + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(terminalResult("Signal setup cancelled.")); + const { page } = await mountPage(createContext(request).context); + + await waitForFast(() => expect(page.querySelector(".wizard-step__qr")).not.toBeNull()); + const cancelButton = Array.from( + page.querySelectorAll(".custodian__wizard-step .btn"), + ).find((button) => button.textContent?.trim() === "Cancel"); + cancelButton?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); + + expect(request.mock.calls[1]?.[1]).toEqual({ + sessionId: SESSION_ID, + wizardAnswer: { stepId: "qr-step", value: false }, + }); + await waitForFast(() => expect(page.textContent).toContain("Signal setup cancelled.")); + }); + + it("keeps polling when acknowledgement advances directly to another QR", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(qrResult(60_000, "qr-step-2")) + .mockResolvedValueOnce(terminalResult("Both devices are linked.")); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + page.querySelector(".custodian__wizard-step .btn.primary")?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); + expect(page.store.messages.some((message) => message.step?.id === "qr-step-2")).toBe(true); + + await vi.advanceTimersByTimeAsync(1_000); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(3)); + expect(request.mock.calls[2]?.[1]).toEqual({ + sessionId: SESSION_ID, + pollStepId: "qr-step-2", + }); + }); + + it("keeps the QR and resumes polling when Continue was definitely unsent", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockRejectedValueOnce(new Error("socket send failed")) + .mockResolvedValueOnce(terminalResult()); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + const continueButton = page.querySelector( + ".custodian__wizard-step .btn.primary", + ); + await waitForFast(() => expect(continueButton?.disabled).toBe(false)); + continueButton?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); + await vi.advanceTimersByTimeAsync(0); + await page.updateComplete; + + expect(page.querySelector(".wizard-step__qr")?.src).toContain(QR_DATA_URL); + expect(page.store.messages.some((message) => message.step?.qrDataUrl === QR_DATA_URL)).toBe( + true, + ); + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(3); + expect(request.mock.calls[2]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.textContent).toContain("Signal is configured."); + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false); + }); + + it("scrubs the QR but keeps polling when Continue delivery is uncertain", async () => { + vi.useFakeTimers(); + let rejectAcknowledgement!: (error: Error) => void; + const acknowledgement = new Promise((_resolve, reject) => { + rejectAcknowledgement = reject; + }); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockImplementationOnce( + async ( + _method: string, + _params: unknown, + options?: { onSent?: () => void }, + ): Promise => { + options?.onSent?.(); + return await acknowledgement; + }, + ) + .mockResolvedValueOnce(terminalResult()); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + const continueButton = page.querySelector( + ".custodian__wizard-step .btn.primary", + ); + await waitForFast(() => expect(continueButton?.disabled).toBe(false)); + continueButton?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); + await vi.advanceTimersByTimeAsync(0); + await waitForFast(() => expect(page.querySelector(".wizard-step__qr")).toBeNull()); + await waitForFast(() => + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false), + ); + rejectAcknowledgement(new Error("connection closed after send")); + await vi.advanceTimersByTimeAsync(0); + expect(page.store.error).toContain("connection closed after send"); + expect(page.querySelector('[role="alert"]')).not.toBeNull(); + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request.mock.calls[2]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.textContent).toContain("Signal is configured."); + expect(page.store.hasUnresolvedQuestion()).toBe(false); + expect(page.store.error).toBeNull(); + expect(page.querySelector('[role="alert"]')).toBeNull(); + }); + + it("clears a failed acknowledgement error when recovery returns the active QR", async () => { + vi.useFakeTimers(); + let rejectAcknowledgement!: (error: Error) => void; + const acknowledgement = new Promise((_resolve, reject) => { + rejectAcknowledgement = reject; + }); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockImplementationOnce( + async ( + _method: string, + _params: unknown, + options?: { onSent?: () => void }, + ): Promise => { + options?.onSent?.(); + return await acknowledgement; + }, + ) + .mockResolvedValueOnce(qrResult(30_000)); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + const continueButton = page.querySelector( + ".custodian__wizard-step .btn.primary", + ); + await waitForFast(() => expect(continueButton?.disabled).toBe(false)); + continueButton?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); + rejectAcknowledgement(new Error("connection closed after send")); + await vi.advanceTimersByTimeAsync(0); + expect(page.store.error).toContain("connection closed after send"); + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request.mock.calls[2]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.store.error).toBeNull(); + expect(page.querySelector('[role="alert"]')).toBeNull(); + expect(page.querySelector(".wizard-step__qr")?.src).toContain(QR_DATA_URL); + }); + + it("polls without duplicating the QR transcript and shows owner completion", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(qrResult(30_000)) + .mockResolvedValueOnce(terminalResult()); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + const messageCount = page.store.messages.length; + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request.mock.calls[1]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.store.messages).toHaveLength(messageCount); + const qrMessages = page.store.messages.filter((message) => message.step?.id === "qr-step"); + expect(qrMessages).toHaveLength(1); + expect(qrMessages[0]?.step?.expiresInMs).toBe(30_000); + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request.mock.calls[2]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.store.messages).toHaveLength(messageCount + 1); + expect(page.textContent).toContain("Signal is configured."); + expect(page.querySelector(".wizard-step__qr")).toBeNull(); + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false); + }); + + it("retries a transient QR poll failure", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockRejectedValueOnce(new Error("temporary poll failure")) + .mockResolvedValueOnce(terminalResult()); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + await vi.advanceTimersByTimeAsync(2_000); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(3); + expect(request.mock.calls[2]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.textContent).toContain("Signal is configured."); + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false); + }); + + it("resumes polling a pending QR step after a same-client reconnect", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(terminalResult()); + const { context, setGatewaySnapshot } = createContext(request, ["openclaw.chat"], { + connectionId: "connection-1", + deviceId: "device-1", + processInstanceId: "gateway-process-1", + }); + setGatewaySnapshot({ + selfUser: { id: "profile-1", email: "owner@example.com" }, + }); + const hello = context.gateway.snapshot.hello; + const { page } = await mountPage(context); + await vi.advanceTimersByTimeAsync(0); + + setGatewaySnapshot({ phase: "reconnecting", hello: null, selfUser: null }); + await page.updateComplete; + expect(page.querySelector(".wizard-step__qr")).toBeNull(); + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false); + + setGatewaySnapshot({ + phase: "connected", + hello, + selfUser: { id: "profile-1", email: "owner@example.com" }, + }); + await page.updateComplete; + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(2); + expect(request.mock.calls[1]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.textContent).toContain("Signal is configured."); + }); + + it("keeps a device-owned QR through late presence and a same-user reconnect", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(terminalResult()); + const { context, setGatewaySnapshot } = createContext(request, ["openclaw.chat"], { + connectionId: "connection-1", + deviceId: "device-1", + processInstanceId: "gateway-process-1", + }); + const { page } = await mountPage(context); + await vi.advanceTimersByTimeAsync(0); + + expect(request).toHaveBeenCalledOnce(); + expect(page.querySelector(".wizard-step__qr")).not.toBeNull(); + + setGatewaySnapshot({ + selfUser: { id: "profile-1", email: "owner@example.com" }, + }); + await page.updateComplete; + + expect(request).toHaveBeenCalledOnce(); + expect(page.querySelector(".wizard-step__qr")).not.toBeNull(); + + const hello = context.gateway.snapshot.hello; + if (!hello) { + throw new Error("expected connected Gateway hello"); + } + setGatewaySnapshot({ + client: { + request, + authenticatedDeviceId: "device-2", + } as unknown as GatewayBrowserClient, + hello: { + ...hello, + server: { connId: "connection-2" }, + }, + selfUser: { id: "profile-1", email: "owner@example.com" }, + }); + await page.updateComplete; + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(2); + expect(request.mock.calls[1]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.textContent).toContain("Signal is configured."); + }); + + it("starts fresh when late presence identifies a different user", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(terminalResult("Fresh session ready.", "replacement-session")); + const { context, setGatewaySnapshot } = createContext(request, ["openclaw.chat"], { + connectionId: "connection-1", + deviceId: "device-1", + processInstanceId: "gateway-process-1", + }); + const { page } = await mountPage(context); + await vi.advanceTimersByTimeAsync(0); + + setGatewaySnapshot({ selfUser: { id: "profile-1", email: "owner@example.com" } }); + await page.updateComplete; + const hello = context.gateway.snapshot.hello; + if (!hello) { + throw new Error("expected connected Gateway hello"); + } + setGatewaySnapshot({ + client: { + request, + authenticatedDeviceId: "device-2", + } as unknown as GatewayBrowserClient, + hello: { ...hello, server: { connId: "connection-2" } }, + selfUser: { id: "profile-2", email: "other@example.com" }, + }); + await vi.advanceTimersByTimeAsync(0); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(2); + expect(request.mock.calls[1]?.[1]).not.toHaveProperty("pollStepId"); + expect(request.mock.calls[1]?.[1]?.sessionId).not.toBe(SESSION_ID); + expect(page.textContent).toContain("Fresh session ready."); + }); + + it("resumes the same QR session after a client replacement", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(terminalResult()); + const { context, setGatewaySnapshot } = createContext(request, ["openclaw.chat"], { + connectionId: "connection-1", + deviceId: "device-1", + deviceToken: "device-token-1", + processInstanceId: "gateway-process-1", + }); + const { page } = await mountPage(context); + await vi.advanceTimersByTimeAsync(0); + const hello = context.gateway.snapshot.hello; + if (!hello) { + throw new Error("expected connected Gateway hello"); + } + + setGatewaySnapshot({ + client: { + request, + authenticatedDeviceId: "device-1", + } as unknown as GatewayBrowserClient, + hello: { + ...hello, + auth: { + role: "operator", + scopes: ["operator.admin"], + deviceToken: "device-token-2", + }, + server: { connId: "connection-2" }, + }, + }); + await page.updateComplete; + expect(page.querySelector(".wizard-step__qr")).toBeNull(); + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(2); + expect(request.mock.calls[1]?.[1]).toEqual({ sessionId: SESSION_ID, pollStepId: "qr-step" }); + expect(page.textContent).toContain("Signal is configured."); + }); + + it.each([ + { + name: "starts fresh when a connection-owned QR loses its owner", + nextProcessInstanceId: "gateway-process-1", + }, + { + name: "starts fresh when a device-owned QR loses its Gateway process", + deviceId: "device-1", + deviceToken: "device-token-1", + nextProcessInstanceId: "gateway-process-2", + }, + ])("$name", async ({ deviceId, deviceToken, nextProcessInstanceId }) => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockResolvedValueOnce(terminalResult("Fresh session ready.", "replacement-session")); + const { context, setGatewaySnapshot } = createContext(request, ["openclaw.chat"], { + connectionId: "connection-1", + ...(deviceId ? { deviceId } : {}), + ...(deviceToken ? { deviceToken } : {}), + processInstanceId: "gateway-process-1", + }); + const { page } = await mountPage(context); + await vi.advanceTimersByTimeAsync(0); + const hello = context.gateway.snapshot.hello; + if (!hello) { + throw new Error("expected connected Gateway hello"); + } + + setGatewaySnapshot({ + client: { + request, + authenticatedDeviceId: deviceId ?? null, + } as unknown as GatewayBrowserClient, + hello: { + ...hello, + server: { connId: "connection-2" }, + snapshot: { processInstanceId: nextProcessInstanceId }, + }, + }); + await vi.advanceTimersByTimeAsync(0); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(2); + expect(request.mock.calls[1]?.[1]).not.toHaveProperty("pollStepId"); + expect(request.mock.calls[1]?.[1]?.sessionId).not.toBe(SESSION_ID); + expect(page.textContent).toContain("Fresh session ready."); + }); + + it("scrubs the QR and starts fresh after poll session invalidation", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockRejectedValueOnce( + new GatewayProtocolRequestError({ + code: "INVALID_REQUEST", + message: "QR session was evicted.", + details: buildSystemAgentSessionInvalidatedErrorDetails(), + }), + ) + .mockResolvedValueOnce(terminalResult("Fresh session ready.", "replacement-session")); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(3); + expect(request.mock.calls[2]?.[1]).not.toHaveProperty("pollStepId"); + expect(request.mock.calls[2]?.[1]?.sessionId).not.toBe(SESSION_ID); + expect(page.textContent).toContain("Fresh session ready."); + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false); + }); + + it("ignores a stale acknowledgement callback after session rotation", async () => { + vi.useFakeTimers(); + const request = vi + .fn() + .mockResolvedValueOnce(qrResult()) + .mockRejectedValueOnce( + new GatewayProtocolRequestError({ + code: "INVALID_REQUEST", + message: "QR session was evicted.", + details: buildSystemAgentSessionInvalidatedErrorDetails(), + }), + ) + .mockResolvedValueOnce(terminalResult("Fresh session ready.", "replacement-session")); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + page.querySelector(".custodian__wizard-step .btn.primary")?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(3)); + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + + expect(request).toHaveBeenCalledTimes(3); + expect(request.mock.calls[2]?.[1]).not.toHaveProperty("pollStepId"); + expect(page.textContent).toContain("Fresh session ready."); + }); + + it("scrubs expired image bytes while a result poll is still pending", async () => { + vi.useFakeTimers(); + const pendingPoll = new Promise(() => {}); + const request = vi.fn().mockResolvedValueOnce(qrResult(2_000)).mockReturnValueOnce(pendingPoll); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + await vi.advanceTimersByTimeAsync(2_000); + await page.updateComplete; + + expect(page.querySelector(".wizard-step__qr")).toBeNull(); + expect(page.textContent).toContain("This QR code expired."); + expect(page.store.messages.some((message) => message.step?.qrDataUrl)).toBe(false); + }); + + it("keeps polling and offers typed cancellation after QR expiry", async () => { + vi.useFakeTimers(); + const pendingResult = { + sessionId: SESSION_ID, + reply: "Setup is still finishing the QR attempt.", + action: "none" as const, + wizardInputPending: true, + }; + const request = vi + .fn() + .mockResolvedValueOnce(qrResult(1_000)) + .mockResolvedValueOnce(pendingResult) + .mockResolvedValueOnce(pendingResult) + .mockResolvedValueOnce(terminalResult("Signal setup cancelled.")); + const { page } = await mountPage(createContext(request).context); + await vi.advanceTimersByTimeAsync(0); + + await vi.advanceTimersByTimeAsync(1_000); + await page.updateComplete; + expect(page.querySelector(".wizard-step__qr")).toBeNull(); + expect(page.textContent).toContain("This QR code expired."); + + await vi.advanceTimersByTimeAsync(1_000); + expect(request.mock.calls.filter((call) => call[1]?.pollStepId === "qr-step")).toHaveLength(2); + + const cancelButton = Array.from( + page.querySelectorAll(".custodian__wizard-step .btn"), + ).find((button) => button.textContent?.trim() === "Cancel"); + cancelButton?.click(); + await waitForFast(() => expect(request).toHaveBeenCalledTimes(4)); + expect(request.mock.calls[3]?.[1]).toEqual({ + sessionId: SESSION_ID, + wizardAnswer: { stepId: "qr-step", value: false }, + }); + await waitForFast(() => expect(page.textContent).toContain("Signal setup cancelled.")); + }); +}); diff --git a/ui/src/pages/custodian/custodian-page.test-harness.ts b/ui/src/pages/custodian/custodian-page.test-harness.ts index 966363e70d2f..4631ea424ca6 100644 --- a/ui/src/pages/custodian/custodian-page.test-harness.ts +++ b/ui/src/pages/custodian/custodian-page.test-harness.ts @@ -40,9 +40,16 @@ export function createContext( agentsList?: ApplicationContext["agents"]["state"]["agentsList"]; channelsSnapshot?: ChannelsStatusSnapshot | null; gatewayCapabilities?: string[]; + connectionId?: string; + deviceId?: string; + deviceToken?: string; + processInstanceId?: string; } = {}, ): ContextHarness { - const client = { request } as unknown as GatewayBrowserClient; + const client = { + request, + authenticatedDeviceId: options.deviceId ?? null, + } as unknown as GatewayBrowserClient; let snapshot: ApplicationGatewaySnapshot = { client, phase: "connected", @@ -51,13 +58,21 @@ export function createContext( hello: { type: "hello-ok" as const, protocol: 1, - auth: { role: "operator", scopes: ["operator.admin"] }, + auth: { + role: "operator", + scopes: ["operator.admin"], + ...(options.deviceToken ? { deviceToken: options.deviceToken } : {}), + }, features: { methods, capabilities: options.gatewayCapabilities ?? [ GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, ], }, + ...(options.connectionId ? { server: { connId: options.connectionId } } : {}), + ...(options.processInstanceId + ? { snapshot: { processInstanceId: options.processInstanceId } } + : {}), }, assistantAgentId: "main", sessionKey: "main", diff --git a/ui/src/pages/custodian/custodian-session-store.ts b/ui/src/pages/custodian/custodian-session-store.ts index 5e5213c78c34..7826c0bb2646 100644 --- a/ui/src/pages/custodian/custodian-session-store.ts +++ b/ui/src/pages/custodian/custodian-session-store.ts @@ -5,7 +5,6 @@ import { type SystemAgentChatResult, } from "@openclaw/gateway-protocol"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; -import type { WizardStep } from "../../api/types.ts"; import { selectApplicationSession } from "../../app/agent-selection.ts"; import type { ApplicationContext } from "../../app/context.ts"; import { t } from "../../i18n/index.ts"; @@ -14,17 +13,30 @@ import { isGatewayCapabilityAdvertised, isGatewayMethodAdvertised, } from "../../lib/gateway-methods.ts"; -import { buildAgentMainSessionKey, normalizeAgentId } from "../../lib/sessions/session-key.ts"; +import { buildAgentMainSessionKey } from "../../lib/sessions/session-key.ts"; import { pathForCustodianAgentHandoff } from "./custodian-navigation.ts"; -import { custodianWizardSubmission, initialCustodianWizardValue } from "./custodian-wizard-step.ts"; +import { + CustodianQrScheduler, + custodianWizardSubmission, + findCustodianQrStep, + initialCustodianWizardValue, + replaceCustodianQrStep, + scrubCustodianQrSteps, +} from "./custodian-wizard-step.ts"; import * as eventNudgeState from "./event-nudge.ts"; import { custodianChatParams, + hasCustodianUserInput, isCustodianSessionInvalidatedError, + resolveCustodianConfiguredInferenceState, + resolveCustodianSessionContinuity, + type CustodianConfiguredInferenceState, + type CustodianSessionContinuity, type CustodianSessionVariant, } from "./session-lifecycle.ts"; -import { parseCustodianQuestion, type CustodianStructuredQuestion } from "./structured-question.ts"; +import { parseCustodianQuestion } from "./structured-question.ts"; import { + createCustodianAssistantMessage, createCustodianSessionId, createCustodianTranscriptMessages, custodianErrorMessage, @@ -37,16 +49,6 @@ import { const SYSTEM_AGENT_CHAT_TIMEOUT_MS = 190_000; const SILENT_REPLY_PATTERN = /^\s*NO_REPLY\s*$/; -function hasCustodianUserInput(params: SystemAgentChatParams): boolean { - return ( - params.message !== undefined || - params.wizardAnswer !== undefined || - params.wizardCancel !== undefined - ); -} - -type StoreListener = () => void; -type ConfiguredInferenceState = "unresolved" | "required" | "ready"; type CustodianSetupIssue = "missing" | "unavailable"; /** One process-local conversation owner shared by the full page and dock surface. */ @@ -80,17 +82,32 @@ export class CustodianSessionStore { private nextMessageId = 1; private retryParams: SystemAgentChatParams | null = null; private sessionClient: GatewayBrowserClient | null = null; - private sessionOwnershipKey: string | null = null; + private sessionContinuity: CustodianSessionContinuity | null = null; + private lastContinuity: CustodianSessionContinuity | null = null; private sessionStarted = false; - private lastHelloDeviceToken = ""; - private configuredInferenceState: ConfiguredInferenceState = "unresolved"; + private configuredInferenceState: CustodianConfiguredInferenceState = "unresolved"; private eventNudgeClosed = false; private gatewayCleanup: (() => void) | null = null; private agentCleanup: (() => void) | null = null; private eventCleanup: (() => void) | null = null; - private readonly listeners = new Set(); + private readonly listeners = new Set<() => void>(); + private readonly qrScheduler = new CustodianQrScheduler({ + onExpire: (stepId, notify) => { + this.messages = scrubCustodianQrSteps(this.messages, stepId); + if (notify) { + this.emit(); + } + }, + onPoll: (client, stepId) => { + void this.requestReply( + client, + { sessionId: this.sessionId, pollStepId: stepId }, + { pollStepId: stepId }, + ); + }, + }); - subscribe(listener: StoreListener): () => void { + subscribe(listener: () => void): () => void { this.listeners.add(listener); return () => this.listeners.delete(listener); } @@ -146,9 +163,7 @@ export class CustodianSessionStore { this.emit(); } - hasRealUserTurn(): boolean { - return this.messages.some((message) => message.role === "user"); - } + hasRealUserTurn = () => this.messages.some((message) => message.role === "user"); get activeVariant(): CustodianSessionVariant { return this.variant; @@ -164,9 +179,7 @@ export class CustodianSessionStore { ); } - canRetry(): boolean { - return this.retryParams !== null && !hasCustodianUserInput(this.retryParams); - } + canRetry = () => this.retryParams !== null && !hasCustodianUserInput(this.retryParams); get setupRequired(): boolean { return this.setupIssue !== null; @@ -218,6 +231,7 @@ export class CustodianSessionStore { params: SystemAgentChatParams, displayText: string, questionReply: boolean, + onSent?: () => void, ): Promise { const questionState = [this.answeredQuestions, this.questionReplyUncertain] as const; if (questionReply) { @@ -238,7 +252,7 @@ export class CustodianSessionStore { ]; this.input = ""; this.emit(); - const reply = this.requestReply(client, params); + const reply = this.requestReply(client, params, { onSent }); const replyEpoch = this.requestEpoch; const outcome = await reply; if (questionReply && this.requestEpoch === replyEpoch) { @@ -327,12 +341,22 @@ export class CustodianSessionStore { return; } const displayText = message.step.sensitive ? t("custodian.sensitiveReply") : submission.display; + const sessionContinuity = this.sessionContinuity; + const settleQrAcknowledgement = + message.step.type === "qr" + ? this.qrScheduler.beginAcknowledgement( + client, + message.step.id, + () => this.sessionContinuity === sessionContinuity && this.activeClient === client, + ) + : undefined; void this.sendUserTurn( client, { sessionId: this.sessionId, wizardAnswer: submission.answer }, displayText, true, - ); + () => settleQrAcknowledgement?.("sent"), + ).then(settleQrAcknowledgement); } cancelWizardStep(message: CustodianMessage): void { @@ -380,23 +404,18 @@ export class CustodianSessionStore { this.context?.navigate("model-setup"); } - private emit(): void { - for (const listener of this.listeners) { - listener(); - } - } + private emit = () => this.listeners.forEach((listener) => listener()); - private currentSessionOwnershipKey(): string { + private currentSessionContinuity(): CustodianSessionContinuity { const context = this.context; if (!context) { - return ""; + return { key: "", ownerKey: null, authenticatedUserKey: null, processInstanceId: null }; } - const { gatewayUrl, token, password, bootstrapToken } = context.gateway.connection; - const auth = context.gateway.snapshot.hello?.auth; - if (auth) { - this.lastHelloDeviceToken = auth.deviceToken ?? ""; - } - return JSON.stringify([gatewayUrl, token, password, bootstrapToken, this.lastHelloDeviceToken]); + return (this.lastContinuity = resolveCustodianSessionContinuity({ + connection: context.gateway.connection, + snapshot: context.gateway.snapshot, + previous: this.lastContinuity, + })); } private startSession( @@ -407,7 +426,7 @@ export class CustodianSessionStore { this.sessionId = createCustodianSessionId(); this.sessionVariant = variant; this.sessionClient = client; - this.sessionOwnershipKey = this.currentSessionOwnershipKey(); + this.sessionContinuity = this.currentSessionContinuity(); this.sessionStarted = true; void this.initializeSession( client, @@ -425,10 +444,7 @@ export class CustodianSessionStore { this.abandonedTurnOutcomeUnknown = true; } - private rotateVolatileSession( - client: GatewayBrowserClient, - variant: CustodianSessionVariant, - ): void { + private rotateVolatileSession(client: GatewayBrowserClient, variant: CustodianSessionVariant) { this.answeredQuestions = retireCustodianQuestions(this.messages, this.answeredQuestions); this.retryParams = null; this.input = ""; @@ -450,18 +466,21 @@ export class CustodianSessionStore { const client = snapshot.phase === "connected" ? snapshot.client : null; const chatSupported = client !== null && canCallGatewayMethod(snapshot, "openclaw.chat", "operator.admin"); - const configuredInferenceState = this.resolveConfiguredInferenceState(); + const configuredInferenceState = resolveCustodianConfiguredInferenceState(this.context); const inferenceStateChanged = configuredInferenceState !== this.configuredInferenceState; this.configuredInferenceState = configuredInferenceState; const variantChanged = this.sessionStarted && this.sessionVariant !== this.variant; - const ownershipKey = this.currentSessionOwnershipKey(); + const continuity = this.currentSessionContinuity(); const clientReplaced = this.sessionStarted && client !== null && this.sessionClient !== null && client !== this.sessionClient; const ownershipChanged = - this.sessionOwnershipKey !== null && ownershipKey !== this.sessionOwnershipKey; + this.sessionContinuity !== null && continuity.key !== this.sessionContinuity.key; + const pendingQrStepId = this.wizardInputPending + ? this.messages.findLast((message) => message.step?.type === "qr")?.step?.id + : undefined; if ( client === this.activeClient && !variantChanged && @@ -472,6 +491,8 @@ export class CustodianSessionStore { ) { return; } + this.qrScheduler.clear(); + this.messages = scrubCustodianQrSteps(this.messages); const requestWasPending = this.sending && this.retryParams !== null; const pendingParams = requestWasPending ? this.retryParams : null; this.activeClient = client; @@ -494,7 +515,14 @@ export class CustodianSessionStore { } this.chatAvailable = true; this.abandonPendingUserTurn(pendingParams); - this.rotateVolatileSession(client, this.currentSessionVariant()); + if (pendingQrStepId && continuity.ownerKey && continuity.processInstanceId) { + this.retryParams = null; + this.error = null; + this.sessionClient = client; + this.qrScheduler.schedulePoll(client, pendingQrStepId); + return; + } + this.rotateVolatileSession(client, this.variant); return; } else if (requestWasPending) { if (pendingParams?.message === undefined) { @@ -526,35 +554,17 @@ export class CustodianSessionStore { if (!this.retryParams) { this.error = requestWasPending ? this.error : null; } + const pendingStep = this.wizardInputPending + ? this.messages.findLast((message) => message.step !== null)?.step + : null; + if (pendingStep?.type === "qr") { + // A reconnect invalidates the old timer, but the Gateway still owns the QR session. + this.qrScheduler.schedulePoll(client, pendingStep.id); + } return; } this.clearConversation(); - this.startSession(client, this.currentSessionVariant(), true); - } - - private resolveConfiguredInferenceState(): ConfiguredInferenceState { - const context = this.context; - if (!context || context.gateway.snapshot.phase !== "connected") { - return "unresolved"; - } - const agentsList = context.agents.state.agentsList; - if (!agentsList) { - return "unresolved"; - } - const selectedId = normalizeAgentId( - context.gateway.snapshot.assistantAgentId ?? agentsList.defaultId ?? "", - ); - const selectedAgent = agentsList.agents.find( - (agent) => normalizeAgentId(agent.id) === selectedId, - ); - if (!selectedAgent) { - return "unresolved"; - } - return selectedAgent.model?.primary?.trim() ? "ready" : "required"; - } - - private currentSessionVariant(): CustodianSessionVariant { - return this.variant; + this.startSession(client, this.variant, true); } private async initializeSession( @@ -599,6 +609,7 @@ export class CustodianSessionStore { } private clearConversation(): void { + this.qrScheduler.clear(); this.messages = []; this.dismissedQuestions = new Set(); this.answeredQuestions = new Set(); @@ -612,27 +623,10 @@ export class CustodianSessionStore { this.earlierBoundaryAfterId = null; } - private appendAssistant( - reply: string, - question: CustodianStructuredQuestion | null, - step: WizardStep | null, - ): void { - this.messages = [ - ...this.messages, - { - id: this.nextMessageId++, - role: "assistant", - text: reply, - at: Date.now(), - question, - step, - }, - ]; - } - private async requestReply( client: GatewayBrowserClient, params: SystemAgentChatParams, + options?: { pollStepId?: string; onSent?: () => void }, ): Promise { const context = this.context; if (!context) { @@ -648,19 +642,25 @@ export class CustodianSessionStore { this.requestAbort?.abort(); const requestAbort = new AbortController(); this.requestAbort = requestAbort; + const pollStepId = options?.pollStepId; const epoch = ++this.requestEpoch; let delivery: eventNudgeState.CustodianSendDelivery = "unsent"; - this.sending = true; - this.error = null; - if (hasCustodianUserInput(params)) { - this.setupIssue = null; + if (!pollStepId) { + this.sending = true; + this.error = null; + if (hasCustodianUserInput(params)) { + this.setupIssue = null; + } + this.retryParams = params; + this.emit(); } - this.retryParams = params; - this.emit(); try { const result = await client.request("openclaw.chat", params, { timeoutMs: SYSTEM_AGENT_CHAT_TIMEOUT_MS, - onSent: () => (delivery = "sent"), + onSent: () => { + delivery = "sent"; + options?.onSent?.(); + }, signal: requestAbort.signal, }); delivery = "received"; @@ -668,17 +668,49 @@ export class CustodianSessionStore { return "sent"; } this.sessionId = result.sessionId; + // An authoritative recovery response supersedes a failed QR acknowledgement. + this.error = null; + if (pollStepId && result.step?.type === "qr" && result.step.id === pollStepId) { + this.messages = replaceCustodianQrStep(this.messages, result.step); + this.wizardInputPending = result.wizardInputPending === true; + this.qrScheduler.scheduleStep(client, result.step); + return "sent"; + } + if (pollStepId && result.wizardInputPending === true && result.step === undefined) { + // An externally owned QR can outlive its presentation. Keep observations short and + // poll again so Cancel remains responsive while the owner settles in the background. + this.messages = scrubCustodianQrSteps(this.messages, pollStepId); + this.questionReplyUncertain = false; + this.wizardInputPending = true; + this.qrScheduler.schedulePoll(client, pollStepId); + return "sent"; + } + this.qrScheduler.clear(); + if (pollStepId) { + this.messages = scrubCustodianQrSteps(this.messages, pollStepId); + this.questionReplyUncertain = false; + } this.sensitive = result.sensitive === true; this.wizardInputPending = result.wizardInputPending === true; - this.retryParams = null; - this.setupIssue = null; + [this.retryParams, this.setupIssue] = [null, null]; const step = result.step ?? null; const question = step ? null : parseCustodianQuestion(result.question); this.wizardValue = step ? initialCustodianWizardValue(step) : undefined; this.wizardSecretVisible = false; const silentReply = SILENT_REPLY_PATTERN.test(result.reply); if (!silentReply || question || step) { - this.appendAssistant(silentReply ? "" : result.reply, question, step); + this.messages = [ + ...this.messages, + createCustodianAssistantMessage({ + id: this.nextMessageId++, + text: silentReply ? "" : result.reply, + question, + step, + }), + ]; + } + if (step?.type === "qr") { + this.qrScheduler.scheduleStep(client, step); } if (result.action === "open-agent") { let sessionKey = context.gateway.snapshot.sessionKey?.trim(); @@ -711,6 +743,21 @@ export class CustodianSessionStore { } return "sent"; } catch (error) { + if (pollStepId) { + if (epoch === this.requestEpoch && client === this.activeClient) { + if (isCustodianSessionInvalidatedError(error)) { + this.qrScheduler.clear(); + this.messages = scrubCustodianQrSteps(this.messages, pollStepId); + this.rotateVolatileSession(client, this.variant); + return "sent"; + } + const step = findCustodianQrStep(this.messages, pollStepId); + if (step) { + this.qrScheduler.schedulePoll(client, step.id); + } + } + return eventNudgeState.classifyCustodianSendFailure(error, delivery); + } if (epoch === this.requestEpoch && client === this.activeClient) { this.error = custodianErrorMessage(error); const details = @@ -723,7 +770,7 @@ export class CustodianSessionStore { : null; if (hasCustodianUserInput(params) && isCustodianSessionInvalidatedError(error)) { // Retained transcript rows are display context only; the next turn needs a fresh id. - this.rotateVolatileSession(client, this.currentSessionVariant()); + this.rotateVolatileSession(client, this.variant); this.error = t("custodian.sessionRestarted", { error: custodianErrorMessage(error) }); } } @@ -736,7 +783,7 @@ export class CustodianSessionStore { if (this.requestAbort === requestAbort) { this.requestAbort = null; } - if (epoch === this.requestEpoch) { + if (!pollStepId && epoch === this.requestEpoch) { this.sending = false; } this.emit(); diff --git a/ui/src/pages/custodian/custodian-wizard-step.test.ts b/ui/src/pages/custodian/custodian-wizard-step.test.ts index 95f914e6e65f..5c76f4aac2cf 100644 --- a/ui/src/pages/custodian/custodian-wizard-step.test.ts +++ b/ui/src/pages/custodian/custodian-wizard-step.test.ts @@ -8,24 +8,20 @@ const options = [ { label: "Twitch", value: "twitch" }, ]; -type TestStepPatch = - | { type?: "select"; initialValue?: unknown } - | { type: "multiselect"; initialValue?: unknown } - | { type: "confirm"; initialValue?: unknown } - | { type: "text"; initialValue?: unknown } - | { type: "action"; initialValue?: unknown }; +type NonQrWizardStep = Exclude; -function step(patch: TestStepPatch): WizardStep { - switch (patch.type) { - case "multiselect": - return { id: "step", options, ...patch }; - case "confirm": - case "text": - case "action": - return { id: "step", ...patch }; - default: - return { id: "step", type: "select", options, ...patch }; - } +function step(patch: Partial): NonQrWizardStep { + return { id: "step", type: "select", options, ...patch }; +} + +function qrStep(): WizardStep { + return { + id: "step", + type: "qr", + executor: "client", + qrDataUrl: "data:image/png;base64,AAAA", + expiresInMs: 60_000, + }; } describe("Custodian rich wizard answers", () => { @@ -59,6 +55,14 @@ describe("Custodian rich wizard answers", () => { answer: { stepId: "step" }, display: "Continue", }); + expect(custodianWizardSubmission(qrStep(), undefined)).toEqual({ + answer: { stepId: "step", value: true }, + display: "Continue", + }); + expect(custodianWizardSubmission(qrStep(), false)).toEqual({ + answer: { stepId: "step", value: false }, + display: "Cancel", + }); }); it("copies multiselect defaults and rejects values outside the step", () => { diff --git a/ui/src/pages/custodian/custodian-wizard-step.ts b/ui/src/pages/custodian/custodian-wizard-step.ts index 0fb1b04f3001..93dab2d447f9 100644 --- a/ui/src/pages/custodian/custodian-wizard-step.ts +++ b/ui/src/pages/custodian/custodian-wizard-step.ts @@ -1,23 +1,157 @@ +import { resolveSafeTimeoutDelayMs } from "@openclaw/gateway-client/browser"; import type { WizardAnswer } from "@openclaw/gateway-protocol"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; import type { WizardStep } from "../../api/types.ts"; +import type { WizardStepPresentation } from "../../components/wizard-step-controls.ts"; import { t } from "../../i18n/index.ts"; +import type { CustodianSendOutcome } from "./event-nudge.ts"; +import type { CustodianMessage } from "./transcript.ts"; type CustodianWizardSubmission = { answer: WizardAnswer; display: string; }; -function findOption(step: WizardStep, value: unknown) { +const SYSTEM_AGENT_QR_POLL_INTERVAL_MS = 1_000; + +export class CustodianQrScheduler { + private pollTimer: ReturnType | null = null; + private expiryTimer: ReturnType | null = null; + private stepId: string | null = null; + + constructor( + private readonly callbacks: { + onExpire: (stepId: string, notify: boolean) => void; + onPoll: (client: GatewayBrowserClient, stepId: string) => void; + }, + ) {} + + clear(): void { + if (this.pollTimer !== null) { + clearTimeout(this.pollTimer); + this.pollTimer = null; + } + if (this.expiryTimer !== null) { + clearTimeout(this.expiryTimer); + this.expiryTimer = null; + } + this.stepId = null; + } + + beginAcknowledgement( + client: GatewayBrowserClient, + stepId: string, + isCurrent: () => boolean, + ): (outcome: CustodianSendOutcome) => void { + if (this.pollTimer !== null) { + clearTimeout(this.pollTimer); + this.pollTimer = null; + } + let delivered = false; + return (outcome) => { + if (!isCurrent()) { + return; + } + if (outcome !== "rejected" && !delivered) { + delivered = true; + if (this.stepId === stepId) { + this.clear(); + } + this.callbacks.onExpire(stepId, true); + } + if (outcome !== "sent") { + this.schedulePoll(client, stepId); + } + }; + } + + scheduleStep(client: GatewayBrowserClient, step: WizardStep): void { + this.clear(); + if (step.type !== "qr") { + return; + } + this.stepId = step.id; + const expiresInMs = step.expiresInMs; + if (typeof expiresInMs === "number" && Number.isFinite(expiresInMs)) { + if (expiresInMs <= 0) { + this.callbacks.onExpire(step.id, false); + } else { + this.expiryTimer = setTimeout( + () => { + this.expiryTimer = null; + this.callbacks.onExpire(step.id, true); + }, + resolveSafeTimeoutDelayMs(expiresInMs, { minMs: 0 }), + ); + } + } + this.schedulePoll(client, step.id); + } + + schedulePoll(client: GatewayBrowserClient, stepId: string): void { + if (this.pollTimer !== null) { + clearTimeout(this.pollTimer); + } + this.stepId = stepId; + this.pollTimer = setTimeout(() => { + this.pollTimer = null; + this.callbacks.onPoll(client, stepId); + }, SYSTEM_AGENT_QR_POLL_INTERVAL_MS); + } +} + +export function scrubCustodianQrSteps( + messages: readonly CustodianMessage[], + stepId?: string, +): CustodianMessage[] { + return messages.map((message) => { + const step = message.step; + if (step?.type !== "qr" || (stepId !== undefined && step.id !== stepId)) { + return message; + } + const { qrDataUrl: _qrDataUrl, ...scrubbedStep } = step; + return { ...message, step: { ...scrubbedStep, expiresInMs: 0 } }; + }); +} + +export function replaceCustodianQrStep( + messages: readonly CustodianMessage[], + step: WizardStep, +): CustodianMessage[] { + return messages.map((message) => (message.step?.id === step.id ? { ...message, step } : message)); +} + +export function findCustodianQrStep( + messages: readonly CustodianMessage[], + stepId: string, +): WizardStepPresentation | null { + return ( + messages.findLast((message) => message.step?.type === "qr" && message.step.id === stepId) + ?.step ?? null + ); +} + +function findOption(step: WizardStepPresentation, value: unknown) { return step.options?.find((option) => Object.is(option.value, value)); } /** Build the typed answer sent by a client rendering the current wizard step. */ export function custodianWizardSubmission( - step: WizardStep, + step: WizardStepPresentation, value: unknown, ): CustodianWizardSubmission | null { if (step.type === "note" || step.type === "action" || step.type === "progress") { - return { answer: { stepId: step.id }, display: t("common.continue") }; + return { + answer: { stepId: step.id }, + display: t("common.continue"), + }; + } + if (step.type === "qr") { + const confirmed = value !== false; + return { + answer: { stepId: step.id, value: confirmed }, + display: t(confirmed ? "common.continue" : "common.cancel"), + }; } if (step.type === "text") { return typeof value === "string" @@ -53,7 +187,7 @@ export function custodianWizardSubmission( }; } -export function initialCustodianWizardValue(step: WizardStep): unknown { +export function initialCustodianWizardValue(step: WizardStepPresentation): unknown { return step.type === "multiselect" ? Array.isArray(step.initialValue) ? [...step.initialValue] diff --git a/ui/src/pages/custodian/session-lifecycle.ts b/ui/src/pages/custodian/session-lifecycle.ts index bfe2800af234..04d8a3efd1f0 100644 --- a/ui/src/pages/custodian/session-lifecycle.ts +++ b/ui/src/pages/custodian/session-lifecycle.ts @@ -3,8 +3,138 @@ import { type SystemAgentChatParams, } from "@openclaw/gateway-protocol"; import { inferBasePathFromPathname, routeIdFromPath } from "../../app-route-paths.ts"; +import type { ApplicationContext } from "../../app/context.ts"; +import type { + ApplicationGatewayConnection, + ApplicationGatewaySnapshot, +} from "../../app/gateway.ts"; +import { normalizeAgentId } from "../../lib/sessions/session-key.ts"; export type CustodianSessionVariant = "onboarding" | "new-agent" | "caretaker"; +export type CustodianConfiguredInferenceState = "unresolved" | "required" | "ready"; + +export type CustodianSessionContinuity = { + key: string; + ownerKey: string | null; + authenticatedUserKey: string | null; + processInstanceId: string | null; +}; + +function readRecordString(value: unknown, key: string): string | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const field = (value as Record)[key]; + return typeof field === "string" && field.trim() ? field : null; +} + +function resolveCustodianOwner(params: { + hello: ApplicationGatewaySnapshot["hello"]; + snapshot: ApplicationGatewaySnapshot; + previous: CustodianSessionContinuity | null; +}): Pick { + if (!params.hello) { + return { + ownerKey: params.previous?.ownerKey ?? null, + authenticatedUserKey: params.previous?.authenticatedUserKey ?? null, + }; + } + const userId = params.snapshot.selfUser?.email?.trim() || params.snapshot.selfUser?.id.trim(); + const deviceId = params.snapshot.client?.authenticatedDeviceId?.trim(); + const userOwner = userId ? `user:${userId}` : null; + const deviceOwner = deviceId ? `device:${deviceId}` : null; + const connectionOwner = params.hello.server?.connId + ? `connection:${params.hello.server.connId}` + : null; + const previousOwner = params.previous?.ownerKey ?? null; + const previousUser = params.previous?.authenticatedUserKey ?? null; + if (userOwner) { + if (previousUser && previousUser !== userOwner) { + return { ownerKey: userOwner, authenticatedUserKey: userOwner }; + } + // Presence may identify the authenticated user after setup starts. Remember + // that identity for reconnects without rotating the live session it now owns. + return { + ownerKey: previousOwner ?? userOwner, + authenticatedUserKey: userOwner, + }; + } + if (previousUser) { + return { ownerKey: previousOwner, authenticatedUserKey: previousUser }; + } + if (previousOwner?.startsWith("device:")) { + const ownerKey = + deviceOwner === previousOwner + ? previousOwner + : (deviceOwner ?? connectionOwner ?? previousOwner); + return { ownerKey, authenticatedUserKey: null }; + } + if (previousOwner?.startsWith("connection:")) { + const ownerKey = + connectionOwner === previousOwner + ? previousOwner + : (deviceOwner ?? connectionOwner ?? previousOwner); + return { ownerKey, authenticatedUserKey: null }; + } + return { + ownerKey: deviceOwner ?? connectionOwner ?? previousOwner, + authenticatedUserKey: null, + }; +} + +/** Pins continuity to authenticated lineage so later presence cannot rotate live setup. */ +export function resolveCustodianSessionContinuity(params: { + connection: ApplicationGatewayConnection; + snapshot: ApplicationGatewaySnapshot; + previous: CustodianSessionContinuity | null; +}): CustodianSessionContinuity { + const hello = params.snapshot.hello; + const processInstanceId = hello + ? readRecordString(hello.snapshot, "processInstanceId") + : (params.previous?.processInstanceId ?? null); + const { ownerKey, authenticatedUserKey } = resolveCustodianOwner({ + hello, + snapshot: params.snapshot, + previous: params.previous, + }); + const { gatewayUrl, token, password, bootstrapToken } = params.connection; + return { + ownerKey, + authenticatedUserKey, + processInstanceId, + key: JSON.stringify([gatewayUrl, token, password, bootstrapToken, ownerKey, processInstanceId]), + }; +} + +export function hasCustodianUserInput(params: SystemAgentChatParams): boolean { + return ( + params.message !== undefined || + params.wizardAnswer !== undefined || + params.wizardCancel !== undefined + ); +} + +export function resolveCustodianConfiguredInferenceState( + context: ApplicationContext | null, +): CustodianConfiguredInferenceState { + if (!context || context.gateway.snapshot.phase !== "connected") { + return "unresolved"; + } + const agentsList = context.agents.state.agentsList; + if (!agentsList) { + return "unresolved"; + } + const selectedId = normalizeAgentId( + context.gateway.snapshot.assistantAgentId ?? agentsList.defaultId ?? "", + ); + const selectedAgent = agentsList.agents.find( + (agent) => normalizeAgentId(agent.id) === selectedId, + ); + if (!selectedAgent) { + return "unresolved"; + } + return selectedAgent.model?.primary?.trim() ? "ready" : "required"; +} export function sessionVariant( onboarding: boolean, diff --git a/ui/src/pages/custodian/transcript.ts b/ui/src/pages/custodian/transcript.ts index f5e14d690cc6..0197b6182d9e 100644 --- a/ui/src/pages/custodian/transcript.ts +++ b/ui/src/pages/custodian/transcript.ts @@ -4,8 +4,10 @@ import type { } from "@openclaw/gateway-protocol"; import { html, nothing } from "lit"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; -import type { WizardStep } from "../../api/types.ts"; -import { renderWizardStepControls } from "../../components/wizard-step-controls.ts"; +import { + renderWizardStepControls, + type WizardStepPresentation, +} from "../../components/wizard-step-controls.ts"; import { t } from "../../i18n/index.ts"; import type { MessageGroup } from "../../lib/chat/chat-types.ts"; import { renderChatDivider } from "../chat/components/chat-divider.ts"; @@ -21,9 +23,22 @@ export type CustodianMessage = { text: string; at: number; question: CustodianStructuredQuestion | null; - step: WizardStep | null; + step: WizardStepPresentation | null; }; +export function createCustodianAssistantMessage(params: { + id: number; + text: string; + question: CustodianMessage["question"]; + step: CustodianMessage["step"]; +}): CustodianMessage { + return { + ...params, + role: "assistant", + at: Date.now(), + }; +} + export function hasUnresolvedCustodianQuestion( messages: readonly CustodianMessage[], dismissedQuestions: ReadonlySet, diff --git a/ui/src/styles/wizard-step-controls.css b/ui/src/styles/wizard-step-controls.css index a16b5f6318a0..c5b6fbf3be47 100644 --- a/ui/src/styles/wizard-step-controls.css +++ b/ui/src/styles/wizard-step-controls.css @@ -23,6 +23,15 @@ letter-spacing: 0.08em; } +.wizard-step__qr { + display: block; + width: min(100%, 360px); + aspect-ratio: 1; + object-fit: contain; + border-radius: var(--radius-md); + background: white; +} + .wizard-step__options { display: grid; gap: 8px;