From f12ef8423818bfeb332450634f236e972d3b666b Mon Sep 17 00:00:00 2001 From: jesse-merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:22:42 +1000 Subject: [PATCH] feat(ui): show Custodian setup action receipts --- .../OpenClawProtocol/GatewayModels.swift | 10 +- .../agent-harness-runtime.json | 2 +- .../agent-harness.json | 2 +- .../plugin-sdk-api-baseline/channel-core.json | 2 +- .../channel-entry-contract.json | 2 +- .../channel-message.json | 2 +- .../channel-outbound.json | 2 +- .../channel-plugin-common.json | 2 +- .../plugin-sdk-api-baseline/core.json | 2 +- .../plugin-sdk-api-baseline/discord.json | 2 +- .../gateway-runtime.json | 2 +- .../inbound-reply-dispatch.json | 2 +- .../meeting-runtime.json | 2 +- .../plugin-sdk-api-baseline/plugin-entry.json | 2 +- .../plugin-runtime.json | 2 +- .../provider-catalog-runtime.json | 2 +- .../plugin-sdk-api-baseline/tool-plugin.json | 2 +- .../webhook-ingress.json | 2 +- packages/gateway-protocol/CHANGELOG.md | 1 + .../gateway-protocol/src/schema/frames.ts | 1 + .../src/schema/openclaw.test.ts | 54 ++++ .../gateway-protocol/src/schema/openclaw.ts | 13 + .../system-agent-chat-history.test.ts | 59 +++++ .../system-agent-chat-history.ts | 26 +- .../system-agent-chat-turn.test.ts | 2 + .../server-methods/system-agent-chat-turn.ts | 3 + src/gateway/server-methods/system-agent.ts | 34 +-- .../server.auth.default-token.suite.ts | 3 + .../server/ws-connection/connect-hello.ts | 1 + src/system-agent/chat-engine.ts | 20 +- src/system-agent/chat-wizard-host.test.ts | 36 ++- src/system-agent/chat-wizard-host.ts | 26 +- .../custodian/custodian-panel.test.ts | 41 ++- ui/src/e2e/custodian-event-nudge.e2e.test.ts | 54 +++- ui/src/i18n/locales/en.ts | 7 + .../custodian-page.reload-recovery.test.ts | 169 +++++++++++- .../custodian-page.structured-wizard.test.ts | 242 ++++++++++++++++++ .../custodian/custodian-page.test-harness.ts | 1 + ui/src/pages/custodian/custodian-page.test.ts | 30 ++- .../custodian/custodian-session-store.ts | 106 ++++---- .../custodian-structured-interaction.ts | 144 +++++++++++ .../custodian/custodian-transcript-state.ts | 1 + ui/src/pages/custodian/event-nudge.ts | 4 +- ui/src/pages/custodian/transcript.ts | 159 +++++++++--- ui/src/styles/custodian.css | 60 ++++- 45 files changed, 1187 insertions(+), 154 deletions(-) create mode 100644 ui/src/pages/custodian/custodian-page.structured-wizard.test.ts create mode 100644 ui/src/pages/custodian/custodian-structured-interaction.ts diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index a5e1fffd0370..5b427eba85f7 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -10872,6 +10872,7 @@ public struct SystemAgentChatResult: Codable, Sendable { public let reply: String public let sensitive: Bool? public let wizardinputpending: Bool? + public let wizardactionaccepted: Bool? public let action: AnyCodable public let agentdraft: String? public let agentid: String? @@ -10885,6 +10886,7 @@ public struct SystemAgentChatResult: Codable, Sendable { reply: String, sensitive: Bool? = nil, wizardinputpending: Bool? = nil, + wizardactionaccepted: Bool? = nil, action: AnyCodable, agentdraft: String? = nil, agentid: String? = nil, @@ -10897,6 +10899,7 @@ public struct SystemAgentChatResult: Codable, Sendable { self.reply = reply self.sensitive = sensitive self.wizardinputpending = wizardinputpending + self.wizardactionaccepted = wizardactionaccepted self.action = action self.agentdraft = agentdraft self.agentid = agentid @@ -10911,6 +10914,7 @@ public struct SystemAgentChatResult: Codable, Sendable { case reply case sensitive case wizardinputpending = "wizardInputPending" + case wizardactionaccepted = "wizardActionAccepted" case action case agentdraft = "agentDraft" case agentid = "agentId" @@ -10943,21 +10947,25 @@ public struct SystemAgentChatHistoryTurn: Codable, Sendable { public let role: AnyCodable public let text: String public let at: Double + public let wizardaction: [String: AnyCodable]? public init( role: AnyCodable, text: String, - at: Double) + at: Double, + wizardaction: [String: AnyCodable]? = nil) { self.role = role self.text = text self.at = at + self.wizardaction = wizardaction } private enum CodingKeys: String, CodingKey { case role case text case at + case wizardaction = "wizardAction" } } diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json index 20a72766e0df..35b6293a2925 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json @@ -1 +1 @@ -{"contentHash":"daefcf9a55c958e5724fb853b5a8937655770cb40c809ce0541d18f5e0b851f4","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} +{"contentHash":"79a4c99fa78209550335cbcf81b39ce261be41a90de04b64eb69526ef84349bc","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json index 72e133e6cdf3..df699308c24b 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json @@ -1 +1 @@ -{"contentHash":"aeb886b36931a7aa47cedcfab228ca2c9cdee8f6029683c169623758a3726c9c","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} +{"contentHash":"fdcb3e2536e7d4e3090322fac2793f497a0b6f331a8b583fa6156007721b177f","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-core.json b/docs/.generated/plugin-sdk-api-baseline/channel-core.json index 2a729de4cd98..a7a1893477d5 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-core.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-core.json @@ -1 +1 @@ -{"contentHash":"56054f5fe80bdef27618a0163a04091777dc156391221d25e5ab69f9958cf00c","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} +{"contentHash":"3bdfd31e7decd15278ea7864d8c16a0674be246e827b55b389593dcba3f2066c","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json index a2eed76ec142..d8be27672e88 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json @@ -1 +1 @@ -{"contentHash":"cddc0ebd6cc887249ff8aaea292c7dfc400fb39efb08d18c1fda00340e6c2cc0","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} +{"contentHash":"a5ff017a6e98b53dee31102591e7304d4d754b150b2a7684c257059921fa2523","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-message.json b/docs/.generated/plugin-sdk-api-baseline/channel-message.json index ba9528a59c4e..d80911b2febe 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-message.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-message.json @@ -1 +1 @@ -{"contentHash":"d413b2031a7ab19e5a6554058418625ec017f9fe5fa3a980a095268f09d69bd8","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} +{"contentHash":"5d62b0618558e9a0093e4c8ad09b6cd143ed5b4c214de2eeefb255781a34c3bd","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json index c0718b5fb4b5..a2e5cdce5147 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json @@ -1 +1 @@ -{"contentHash":"cc4911393909d8e0bc3a1ce72a417af3dcc9b741e4106c0a135301d5e8a405e6","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} +{"contentHash":"32d2d5450bc2e08ebd3e2afc9302cf321469684f5997a059dddf28ace36675a8","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json index a896e209b77f..d214c96e6405 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json @@ -1 +1 @@ -{"contentHash":"6f15d12df2378c1185bce1e8ef119d170b769b275dafc03203e2bef2441faf5f","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} +{"contentHash":"3fea3804bec8232a55c95d3abb5bab8de5d8778652bddfafbb6362d27a4b3460","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} diff --git a/docs/.generated/plugin-sdk-api-baseline/core.json b/docs/.generated/plugin-sdk-api-baseline/core.json index 14bf54e5d3fd..7a4fc83b4b38 100644 --- a/docs/.generated/plugin-sdk-api-baseline/core.json +++ b/docs/.generated/plugin-sdk-api-baseline/core.json @@ -1 +1 @@ -{"contentHash":"68125b14406c4772cfbda02053929383acf0f895e19446a1e53ad5b36f3c1c4a","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} +{"contentHash":"117093ae03e310c9ce4455f24e43fc678b8737af600eea1d0066137c2fa4651c","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/discord.json b/docs/.generated/plugin-sdk-api-baseline/discord.json index 0b3ad7f546f4..2fe6b5157a12 100644 --- a/docs/.generated/plugin-sdk-api-baseline/discord.json +++ b/docs/.generated/plugin-sdk-api-baseline/discord.json @@ -1 +1 @@ -{"contentHash":"1e7d0b76629055a48e9156916667dba769f3b050ec71a607e6cb45ebde4a196e","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} +{"contentHash":"b5d69672e4813d9de0f99bf51d1baedd73b167a54126a0c984f65dd84945699c","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} diff --git a/docs/.generated/plugin-sdk-api-baseline/gateway-runtime.json b/docs/.generated/plugin-sdk-api-baseline/gateway-runtime.json index 40beedb5dce6..a464d55adb0e 100644 --- a/docs/.generated/plugin-sdk-api-baseline/gateway-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/gateway-runtime.json @@ -1 +1 @@ -{"contentHash":"1740335f4eaa15f70266b6d3b1a92b378ab40cad4b33fbd066079ac444ecdc7a","entrypoint":"gateway-runtime","importSpecifier":"openclaw/plugin-sdk/gateway-runtime"} +{"contentHash":"096282e453532f80d63ac296d2cc53ad65eb38f23703ea27e359cc215c7adfe4","entrypoint":"gateway-runtime","importSpecifier":"openclaw/plugin-sdk/gateway-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json index 8fa91a94b709..d5adfaf16c24 100644 --- a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json +++ b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json @@ -1 +1 @@ -{"contentHash":"7cd9f27750463d173499efaeaf378088fb03acff4b3ccede9612b20f8de272d7","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} +{"contentHash":"d93e8de2beeea1902998af40aae725e6d6318cb011810d3478a776a016f62723","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} diff --git a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json index dd2e4bc6bb2c..af1a417186a4 100644 --- a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json @@ -1 +1 @@ -{"contentHash":"747e38c5637b89bebd30c1ad5872e59b5d49ca192b7d683ab5bc520007e38991","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} +{"contentHash":"60a4f0f0501fb64019a7aa9c3e66f64ce9c388fb38dadd69d71b64a467ecbcbd","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json index 2af5525fc04d..07d8803b10f4 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json @@ -1 +1 @@ -{"contentHash":"6856c12fd342a8779ffbddbb10e0792e397ff6bb0cff74f328110f5b80ccf23b","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} +{"contentHash":"283ada2bb6c4499207b08f1d23c80919daa4fd139c6e02810c2c15f7a90fec58","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json index 91e3272685af..fc23cad8c835 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json @@ -1 +1 @@ -{"contentHash":"ad5d7b53a87a8f4e23e1a71c3dc6c796d28bd733dd1b78a3a793256dbb53ee5f","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} +{"contentHash":"e5b10b255ad2bd9ed54f10017e1e09440962ed846ba885059e13dab10b27811f","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json index 12380dd1677c..68d9b93b88f5 100644 --- a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json @@ -1 +1 @@ -{"contentHash":"c251e0eabb3292d4da6814f775ade4cbfdb96df889d384be9fa1380a6c7fe88c","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} +{"contentHash":"5055e33c4e0ffe0cb368d7a8e7970d5e97fd7d90913485e8a2bcecb1f97f92ef","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json index 1519c7b5fa0d..790a9c522529 100644 --- a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json +++ b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json @@ -1 +1 @@ -{"contentHash":"3fa3d78190c0b7785cb6003f1af5e3249af3585aded8608c26fce00d6b36a9a5","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} +{"contentHash":"7f5f39112df30ff5d32513258437bef16e8304e01e1d61a50f754c8e26831cbc","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} diff --git a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json index 2f4679572d1a..815557058094 100644 --- a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json +++ b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json @@ -1 +1 @@ -{"contentHash":"b733af3f69aa036b33f3e7586619fb6d988d89a51bc4c3bfff0af0fc5e7a56fc","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} +{"contentHash":"983f60fcee19de0cc99dd2db216bea32e45054bffc50cfe3d6d382a7f51c7a41","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} diff --git a/packages/gateway-protocol/CHANGELOG.md b/packages/gateway-protocol/CHANGELOG.md index 0250212f2ec9..7eb102a62c15 100644 --- a/packages/gateway-protocol/CHANGELOG.md +++ b/packages/gateway-protocol/CHANGELOG.md @@ -15,6 +15,7 @@ version and the additive schema surface. Dates are authoring dates (2026). - Remove dead protocol surfaces and add since-vintage metadata to retained schemas and methods. - Add optional `step` on `SystemAgentChatResult` carrying the full awaited wizard step. - Add owner-bound active-wizard snapshots to Custodian chat history for reload recovery. +- Add capability-gated accepted-action outcomes and receipt prompts to Custodian chat turns. ## Protocol v4 (current) diff --git a/packages/gateway-protocol/src/schema/frames.ts b/packages/gateway-protocol/src/schema/frames.ts index 9bb01aecf996..6fdf704a73c1 100644 --- a/packages/gateway-protocol/src/schema/frames.ts +++ b/packages/gateway-protocol/src/schema/frames.ts @@ -10,6 +10,7 @@ export const GATEWAY_SERVER_CAPS = { BOARD_WIDGET_PUT_CANVAS_DOC: "board-widget-put-canvas-doc", CHAT_SEND_ROUTING_CONTRACT: "chat-send-routing-contract", SYSTEM_AGENT_WIZARD_CANCEL: "openclaw-chat-wizard-cancel", + SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS: "openclaw-chat-wizard-action-receipts", SYSTEM_AGENT_CHAT_HISTORY_SESSION_RECOVERY: "openclaw-chat-history-session-recovery", SYSTEM_AGENT_SETUP_MODEL_REF: "openclaw-setup-model-ref", TASK_SUGGESTIONS_ACCEPT_MODES: "taskSuggestions.acceptModes", diff --git a/packages/gateway-protocol/src/schema/openclaw.test.ts b/packages/gateway-protocol/src/schema/openclaw.test.ts index 4f83263951e1..90dd14ca2dec 100644 --- a/packages/gateway-protocol/src/schema/openclaw.test.ts +++ b/packages/gateway-protocol/src/schema/openclaw.test.ts @@ -7,6 +7,7 @@ import { } from "../index.js"; import { SystemAgentChatQuestionSchema, + SystemAgentChatResultSchema, SystemAgentChatHistoryResultSchema, SystemAgentSetupDetectResultSchema, SystemAgentSetupVerifyResultSchema, @@ -88,6 +89,23 @@ describe("OpenClaw chat question protocol", () => { }); }); +describe("OpenClaw chat result protocol", () => { + const result = { sessionId: "session-1", reply: "Choose again.", action: "none" }; + + it("accepts explicit typed wizard-action outcomes", () => { + expect(Value.Check(SystemAgentChatResultSchema, result)).toBe(true); + expect( + Value.Check(SystemAgentChatResultSchema, { ...result, wizardActionAccepted: true }), + ).toBe(true); + expect( + Value.Check(SystemAgentChatResultSchema, { ...result, wizardActionAccepted: false }), + ).toBe(true); + expect( + Value.Check(SystemAgentChatResultSchema, { ...result, wizardActionAccepted: "yes" }), + ).toBe(false); + }); +}); + describe("OpenClaw chat history protocol", () => { it("accepts the default request and bounds explicit limits", () => { expect(validateSystemAgentChatHistoryParams({})).toBe(true); @@ -131,6 +149,42 @@ describe("OpenClaw chat history protocol", () => { }), ).toBe(true); }); + + it("accepts minimal wizard-action presentation metadata on live session turns", () => { + const turn = { + role: "user", + text: "Slack bot", + at: 2, + wizardAction: { + kind: "answer", + prompt: "How should OpenClaw appear in Slack?", + }, + }; + expect(Value.Check(SystemAgentChatHistoryResultSchema, { turns: [turn] })).toBe(true); + expect( + Value.Check(SystemAgentChatHistoryResultSchema, { + turns: [{ ...turn, sessionId: "slack-session" }], + }), + ).toBe(false); + expect( + Value.Check(SystemAgentChatHistoryResultSchema, { + turns: [{ ...turn, wizardAction: { ...turn.wizardAction, kind: "unknown" } }], + }), + ).toBe(false); + expect( + Value.Check(SystemAgentChatHistoryResultSchema, { + turns: [ + { + ...turn, + wizardAction: { + ...turn.wizardAction, + externalUrl: "https://auth.example.test/device?token=secret", + }, + }, + ], + }), + ).toBe(false); + }); }); describe("OpenClaw setup detection protocol", () => { diff --git a/packages/gateway-protocol/src/schema/openclaw.ts b/packages/gateway-protocol/src/schema/openclaw.ts index 791ed07d670f..3772efc8f7ca 100644 --- a/packages/gateway-protocol/src/schema/openclaw.ts +++ b/packages/gateway-protocol/src/schema/openclaw.ts @@ -86,6 +86,8 @@ export const SystemAgentChatResultSchema = closedObject({ sensitive: Type.Optional(Type.Boolean()), /** The hosted wizard will consume the next message as its current step answer. */ wizardInputPending: Type.Optional(Type.Boolean()), + /** Whether the submitted typed wizard action passed owner-side validation. */ + wizardActionAccepted: Type.Optional(Type.Boolean()), action: Type.Union([ Type.Literal("none"), // The user asked to talk to their agent; clients should move to their @@ -113,10 +115,18 @@ export const SystemAgentChatHistoryParamsSchema = closedObject({ sessionId: Type.Optional(NonEmptyString), }); +export const SystemAgentChatHistoryWizardActionSchema = closedObject({ + kind: Type.Union([Type.Literal("answer"), Type.Literal("cancel")]), + /** Ordinary non-sensitive prompt copy used to label the receipt. */ + prompt: Type.Optional(Type.String()), +}); + export const SystemAgentChatHistoryTurnSchema = closedObject({ role: Type.Union([Type.Literal("user"), Type.Literal("assistant")]), text: Type.String(), at: Type.Number(), + /** Present only on accepted typed controls from a live recovered session. */ + wizardAction: Type.Optional(SystemAgentChatHistoryWizardActionSchema), }); export const SystemAgentChatHistoryResultSchema = closedObject({ @@ -379,6 +389,9 @@ export type SystemAgentWizardCancel = Static; export type SystemAgentChatResult = Static; export type SystemAgentChatHistoryParams = Static; +export type SystemAgentChatHistoryWizardAction = Static< + typeof SystemAgentChatHistoryWizardActionSchema +>; export type SystemAgentChatHistoryTurn = Static; export type SystemAgentChatHistoryResult = Static; export type SystemChangeEntry = Static; diff --git a/src/gateway/server-methods/system-agent-chat-history.test.ts b/src/gateway/server-methods/system-agent-chat-history.test.ts index 6785c14b682c..16e8b177a800 100644 --- a/src/gateway/server-methods/system-agent-chat-history.test.ts +++ b/src/gateway/server-methods/system-agent-chat-history.test.ts @@ -1,7 +1,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { createDeferred } from "../../../test/helpers/promise.js"; +import { appendTranscriptTurn } from "../../system-agent/transcript-store.js"; import { appendSystemAgentRecoveryHistory, + persistSystemAgentEngineHistory, setSystemAgentRecoveryHistory, systemAgentChatHistoryHandler, } from "./system-agent-chat-history.js"; @@ -15,10 +17,12 @@ const turns = [ ]; const transcriptStoreMocks = vi.hoisted(() => ({ + appendTranscriptTurn: vi.fn(), readTranscriptTail: vi.fn(), })); vi.mock("../../system-agent/transcript-store.js", () => ({ + appendTranscriptTurn: transcriptStoreMocks.appendTranscriptTurn, readTranscriptTail: transcriptStoreMocks.readTranscriptTail, })); @@ -62,9 +66,64 @@ function makeInvocation(params: { describe("openclaw.chat.history wizard recovery", () => { beforeEach(() => { + transcriptStoreMocks.appendTranscriptTurn.mockReset(); transcriptStoreMocks.readTranscriptTail.mockReset().mockReturnValue(turns); }); + it("keeps accepted action metadata on live recovery turns only", () => { + const wizardAction = { + kind: "cancel" as const, + }; + const recoveryTurns = persistSystemAgentEngineHistory( + { + historySince: () => [ + { role: "user", text: "Cancel" }, + { role: "assistant", text: "Twitch setup cancelled." }, + ], + }, + 0, + { wizardAction, wizardActionAccepted: true }, + ); + + expect(recoveryTurns).toEqual([ + expect.objectContaining({ + role: "user", + wizardAction, + }), + expect.objectContaining({ + role: "assistant", + }), + ]); + expect(vi.mocked(appendTranscriptTurn).mock.calls.map(([turn]) => turn)).toEqual([ + expect.objectContaining({ role: "user", text: "Cancel" }), + expect.objectContaining({ role: "assistant", text: "Twitch setup cancelled." }), + ]); + for (const [turn] of vi.mocked(appendTranscriptTurn).mock.calls) { + expect(turn).not.toHaveProperty("wizardAction"); + } + }); + + it("omits action metadata when the engine rejects the typed answer", () => { + persistSystemAgentEngineHistory( + { + historySince: () => [ + { role: "user", text: "Invalid value" }, + { role: "assistant", text: "Choose again." }, + ], + }, + 0, + { + wizardAction: { kind: "answer", prompt: "Port" }, + wizardActionAccepted: false, + }, + ); + + expect(vi.mocked(appendTranscriptTurn)).toHaveBeenCalledTimes(2); + for (const [turn] of vi.mocked(appendTranscriptTurn).mock.calls) { + expect(turn).not.toHaveProperty("wizardAction"); + } + }); + it("returns an active wizard only to its bound owner", async () => { const activeWizardStep = vi.fn().mockResolvedValue({ id: "secret", diff --git a/src/gateway/server-methods/system-agent-chat-history.ts b/src/gateway/server-methods/system-agent-chat-history.ts index cd8b9a359c69..890e41e5556e 100644 --- a/src/gateway/server-methods/system-agent-chat-history.ts +++ b/src/gateway/server-methods/system-agent-chat-history.ts @@ -1,10 +1,11 @@ import { validateSystemAgentChatHistoryParams, type SystemAgentChatHistoryTurn, + type SystemAgentChatHistoryWizardAction, } from "../../../packages/gateway-protocol/src/index.js"; import type { SystemAgentChatEngine } from "../../system-agent/chat-engine.js"; import { resolveSystemAgentDelegationKey } from "../../system-agent/delegation-session.js"; -import { readTranscriptTail } from "../../system-agent/transcript-store.js"; +import { appendTranscriptTurn, readTranscriptTail } from "../../system-agent/transcript-store.js"; import { runSystemAgentGatewayTask } from "./system-agent-gateway-queue.js"; import { getSystemAgentSessionQueue } from "./system-agent-session-queue.js"; import type { GatewayClient, GatewayRequestHandler } from "./types.js"; @@ -36,6 +37,29 @@ function readSystemAgentRecoveryHistory( return (recoveryTurnsByEngine.get(engine) ?? []).slice(-limit); } +export function persistSystemAgentEngineHistory( + engine: Pick, + startIndex: number, + params: { + wizardAction?: SystemAgentChatHistoryWizardAction; + wizardActionAccepted?: boolean; + } = {}, +): SystemAgentChatHistoryTurn[] { + const at = Date.now(); + let wizardAction = params.wizardActionAccepted === true ? params.wizardAction : undefined; + const recoveryTurns: SystemAgentChatHistoryTurn[] = []; + for (const turn of engine.historySince(startIndex)) { + const action = turn.role === "user" ? wizardAction : undefined; + const recoveryTurn = { ...turn, at, ...(action ? { wizardAction: action } : {}) }; + appendTranscriptTurn({ ...turn, at }); + recoveryTurns.push(recoveryTurn); + if (action) { + wizardAction = undefined; + } + } + return recoveryTurns; +} + export function resolveSystemAgentSessionOwnerKey(params: { delegation?: { agentId?: string; sessionKey?: string }; client: GatewayClient | null; diff --git a/src/gateway/server-methods/system-agent-chat-turn.test.ts b/src/gateway/server-methods/system-agent-chat-turn.test.ts index 822091b99f94..9add1c1b663f 100644 --- a/src/gateway/server-methods/system-agent-chat-turn.test.ts +++ b/src/gateway/server-methods/system-agent-chat-turn.test.ts @@ -123,6 +123,7 @@ describe("system-agent chat input", () => { reply: { text: "Choose a channel.", action: "none", + wizardActionAccepted: false, step: { id: "channel", type: "select", @@ -135,6 +136,7 @@ describe("system-agent chat input", () => { sessionId: "s1", reply: "Choose a channel.", action: "none", + wizardActionAccepted: false, step: { id: "channel", type: "select" }, }); }); diff --git a/src/gateway/server-methods/system-agent-chat-turn.ts b/src/gateway/server-methods/system-agent-chat-turn.ts index 34b730bce5ee..02f8b45e1bc2 100644 --- a/src/gateway/server-methods/system-agent-chat-turn.ts +++ b/src/gateway/server-methods/system-agent-chat-turn.ts @@ -82,6 +82,9 @@ export function buildSystemAgentChatResult(params: { : {}), ...(params.reply.sensitive === true ? { sensitive: true } : {}), ...(params.reply.wizardInputPending === true ? { wizardInputPending: true } : {}), + ...(params.reply.wizardActionAccepted !== undefined + ? { wizardActionAccepted: params.reply.wizardActionAccepted } + : {}), ...(params.reply.question ? { question: params.reply.question } : {}), ...(params.reply.step ? { step: params.reply.step } : {}), ...(params.proposalId ? { needsApproval: true, proposalId: params.proposalId } : {}), diff --git a/src/gateway/server-methods/system-agent.ts b/src/gateway/server-methods/system-agent.ts index d8777bbb1af0..76eab7261ab9 100644 --- a/src/gateway/server-methods/system-agent.ts +++ b/src/gateway/server-methods/system-agent.ts @@ -10,7 +10,6 @@ import { validateSystemAgentSetupAuthStartParams, validateSystemAgentSetupDetectParams, validateSystemAgentSetupVerifyParams, - type SystemAgentChatHistoryTurn, type SystemAgentChatQuestion, } from "../../../packages/gateway-protocol/src/index.js"; import { @@ -34,11 +33,7 @@ import { isSystemAgentInferenceUnavailableError } from "../../system-agent/infer import { buildNewAgentWelcome } from "../../system-agent/new-agent-welcome.js"; import { buildOnboardingWelcome } from "../../system-agent/onboarding-welcome.js"; import { describeSystemAgentPersistentOperation } from "../../system-agent/operations.js"; -import { - appendTranscriptReset, - appendTranscriptTurn, - readTranscriptTail, -} from "../../system-agent/transcript-store.js"; +import { appendTranscriptReset, readTranscriptTail } from "../../system-agent/transcript-store.js"; import { resolveUserPath } from "../../utils.js"; import { WizardSession } from "../../wizard/session.js"; import { @@ -54,6 +49,7 @@ import { } from "./setup-admission.js"; import { appendSystemAgentRecoveryHistory, + persistSystemAgentEngineHistory, resolveSystemAgentSessionOwnerKey, setSystemAgentRecoveryHistory, systemAgentChatHistoryHandler, @@ -121,23 +117,6 @@ async function evictOldestSession( } } -function persistEngineHistory( - engine: SystemAgentChatSession["engine"], - startIndex: number, -): SystemAgentChatHistoryTurn[] { - const at = Date.now(); - const turns: SystemAgentChatHistoryTurn[] = []; - for (const turn of engine.historySince(startIndex)) { - turns.push({ ...turn, at }); - } - for (const turn of turns) { - // Engine history is authoritative here: sensitive user text has already - // been replaced by the mask marker before it crosses this boundary. - appendTranscriptTurn(turn); - } - return turns; -} - function queueDelegatedApproval(params: { context: GatewayRequestContext; sessions: Map; @@ -577,7 +556,7 @@ export const systemAgentHandlers: GatewayRequestHandlers = { respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, error.message)); return; } - const recoveryTurns = persistEngineHistory(engine, welcomeHistoryStart); + const recoveryTurns = persistSystemAgentEngineHistory(engine, welcomeHistoryStart); await evictOldestSession(sessions, context); session = { engine, @@ -645,7 +624,7 @@ export const systemAgentHandlers: GatewayRequestHandlers = { } catch (error) { appendSystemAgentRecoveryHistory( session.engine, - persistEngineHistory(session.engine, historyStart), + persistSystemAgentEngineHistory(session.engine, historyStart), ); if (error instanceof SystemAgentWizardAnswerError) { respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, error.message)); @@ -677,7 +656,10 @@ export const systemAgentHandlers: GatewayRequestHandlers = { } appendSystemAgentRecoveryHistory( session.engine, - persistEngineHistory(session.engine, historyStart), + persistSystemAgentEngineHistory(session.engine, historyStart, { + wizardAction: reply.wizardAction, + wizardActionAccepted: reply.wizardActionAccepted, + }), ); const delegation = params.delegation; let proposalId: string | undefined; diff --git a/src/gateway/server.auth.default-token.suite.ts b/src/gateway/server.auth.default-token.suite.ts index 0cf77f20a87b..cb1d70b016ee 100644 --- a/src/gateway/server.auth.default-token.suite.ts +++ b/src/gateway/server.auth.default-token.suite.ts @@ -162,6 +162,9 @@ export function registerDefaultAuthTokenSuite(): void { expect(payload?.features?.capabilities).toContain( GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, ); + expect(payload?.features?.capabilities).toContain( + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, + ); expect(payload?.features?.capabilities).toContain( GATEWAY_SERVER_CAPS.SYSTEM_AGENT_CHAT_HISTORY_SESSION_RECOVERY, ); diff --git a/src/gateway/server/ws-connection/connect-hello.ts b/src/gateway/server/ws-connection/connect-hello.ts index 04a5d2b4ee83..f2034fe0d911 100644 --- a/src/gateway/server/ws-connection/connect-hello.ts +++ b/src/gateway/server/ws-connection/connect-hello.ts @@ -121,6 +121,7 @@ export async function sendGatewayHello( GATEWAY_SERVER_CAPS.BOARD_WIDGET_PUT_CANVAS_DOC, GATEWAY_SERVER_CAPS.CHAT_SEND_ROUTING_CONTRACT, GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, GATEWAY_SERVER_CAPS.SYSTEM_AGENT_CHAT_HISTORY_SESSION_RECOVERY, GATEWAY_SERVER_CAPS.SYSTEM_AGENT_SETUP_MODEL_REF, GATEWAY_SERVER_CAPS.TASK_SUGGESTIONS_ACCEPT_MODES, diff --git a/src/system-agent/chat-engine.ts b/src/system-agent/chat-engine.ts index 967ec1572b90..8c1b9d341ac9 100644 --- a/src/system-agent/chat-engine.ts +++ b/src/system-agent/chat-engine.ts @@ -173,7 +173,15 @@ export class SystemAgentChatEngine { const turn = this.turnQueue.then(async () => { await this.requireVerifiedInference(); const result = await this.router.answerWizard(this.wizard.answer(answer)); - return this.completeTurn({ text: result.text, action: "none" }, result.userHistoryText); + return this.completeTurn( + { + text: result.text, + action: "none", + wizardActionAccepted: result.accepted, + wizardAction: result.wizardAction, + }, + result.userHistoryText, + ); }); this.turnQueue = turn.catch(() => undefined); return await turn; @@ -182,7 +190,15 @@ export class SystemAgentChatEngine { async cancelWizard(cancel: SystemAgentWizardCancel): Promise { const turn = this.turnQueue.then(async () => { const result = await this.router.answerWizard(this.wizard.cancel(cancel)); - return this.completeTurn({ text: result.text, action: "none" }, result.userHistoryText); + return this.completeTurn( + { + text: result.text, + action: "none", + wizardActionAccepted: result.accepted, + wizardAction: result.wizardAction, + }, + result.userHistoryText, + ); }); this.turnQueue = turn.catch(() => undefined); return await turn; diff --git a/src/system-agent/chat-wizard-host.test.ts b/src/system-agent/chat-wizard-host.test.ts index 3cb19d65cd86..01d3a19d9b82 100644 --- a/src/system-agent/chat-wizard-host.test.ts +++ b/src/system-agent/chat-wizard-host.test.ts @@ -481,12 +481,42 @@ describe("SystemAgentChatEngine wizard", () => { const prompt = await engine.handle("connect telegram"); const stepId = expectDefined(prompt.step?.id, "expected an active wizard step"); - await engine.answerWizard({ stepId, value: "beta" }); + const answered = await engine.answerWizard({ stepId, value: "beta" }); expect(selected).toBe("beta"); + expect(answered.wizardActionAccepted).toBe(true); + expect(answered.wizardAction).toEqual({ kind: "answer", prompt: "Choose one" }); expect(engine.historySince(0)).toContainEqual({ role: "user", text: "Beta" }); }); + it("records typed wizard validation acceptance at the wizard owner", async () => { + useTempStateDir(); + const engine = new SystemAgentChatEngine({ + surface: "gateway", + runAgentTurn: async () => null, + planWithAssistant: async () => null, + deps: { loadOverview: fakeOverviewLoader() }, + runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { + await prompter.text({ + message: "Port", + validate: (value) => (value === "18789" ? undefined : "Enter port 18789"), + }); + }, + }); + + const prompt = await engine.handle("connect telegram"); + const stepId = expectDefined(prompt.step?.id, "expected an active wizard step"); + const invalid = await engine.answerWizard({ stepId, value: "banana" }); + expect(invalid.wizardActionAccepted).toBe(false); + expect(invalid.wizardAction).toEqual({ kind: "answer", prompt: "Port" }); + expect(invalid.step?.id).toBe(stepId); + expect(invalid.text).toContain("Enter port 18789"); + + const accepted = await engine.answerWizard({ stepId, value: "18789" }); + expect(accepted.wizardActionAccepted).toBe(true); + expect(accepted.step).toBeUndefined(); + }); + it("cancels the current hosted wizard through a typed direct action", async () => { useTempStateDir(); const engine = new SystemAgentChatEngine({ @@ -495,7 +525,7 @@ describe("SystemAgentChatEngine wizard", () => { planWithAssistant: async () => null, deps: { loadOverview: fakeOverviewLoader() }, runChannelSetupWizard: async (_channel: string, prompter: WizardPrompter) => { - await prompter.text({ message: "Bot token" }); + await prompter.text({ message: "Bot token", sensitive: true }); }, }); @@ -504,6 +534,8 @@ describe("SystemAgentChatEngine wizard", () => { const cancelled = await engine.cancelWizard({ stepId }); expect(cancelled.text).toContain("cancelled"); + expect(cancelled.wizardActionAccepted).toBe(true); + expect(cancelled.wizardAction).toEqual({ kind: "cancel" }); expect(cancelled.step).toBeUndefined(); expect(cancelled.wizardInputPending).toBeUndefined(); expect(engine.historySince(0)).toContainEqual({ role: "user", text: "Cancel" }); diff --git a/src/system-agent/chat-wizard-host.ts b/src/system-agent/chat-wizard-host.ts index d5b7e51dfa51..26087e6c6108 100644 --- a/src/system-agent/chat-wizard-host.ts +++ b/src/system-agent/chat-wizard-host.ts @@ -1,5 +1,6 @@ import type { SystemAgentChatQuestion, + SystemAgentChatHistoryWizardAction, SystemAgentWizardCancel, WizardAnswer, } from "../../packages/gateway-protocol/src/index.js"; @@ -30,6 +31,9 @@ export type SystemAgentChatReply = { agentDraft?: "hatch"; sensitive?: boolean; wizardInputPending?: boolean; + /** The submitted typed wizard action passed or failed owner-side validation. */ + wizardActionAccepted?: boolean; + wizardAction?: SystemAgentChatHistoryWizardAction; handoff?: SystemAgentOperation; question?: SystemAgentChatQuestion; step?: WizardStep; @@ -42,9 +46,22 @@ export type ChatWizardResult = { }; export type ChatWizardAnswerResult = ChatWizardResult & { + accepted: boolean; userHistoryText: string; + wizardAction: SystemAgentChatHistoryWizardAction; }; +function wizardActionReceipt( + step: WizardStep, + kind: SystemAgentChatHistoryWizardAction["kind"], +): SystemAgentChatHistoryWizardAction { + const prompt = + !step.sensitive && !step.deviceCode && !step.externalUrl + ? (step.title ?? step.message) + : undefined; + return { kind, ...(prompt ? { prompt } : {}) }; +} + export type ChatWizardHostDependencies = { runChannelSetupWizard?: ( channel: string, @@ -311,7 +328,9 @@ export class ChatWizardHost { : await this.pump(); return { ...result, + accepted: validationError === undefined, userHistoryText: formatStructuredWizardAnswerForHistory(step, answer.value), + wizardAction: wizardActionReceipt(step, "answer"), }; } @@ -327,7 +346,12 @@ export class ChatWizardHost { if (!bridge.session.cancel()) { throw new SystemAgentWizardAnswerError("The hosted wizard cannot be cancelled right now."); } - return { ...(await this.pump()), userHistoryText: "Cancel" }; + return { + ...(await this.pump()), + accepted: true, + userHistoryText: "Cancel", + wizardAction: wizardActionReceipt(step, "cancel"), + }; } async resolveReply(text: string): Promise { diff --git a/ui/src/components/custodian/custodian-panel.test.ts b/ui/src/components/custodian/custodian-panel.test.ts index e350f7249c9a..f9db66b158ca 100644 --- a/ui/src/components/custodian/custodian-panel.test.ts +++ b/ui/src/components/custodian/custodian-panel.test.ts @@ -61,8 +61,24 @@ describe("custodian panel", () => { store.connect(context, "caretaker"); await vi.waitFor(() => expect(request).toHaveBeenCalledOnce()); store.messages = [ - { id: 1, role: "assistant", text: "Ready.", at: 1, question: null, step: null }, - { id: 2, role: "user", text: "Check this system", at: 2, question: null, step: null }, + { + id: 1, + role: "assistant", + text: "Ready.", + at: 1, + question: null, + step: null, + structuredResponse: null, + }, + { + id: 2, + role: "user", + text: "Check this system", + at: 2, + question: null, + step: null, + structuredResponse: null, + }, ]; panel.suppressed = false; @@ -116,7 +132,15 @@ describe("custodian panel", () => { store.connect(context, "caretaker"); await vi.waitFor(() => expect(request).toHaveBeenCalledOnce()); store.messages = [ - { id: 1, role: "user", text: "Check this system", at: 1, question: null, step: null }, + { + id: 1, + role: "user", + text: "Check this system", + at: 1, + question: null, + step: null, + structuredResponse: null, + }, ]; panel.available = false; panel.suppressed = false; @@ -141,8 +165,17 @@ describe("custodian panel", () => { at: 1, question: null, step: null, + structuredResponse: null, + }, + { + id: 2, + role: "user", + text: "Continue setup", + at: 2, + question: null, + step: null, + structuredResponse: null, }, - { id: 2, role: "user", text: "Continue setup", at: 2, question: null, step: null }, ]; panel.suppressed = false; diff --git a/ui/src/e2e/custodian-event-nudge.e2e.test.ts b/ui/src/e2e/custodian-event-nudge.e2e.test.ts index 59d65ed62c46..8d9c65942372 100644 --- a/ui/src/e2e/custodian-event-nudge.e2e.test.ts +++ b/ui/src/e2e/custodian-event-nudge.e2e.test.ts @@ -332,7 +332,10 @@ suite.define(() => { }, async ({ page }) => { const gateway = await installMockGateway(page, { - featureCapabilities: [GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL], + featureCapabilities: [ + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, + ], featureMethods: ["chat.metadata", "chat.startup", "openclaw.chat"], methodResponses: { "openclaw.chat": { @@ -439,6 +442,7 @@ suite.define(() => { sessionId: "e2e-rich-wizard", reply: "Choose features.", action: "none", + wizardActionAccepted: true, wizardInputPending: true, step: { id: "features", @@ -462,6 +466,7 @@ suite.define(() => { sessionId: "e2e-rich-wizard", reply: "Enter the secret.", action: "none", + wizardActionAccepted: true, sensitive: true, wizardInputPending: true, step: { @@ -488,6 +493,7 @@ suite.define(() => { sessionId: "e2e-rich-wizard", reply: "Name this connection.", action: "none", + wizardActionAccepted: true, wizardInputPending: true, step: { id: "label", @@ -522,6 +528,7 @@ suite.define(() => { sessionId: "e2e-rich-wizard", reply: "Confirm setup.", action: "none", + wizardActionAccepted: true, wizardInputPending: true, step: { id: "confirm", @@ -535,22 +542,40 @@ suite.define(() => { await gateway.deferNext("openclaw.chat"); await yesButton.click(); - await expect.poll(() => noButton.isDisabled()).toBe(true); - await expect.poll(() => yesButton.isDisabled()).toBe(true); - await expect.poll(() => cancelButton.isDisabled()).toBe(true); - for (const button of [noButton, yesButton, cancelButton]) { - const restingStyle = await button.evaluate(readInteractionStyle); - await button.hover(); - expect(await button.evaluate(readInteractionStyle)).toEqual(restingStyle); - expect(restingStyle.cursor).toBe("not-allowed"); + const confirmationReceipt = page.locator(".custodian__structured-response", { + hasText: "Yes", + }); + await confirmationReceipt.waitFor(); + expect( + await confirmationReceipt.locator(".custodian__structured-response-status").textContent(), + ).toBe("Submitting answer"); + expect(await noButton.count()).toBe(0); + expect(await yesButton.count()).toBe(0); + + if (captureUiProofEnabled) { + await page.screenshot({ + animations: "disabled", + path: path.join(uiProofArtifactDir, "06-answer-receipt-submitting-desktop.png"), + }); } await gateway.resolveDeferred("openclaw.chat", { sessionId: "e2e-rich-wizard", reply: "Setup complete.", action: "none", + wizardActionAccepted: true, }); await page.getByText("Setup complete.").waitFor(); + expect( + await confirmationReceipt.locator(".custodian__structured-response-status").textContent(), + ).toBe("Answer submitted"); + + if (captureUiProofEnabled) { + await page.screenshot({ + animations: "disabled", + path: path.join(uiProofArtifactDir, "07-answer-receipt-submitted-desktop.png"), + }); + } const requests = await gateway.getRequests("openclaw.chat"); expect(requests.map((request) => request.params)).toEqual([ @@ -602,6 +627,7 @@ suite.define(() => { const gateway = await installMockGateway(page, { featureCapabilities: [ GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, GATEWAY_SERVER_CAPS.SYSTEM_AGENT_CHAT_HISTORY_SESSION_RECOVERY, ], featureMethods: ["chat.metadata", "chat.startup", "openclaw.chat", "openclaw.chat.history"], @@ -680,6 +706,7 @@ suite.define(() => { sessionId: "e2e-reload-wizard", reply: "Twitch setup cancelled.", action: "none", + wizardActionAccepted: true, }); await page.getByRole("button", { name: "Cancel", exact: true }).click(); await page.getByText("Twitch setup cancelled.").waitFor(); @@ -692,6 +719,15 @@ suite.define(() => { }); expect(requests[1]?.params).not.toHaveProperty("message"); expect(await page.locator(".custodian__wizard-step").count()).toBe(0); + expect(await page.locator(".custodian__structured-response-status").textContent()).toBe( + "Setup cancelled", + ); + if (captureUiProofEnabled) { + await page.screenshot({ + animations: "disabled", + path: path.join(uiProofArtifactDir, "08-cancel-receipt-submitted-desktop.png"), + }); + } } finally { await suite.closeBrowserContext(context); } diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index 8b5f2158044c..960d8cbafdd3 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -2365,6 +2365,13 @@ export const en: TranslationMap = { sessionRestarted: "{error} OpenClaw started a fresh session; earlier messages remain for context.", unsupportedGateway: "Update the Gateway to continue setup with OpenClaw.", + structured: { + response: "Setup answer", + submitting: "Submitting answer", + submitted: "Answer submitted", + cancelling: "Cancelling setup", + cancelled: "Setup cancelled", + }, panel: { title: "OpenClaw", close: "Close Ask OpenClaw", diff --git a/ui/src/pages/custodian/custodian-page.reload-recovery.test.ts b/ui/src/pages/custodian/custodian-page.reload-recovery.test.ts index 892ea02dd5c4..e285f72a5c3f 100644 --- a/ui/src/pages/custodian/custodian-page.reload-recovery.test.ts +++ b/ui/src/pages/custodian/custodian-page.reload-recovery.test.ts @@ -16,6 +16,7 @@ const recoveryOwner = { gatewayUrl, recoveryScope }; const recoveryClient = { recoveryScope, recoveryScopeReady: true } as never; const recoveryCapabilities = [ GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, GATEWAY_SERVER_CAPS.SYSTEM_AGENT_CHAT_HISTORY_SESSION_RECOVERY, ]; @@ -34,8 +35,46 @@ describe("Custodian wizard reload recovery", () => { if (params.sessionId === "live-wizard" && !cancelled) { return { turns: [ + { + role: "assistant", + text: "Choose a previous channel.", + at: 0, + }, + { + role: "user", + text: "Cancel", + at: 0, + wizardAction: { + kind: "cancel", + prompt: "Choose a previous channel.", + }, + }, { role: "user", text: "connect twitch", at: 1 }, - { role: "assistant", text: "Enter the secret.", at: 2 }, + { + role: "assistant", + text: [ + "How should OpenClaw appear in Twitch?", + "1. Bot", + "2. User", + "Reply with a number.", + "Say `cancel` to stop this setup.", + ].join("\n"), + at: 2, + }, + { + role: "user", + text: "Bot", + at: 3, + wizardAction: { + kind: "answer", + prompt: "How should OpenClaw appear in Twitch?", + }, + }, + { + role: "assistant", + text: "Enter the secret.\nType your answer.\nSay `cancel` to stop this setup.", + at: 4, + }, ], activeWizard: { sessionId: "live-wizard", @@ -59,6 +98,7 @@ describe("Custodian wizard reload recovery", () => { sessionId: "live-wizard", reply: "Twitch setup cancelled.", action: "none", + wizardActionAccepted: true, }; } freshChatCount += 1; @@ -99,6 +139,18 @@ describe("Custodian wizard reload recovery", () => { }); expect(recoveredInput.value).toBe(""); expect(second.page.textContent).toContain("Enter the secret."); + expect(second.page.textContent).toContain("Choose a previous channel."); + expect(second.page.textContent).toContain("How should OpenClaw appear in Twitch?"); + expect(second.page.querySelectorAll(".custodian__structured-response")).toHaveLength(2); + expect( + second.page.querySelector(".custodian__structured-response-icon--cancelled"), + ).not.toBeNull(); + expect(second.page.textContent).toContain("Setup cancelled"); + expect( + second.page.querySelectorAll(".custodian__structured-response")[1]?.textContent, + ).toContain("Bot"); + expect(second.page.querySelector(".chat-group.user")?.textContent).toContain("connect twitch"); + expect(second.page.querySelector("details")).toBeNull(); expect(request.mock.calls.filter(([method]) => method === "openclaw.chat")).toHaveLength(1); second.page.querySelector(".custodian__wizard-cancel")!.click(); @@ -107,6 +159,8 @@ describe("Custodian wizard reload recovery", () => { sessionId: "live-wizard", wizardCancel: { stepId: "secret" }, }); + expect(second.page.querySelector(".chat-group.user")?.textContent).not.toContain("Cancel"); + expect(second.page.querySelectorAll(".custodian__structured-response")).toHaveLength(3); expect(readCustodianRecoveryForClient(recoveryClient, gatewayUrl)).toBeNull(); second.provider.remove(); @@ -115,6 +169,119 @@ describe("Custodian wizard reload recovery", () => { expect(third.page.querySelector(".custodian__wizard-step")).toBeNull(); }); + it("restores ordinary question answers as user turns instead of wizard receipts", async () => { + reconcileCustodianRecoveryForScope( + recoveryOwner, + { + sessionId: "ordinary-answer-session", + reply: "Choose a channel.", + action: "none", + wizardInputPending: true, + step: { + id: "channel", + type: "select", + message: "Which channel?", + options: [{ label: "WhatsApp", value: "whatsapp" }], + }, + }, + "ordinary-answer-session", + ); + const request = vi.fn(async (method: string) => { + if (method !== "openclaw.chat.history") { + throw new Error(`unexpected method ${method}`); + } + return { + turns: [ + { + role: "assistant", + text: "What would you like to do first?", + at: 1, + }, + { + role: "user", + text: "Connect WhatsApp", + at: 2, + }, + { + role: "assistant", + text: "Choose a channel.", + at: 3, + }, + ], + activeWizard: { + sessionId: "ordinary-answer-session", + step: { + id: "channel", + type: "select", + message: "Which channel?", + options: [{ label: "WhatsApp", value: "whatsapp" }], + }, + }, + }; + }); + const { context } = createContext(request, ["openclaw.chat", "openclaw.chat.history"], { + gatewayCapabilities: recoveryCapabilities, + recoveryScope, + }); + const { page } = await mountPage(context); + + await waitForFast(() => expect(page.querySelector(".custodian__wizard-step")).not.toBeNull()); + expect(page.querySelector(".chat-group.user")?.textContent).toContain("Connect WhatsApp"); + expect(page.querySelector(".custodian__structured-response")).toBeNull(); + }); + + it("restores a validation-rejected wizard answer without a completed receipt", async () => { + reconcileCustodianRecoveryForScope( + recoveryOwner, + { + sessionId: "validation-session", + reply: "Enter a port.", + action: "none", + wizardInputPending: true, + step: { id: "port", type: "text", message: "Gateway port" }, + }, + "validation-session", + ); + const request = vi.fn(async (method: string) => { + if (method !== "openclaw.chat.history") { + throw new Error(`unexpected method ${method}`); + } + return { + turns: [ + { + role: "assistant", + text: "Enter a port.", + at: 1, + }, + { + role: "user", + text: "banana", + at: 2, + }, + { + role: "assistant", + text: "Enter port 18789.", + at: 3, + }, + ], + activeWizard: { + sessionId: "validation-session", + step: { id: "port", type: "text", message: "Gateway port" }, + }, + }; + }); + const { context } = createContext(request, ["openclaw.chat", "openclaw.chat.history"], { + gatewayCapabilities: recoveryCapabilities, + recoveryScope, + }); + const { page } = await mountPage(context); + + await waitForFast(() => expect(page.textContent).toContain("Enter port 18789.")); + expect(page.querySelector(".custodian__structured-response")).toBeNull(); + expect(page.querySelector(".chat-group.user")?.textContent).toContain("banana"); + expect(page.querySelector(".custodian__wizard-step")).not.toBeNull(); + }); + it("waits for the authenticated recovery scope before starting a fresh session", async () => { reconcileCustodianRecoveryForScope( recoveryOwner, diff --git a/ui/src/pages/custodian/custodian-page.structured-wizard.test.ts b/ui/src/pages/custodian/custodian-page.structured-wizard.test.ts new file mode 100644 index 000000000000..2ffe6151dd09 --- /dev/null +++ b/ui/src/pages/custodian/custodian-page.structured-wizard.test.ts @@ -0,0 +1,242 @@ +/* @vitest-environment jsdom */ + +import { GATEWAY_SERVER_CAPS } from "@openclaw/gateway-protocol"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createDeferred } from "../../../../test/helpers/promise.ts"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import { waitForFast } from "../../test-helpers/wait-for.ts"; +import { createContext, mountPage } from "./custodian-page.test-harness.ts"; + +describe("custodian structured wizard", () => { + beforeEach(() => { + vi.spyOn(crypto, "randomUUID").mockReturnValue("00000000-0000-4000-8000-000000000001"); + window.history.replaceState({}, "", "/"); + }); + + afterEach(() => { + document.body.replaceChildren(); + sessionStorage.clear(); + vi.restoreAllMocks(); + }); + + it("keeps a rejected typed answer active without showing a submitted receipt", async () => { + const step = { + id: "port", + type: "text" as const, + message: "Gateway port", + }; + const request = vi + .fn() + .mockResolvedValueOnce({ + sessionId: "validation-session", + reply: "Enter a port.", + action: "none", + wizardInputPending: true, + step, + }) + .mockResolvedValueOnce({ + sessionId: "validation-session", + reply: "Enter port 18789.", + action: "none", + wizardActionAccepted: false, + wizardInputPending: true, + step, + }); + const { context } = createContext(request); + const { page } = await mountPage(context); + + const input = await waitForFast(() => { + const element = page.querySelector( + '.custodian__wizard-step input[name="wizard-text"]', + ); + expect(element).not.toBeNull(); + return element!; + }); + input.value = "banana"; + input.dispatchEvent(new Event("input", { bubbles: true })); + await page.updateComplete; + page.querySelector(".custodian__wizard-step .btn.primary")!.click(); + + await waitForFast(() => expect(page.textContent).toContain("Enter port 18789.")); + expect(request.mock.calls[1]?.[1]).toMatchObject({ + wizardAnswer: { stepId: "port", value: "banana" }, + }); + expect(page.querySelector(".custodian__structured-response")).toBeNull(); + expect(page.querySelector(".custodian__wizard-step")).not.toBeNull(); + expect(page.querySelector(".chat-group.user")).toBeNull(); + }); + + it("does not confirm an answer after its Gateway client is replaced", async () => { + const actionReply = createDeferred<{ + sessionId: string; + reply: string; + action: "none"; + wizardActionAccepted: boolean; + }>(); + const step = { + id: "port", + type: "text" as const, + message: "Gateway port", + }; + const request = vi + .fn() + .mockResolvedValueOnce({ + sessionId: "rotation-session", + reply: "Enter a port.", + action: "none", + wizardInputPending: true, + step, + }) + .mockReturnValueOnce(actionReply.promise); + const harness = createContext(request, ["openclaw.chat"], { + gatewayCapabilities: [ + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, + ], + recoveryScope: "principal-a", + }); + const { page } = await mountPage(harness.context); + + const input = await waitForFast(() => { + const element = page.querySelector( + '.custodian__wizard-step input[name="wizard-text"]', + ); + expect(element).not.toBeNull(); + return element!; + }); + input.value = "18789"; + input.dispatchEvent(new Event("input", { bubbles: true })); + await page.updateComplete; + page.querySelector(".custodian__wizard-step .btn.primary")!.click(); + await waitForFast(() => expect(page.textContent).toContain("Submitting answer")); + + harness.setGatewaySnapshot({ + client: { + request: vi.fn().mockResolvedValue({ + sessionId: "replacement-session", + reply: "Fresh session ready.", + action: "none", + }), + recoveryScope: "principal-a", + recoveryScopeReady: true, + } as unknown as GatewayBrowserClient, + }); + actionReply.resolve({ + sessionId: "rotation-session", + reply: "Accepted by the retired client.", + action: "none", + wizardActionAccepted: true, + }); + + await waitForFast(() => expect(page.textContent).toContain("Gateway connection changed")); + expect(page.textContent).not.toContain("Answer submitted"); + expect(page.querySelector(".custodian__structured-response")).toBeNull(); + }); + + it("keeps older-Gateway wizard answers as plain user turns", async () => { + const step = { + id: "port", + type: "text" as const, + message: "Gateway port", + }; + const request = vi + .fn() + .mockResolvedValueOnce({ + sessionId: "legacy-validation-session", + reply: "Enter a port.", + action: "none", + wizardInputPending: true, + step, + }) + .mockResolvedValueOnce({ + sessionId: "legacy-validation-session", + reply: "Enter port 18789.", + action: "none", + wizardInputPending: true, + step, + }); + const { context } = createContext(request, ["openclaw.chat"], { + gatewayCapabilities: [GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL], + }); + const { page } = await mountPage(context); + + const input = await waitForFast(() => { + const element = page.querySelector( + '.custodian__wizard-step input[name="wizard-text"]', + ); + expect(element).not.toBeNull(); + return element!; + }); + input.value = "banana"; + input.dispatchEvent(new Event("input", { bubbles: true })); + await page.updateComplete; + page.querySelector(".custodian__wizard-step .btn.primary")!.click(); + + await waitForFast(() => expect(page.textContent).toContain("Enter port 18789.")); + expect(page.querySelector(".custodian__structured-response")).toBeNull(); + expect(page.querySelector(".chat-group.user")?.textContent).toContain("banana"); + expect(page.querySelector(".custodian__wizard-step")).not.toBeNull(); + }); + + it("keeps server-authored guidance visible beside typed controls", async () => { + const manifest = JSON.stringify( + { + display_information: { + name: "OpenClaw", + description: "OpenClaw connector for OpenClaw", + }, + }, + null, + 2, + ); + const question = "How do you want to provide this Slack bot token?"; + const request = vi.fn().mockResolvedValue({ + sessionId: "slack-wizard-session", + reply: [ + [ + "**Slack socket mode tokens**", + "1) Create the Slack app from the manifest below", + "2) Enable Socket Mode", + ].join("\n"), + manifest, + [ + question, + "1. Enter Slack bot token — Stores the credential directly in OpenClaw config", + "2. Use external secret provider — Stores a reference to an external provider", + "Reply with a number.", + "Say `cancel` to stop this setup.", + ].join("\n"), + ].join("\n\n"), + action: "none", + wizardInputPending: true, + step: { + id: "slack-token-source", + type: "select", + message: question, + options: [ + { + label: "Enter Slack bot token", + value: "direct", + hint: "Stores the credential directly in OpenClaw config", + }, + { + label: "Use external secret provider", + value: "secret-ref", + hint: "Stores a reference to an external provider", + }, + ], + }, + }); + const { context } = createContext(request); + const { page } = await mountPage(context); + + await waitForFast(() => expect(page.querySelector(".custodian__wizard-step")).not.toBeNull()); + expect(page.querySelector(".chat-group.assistant")?.textContent).toContain( + "Slack socket mode tokens", + ); + expect(page.querySelector(".custodian__wizard-guidance")).toBeNull(); + expect(page.querySelectorAll('.custodian__wizard-step input[type="radio"]')).toHaveLength(2); + expect(page.textContent).toContain("Reply with a number"); + expect(page.textContent).toContain("Say cancel"); + }); +}); diff --git a/ui/src/pages/custodian/custodian-page.test-harness.ts b/ui/src/pages/custodian/custodian-page.test-harness.ts index 42c7ce664065..e7058f679ca0 100644 --- a/ui/src/pages/custodian/custodian-page.test-harness.ts +++ b/ui/src/pages/custodian/custodian-page.test-harness.ts @@ -66,6 +66,7 @@ export function createContext( methods, capabilities: options.gatewayCapabilities ?? [ GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, ], }, }, diff --git a/ui/src/pages/custodian/custodian-page.test.ts b/ui/src/pages/custodian/custodian-page.test.ts index dbca3376694c..e35ca1a95c1e 100644 --- a/ui/src/pages/custodian/custodian-page.test.ts +++ b/ui/src/pages/custodian/custodian-page.test.ts @@ -57,7 +57,6 @@ describe("custodian page", () => { .querySelector("img.chat-avatar.assistant") ?.getAttribute("src"), ).toBe("/favicon.svg"); - // Onboarding strips the header identity; the thread avatar is the only mascot. expect(page.querySelector(".custodian__mark openclaw-mascot")).toBeNull(); const card = page.querySelector("openclaw-option-card")!; await card.updateComplete; @@ -71,13 +70,11 @@ describe("custodian page", () => { await page.updateComplete; expect(request.mock.calls[0]?.[0]).toBe("openclaw.chat"); expect(request.mock.calls[0]?.[1]).toMatchObject({ welcomeVariant: "onboarding" }); - // LLM-authored option cards remain chat messages; wizard controls use wizardAnswer below. expect(request.mock.calls[1]?.[1]).toMatchObject({ welcomeVariant: "onboarding", message: "connect whatsapp", }); - const userGroup = page.querySelector(".chat-group.user")!; - expect(userGroup.textContent).toContain("Connect WhatsApp"); + expect(page.querySelector(".chat-group.user")?.textContent).toContain("Connect WhatsApp"); expect(connectOption.disabled).toBe(true); }); @@ -103,6 +100,7 @@ describe("custodian page", () => { sessionId: "rich-wizard-session", reply: "Choose features.", action: "none", + wizardActionAccepted: true, wizardInputPending: true, step: { id: "features", @@ -119,6 +117,7 @@ describe("custodian page", () => { sessionId: "rich-wizard-session", reply: "Enter the secret.", action: "none", + wizardActionAccepted: true, sensitive: true, wizardInputPending: true, step: { @@ -132,6 +131,7 @@ describe("custodian page", () => { sessionId: "rich-wizard-session", reply: "Setup complete.", action: "none", + wizardActionAccepted: true, }); const { context } = createContext(request); const { page } = await mountPage(context); @@ -139,6 +139,7 @@ describe("custodian page", () => { await waitForFast(() => expect(page.querySelectorAll('.custodian__wizard-step input[type="radio"]')).toHaveLength(5), ); + expect(page.querySelector(".chat-group.assistant")?.textContent).toContain("Choose a channel."); expect(page.querySelector("openclaw-option-card")).toBeNull(); expect(page.querySelector(".agent-chat__composer-shell")).toBeNull(); page @@ -169,7 +170,9 @@ describe("custodian page", () => { await waitForFast(() => expect(request).toHaveBeenCalledTimes(3)); const secretInput = await waitForFast(() => { - const input = page.querySelector("#custodian-wizard-input-5"); + const input = page.querySelector( + '.custodian__wizard-step input[name="wizard-text"]', + ); expect(input).not.toBeNull(); return input!; }); @@ -183,7 +186,9 @@ describe("custodian page", () => { expect(revealSecret).not.toBeNull(); revealSecret!.click(); await page.updateComplete; - const revealedInput = page.querySelector("#custodian-wizard-input-5")!; + const revealedInput = page.querySelector( + '.custodian__wizard-step input[name="wizard-text"]', + )!; expect(revealedInput.type).toBe("text"); revealedInput.value = "fake-client-secret"; revealedInput.dispatchEvent(new Event("input", { bubbles: true })); @@ -197,8 +202,11 @@ describe("custodian page", () => { }); expect(request.mock.calls[3]?.[1]).not.toHaveProperty("message"); expect(page.textContent).toContain("Twitch"); + expect(page.textContent).toContain("Choose a channel."); expect(page.textContent).toContain("Chat, Announcements"); expect(page.textContent).toContain("Sensitive reply sent"); + expect(page.querySelectorAll(".custodian__structured-response")).toHaveLength(3); + expect(page.querySelector(".chat-group.user")).toBeNull(); expect(page.textContent).not.toContain("fake-client-secret"); expect(page.querySelector(".agent-chat__composer-shell")).not.toBeNull(); }); @@ -223,6 +231,7 @@ describe("custodian page", () => { sessionId: "cancel-wizard-session", reply: "Channel setup cancelled.", action: "none", + wizardActionAccepted: true, }); const { context } = createContext(request); const { page } = await mountPage(context); @@ -245,6 +254,15 @@ describe("custodian page", () => { expect(request.mock.calls[1]?.[1]).not.toHaveProperty("message"); await waitForFast(() => expect(page.textContent).toContain("Channel setup cancelled.")); expect(page.querySelector(".custodian__wizard-step")).toBeNull(); + expect(page.querySelector(".chat-group.user")).toBeNull(); + expect(page.querySelector(".custodian__structured-response")?.textContent).toContain("Cancel"); + expect(page.querySelector(".custodian__structured-response")?.textContent).toContain( + "Setup cancelled", + ); + const cancellationStatus = page.querySelector(".custodian__structured-response-status"); + expect(cancellationStatus?.textContent).toBe("Setup cancelled"); + expect(cancellationStatus?.classList.contains("sr-only")).toBe(false); + expect(page.querySelector(".custodian__structured-response-icon--cancelled")).not.toBeNull(); expect(page.querySelector(".agent-chat__composer-shell")).not.toBeNull(); }); diff --git a/ui/src/pages/custodian/custodian-session-store.ts b/ui/src/pages/custodian/custodian-session-store.ts index 392d5770118e..76d5c5384de4 100644 --- a/ui/src/pages/custodian/custodian-session-store.ts +++ b/ui/src/pages/custodian/custodian-session-store.ts @@ -16,11 +16,12 @@ import { import { buildAgentMainSessionKey, normalizeAgentId } from "../../lib/sessions/session-key.ts"; import { pathForCustodianAgentHandoff } from "./custodian-navigation.ts"; import { readCustodianRecoveryForClient } from "./custodian-recovery.ts"; +import { createCustodianStructuredInteraction } from "./custodian-structured-interaction.ts"; import { CustodianTranscriptState, type CustodianTranscriptHistoryOutcome, } from "./custodian-transcript-state.ts"; -import { custodianWizardSubmission, initialCustodianWizardValue } from "./custodian-wizard-step.ts"; +import { initialCustodianWizardValue } from "./custodian-wizard-step.ts"; import * as eventNudgeState from "./event-nudge.ts"; import { custodianChatParams, @@ -47,6 +48,10 @@ function hasCustodianUserInput(params: SystemAgentChatParams): boolean { ); } +function hasCustodianWizardAction(params: SystemAgentChatParams): boolean { + return params.wizardAnswer !== undefined || params.wizardCancel !== undefined; +} + type StoreListener = () => void; type ConfiguredInferenceState = "unresolved" | "required" | "ready"; type CustodianSetupIssue = "missing" | "unavailable"; @@ -79,6 +84,23 @@ export class CustodianSessionStore extends CustodianTranscriptState { private agentCleanup: (() => void) | null = null; private eventCleanup: (() => void) | null = null; private readonly listeners = new Set(); + private readonly structuredInteraction = createCustodianStructuredInteraction({ + state: () => ({ + activeClient: this.activeClient, + chatAvailable: this.chatAvailable, + messages: this.messages, + sending: this.sending, + sessionId: this.sessionId, + setupRequired: this.setupRequired, + wizardCancelAvailable: this.wizardCancelAvailable, + wizardActionReceiptsAvailable: this.wizardActionReceiptsAvailable, + wizardInputPending: this.wizardInputPending, + }), + emit: () => this.emit(), + replaceMessages: (messages) => (this.messages = messages), + sendUserTurn: (client, params, display, appendUserMessage) => + this.sendUserTurn(client, params, display, true, appendUserMessage), + }); subscribe(listener: StoreListener): () => void { this.listeners.add(listener); @@ -137,7 +159,9 @@ export class CustodianSessionStore extends CustodianTranscriptState { } hasRealUserTurn(): boolean { - return this.messages.some((message) => message.role === "user"); + return this.messages.some( + (message) => message.role === "user" || message.structuredResponse !== null, + ); } get activeVariant(): CustodianSessionVariant { @@ -171,6 +195,15 @@ export class CustodianSessionStore extends CustodianTranscriptState { ); } + get wizardActionReceiptsAvailable(): boolean { + return ( + isGatewayCapabilityAdvertised( + this.context?.gateway.snapshot ?? {}, + GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_ACTION_RECEIPTS, + ) === true + ); + } + retry(): void { const client = this.activeClient; const params = this.retryParams; @@ -211,6 +244,7 @@ export class CustodianSessionStore extends CustodianTranscriptState { params: SystemAgentChatParams, displayText: string, questionReply: boolean, + appendUserMessage = true, ): Promise { const questionState = [this.answeredQuestions, this.questionReplyUncertain] as const; if (questionReply) { @@ -218,17 +252,20 @@ export class CustodianSessionStore extends CustodianTranscriptState { } this.abandonedTurnOutcomeUnknown = false; this.answeredQuestions = retireCustodianQuestions(this.messages, this.answeredQuestions); - this.messages = [ - ...this.messages, - { - id: this.nextMessageId++, - role: "user", - text: displayText, - at: Date.now(), - question: null, - step: null, - }, - ]; + if (appendUserMessage) { + this.messages = [ + ...this.messages, + { + id: this.nextMessageId++, + role: "user", + text: displayText, + at: Date.now(), + question: null, + step: null, + structuredResponse: null, + }, + ]; + } this.input = ""; this.emit(); const reply = this.requestReply(client, params); @@ -310,45 +347,11 @@ export class CustodianSessionStore extends CustodianTranscriptState { } answerWizardStep(message: CustodianMessage, value: unknown): void { - if (!message.step || !this.wizardInputPending) { - return; - } - const submission = custodianWizardSubmission(message.step, value); - const client = this.activeClient; - if (!submission || !client || !this.chatAvailable || this.sending || this.setupRequired) { - this.emit(); - return; - } - const displayText = message.step.sensitive ? t("custodian.sensitiveReply") : submission.display; - void this.sendUserTurn( - client, - { sessionId: this.sessionId, wizardAnswer: submission.answer }, - displayText, - true, - ); + this.structuredInteraction.answerWizardStep(message, value); } cancelWizardStep(message: CustodianMessage): void { - const step = message.step; - const client = this.activeClient; - if ( - !step || - !this.wizardInputPending || - !client || - !this.chatAvailable || - !this.wizardCancelAvailable || - this.sending || - this.setupRequired - ) { - this.emit(); - return; - } - void this.sendUserTurn( - client, - { sessionId: this.sessionId, wizardCancel: { stepId: step.id } }, - t("custodian.cancel"), - true, - ); + this.structuredInteraction.cancelWizardStep(message); } exitSetup(): void { @@ -704,7 +707,10 @@ export class CustodianSessionStore extends CustodianTranscriptState { } else if (result.action === "exit") { this.exitSetup(); } - return "sent"; + if (!hasCustodianWizardAction(params) || !this.wizardActionReceiptsAvailable) { + return "sent"; + } + return result.wizardActionAccepted === true ? "accepted" : "rejected"; } catch (error) { if (epoch === this.requestEpoch && client === this.activeClient) { this.error = custodianErrorMessage(error); diff --git a/ui/src/pages/custodian/custodian-structured-interaction.ts b/ui/src/pages/custodian/custodian-structured-interaction.ts new file mode 100644 index 000000000000..992afd7ab076 --- /dev/null +++ b/ui/src/pages/custodian/custodian-structured-interaction.ts @@ -0,0 +1,144 @@ +import type { SystemAgentChatParams } from "@openclaw/gateway-protocol"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import { t } from "../../i18n/index.ts"; +import { custodianWizardSubmission } from "./custodian-wizard-step.ts"; +import type * as eventNudgeState from "./event-nudge.ts"; +import type { CustodianMessage, CustodianStructuredResponse } from "./transcript.ts"; + +type StructuredInteractionState = { + activeClient: GatewayBrowserClient | null; + chatAvailable: boolean; + messages: readonly CustodianMessage[]; + sending: boolean; + sessionId: string; + setupRequired: boolean; + wizardCancelAvailable: boolean; + wizardActionReceiptsAvailable: boolean; + wizardInputPending: boolean; +}; + +type StructuredInteractionHost = { + state: () => StructuredInteractionState; + emit: () => void; + replaceMessages: (messages: CustodianMessage[]) => void; + sendUserTurn: ( + client: GatewayBrowserClient, + params: SystemAgentChatParams, + display: string, + appendUserMessage: boolean, + ) => Promise; +}; + +function withResponse( + messages: readonly CustodianMessage[], + messageId: number, + response: CustodianStructuredResponse | null, +): CustodianMessage[] | null { + const target = messages.find((message) => message.id === messageId); + return target + ? messages.map((message) => + message === target ? { ...message, structuredResponse: response } : message, + ) + : null; +} + +export function createCustodianStructuredInteraction(host: StructuredInteractionHost) { + const submit = async (params: { + client: GatewayBrowserClient; + message: CustodianMessage; + request: SystemAgentChatParams; + display: string; + kind: CustodianStructuredResponse["kind"]; + prompt?: string; + }): Promise => { + const state = host.state(); + if ( + !state.chatAvailable || + state.sending || + state.setupRequired || + !state.messages.includes(params.message) + ) { + host.emit(); + return "rejected"; + } + if (state.wizardActionReceiptsAvailable) { + host.replaceMessages( + withResponse(state.messages, params.message.id, { + display: params.display, + kind: params.kind, + state: "submitting", + ...(params.prompt ? { prompt: params.prompt } : {}), + }) ?? [...state.messages], + ); + host.emit(); + } + const outcome = await host.sendUserTurn( + params.client, + params.request, + params.display, + !state.wizardActionReceiptsAvailable, + ); + if (!state.wizardActionReceiptsAvailable) { + return outcome; + } + const current = host.state(); + const response: CustodianStructuredResponse | null = + outcome !== "accepted" + ? null + : { + display: params.display, + kind: params.kind, + state: "submitted", + ...(params.prompt ? { prompt: params.prompt } : {}), + }; + const messages = withResponse(current.messages, params.message.id, response); + if (messages) { + host.replaceMessages(messages); + host.emit(); + } + return outcome; + }; + + return { + answerWizardStep(message: CustodianMessage, value: unknown): void { + const state = host.state(); + const submission = message.step ? custodianWizardSubmission(message.step, value) : null; + if (!submission || !state.activeClient || !state.wizardInputPending) { + host.emit(); + return; + } + void submit({ + client: state.activeClient, + message, + request: { sessionId: state.sessionId, wizardAnswer: submission.answer }, + display: message.step?.sensitive ? t("custodian.sensitiveReply") : submission.display, + kind: "answer", + prompt: message.step?.title ?? message.step?.message, + }); + }, + + cancelWizardStep(message: CustodianMessage): void { + const state = host.state(); + const step = message.step; + const activeWizardMessage = state.messages.findLast((candidate) => candidate.step !== null); + if ( + !step || + message !== activeWizardMessage || + !state.wizardInputPending || + !state.wizardCancelAvailable || + !state.activeClient + ) { + host.emit(); + return; + } + void submit({ + client: state.activeClient, + message, + request: { sessionId: state.sessionId, wizardCancel: { stepId: step.id } }, + display: t("custodian.cancel"), + kind: "cancel", + prompt: step.title ?? step.message, + }); + }, + }; +} diff --git a/ui/src/pages/custodian/custodian-transcript-state.ts b/ui/src/pages/custodian/custodian-transcript-state.ts index 9f49716d5630..d313cafc7d50 100644 --- a/ui/src/pages/custodian/custodian-transcript-state.ts +++ b/ui/src/pages/custodian/custodian-transcript-state.ts @@ -120,6 +120,7 @@ export abstract class CustodianTranscriptState { at: Date.now(), question, step, + structuredResponse: null, }, ]; } diff --git a/ui/src/pages/custodian/event-nudge.ts b/ui/src/pages/custodian/event-nudge.ts index 6ac39ada2027..0f0cafaeca76 100644 --- a/ui/src/pages/custodian/event-nudge.ts +++ b/ui/src/pages/custodian/event-nudge.ts @@ -11,7 +11,7 @@ export type CustodianEventNudge = { }; export type CustodianSendDelivery = "unsent" | "sent" | "received"; -export type CustodianSendOutcome = "sent" | "rejected" | "unknown"; +export type CustodianSendOutcome = "accepted" | "sent" | "rejected" | "unknown"; export function classifyCustodianSendFailure( error: unknown, @@ -27,7 +27,7 @@ export function classifyCustodianSendFailure( } export function questionUncertainty(previous: boolean, outcome: CustodianSendOutcome): boolean { - if (outcome === "sent") { + if (outcome === "accepted" || outcome === "sent") { return false; } return outcome === "unknown" ? true : previous; diff --git a/ui/src/pages/custodian/transcript.ts b/ui/src/pages/custodian/transcript.ts index 1c1ef9a09074..4faf5ecb6428 100644 --- a/ui/src/pages/custodian/transcript.ts +++ b/ui/src/pages/custodian/transcript.ts @@ -5,6 +5,7 @@ import type { import { html, nothing } from "lit"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; import type { WizardStep } from "../../api/types.ts"; +import { icons } from "../../components/icons.ts"; import { renderWizardStepControls } from "../../components/wizard-step-controls.ts"; import { t } from "../../i18n/index.ts"; import type { MessageGroup } from "../../lib/chat/chat-types.ts"; @@ -22,6 +23,14 @@ export type CustodianMessage = { at: number; question: CustodianStructuredQuestion | null; step: WizardStep | null; + structuredResponse: CustodianStructuredResponse | null; +}; + +export type CustodianStructuredResponse = { + display: string; + kind: "answer" | "cancel"; + prompt?: string; + state: "submitting" | "submitted"; }; export function hasUnresolvedCustodianQuestion( @@ -99,7 +108,7 @@ async function readCustodianTranscript( /** * Sensitive turns are masked server-side before persistence: the engine pushes - * only "" into history (never raw input), so durable turns + * only "" into history (never raw input), so returned turns * cannot carry credentials. This mapping only localizes that marker to the * same display text live sensitive replies use. */ @@ -108,19 +117,65 @@ const SERVER_SENSITIVE_MASK = ""; function createCustodianTranscriptMessages( turns: readonly SystemAgentChatHistoryTurn[], firstMessageId: number, + activeWizard?: SystemAgentChatHistoryResult["activeWizard"], ): { messages: CustodianMessage[]; nextMessageId: number } { let nextMessageId = firstMessageId; - const messages = turns.map((turn) => ({ - id: nextMessageId++, - role: turn.role, - text: + const messages: CustodianMessage[] = []; + for (const turn of turns) { + const display = turn.role === "user" && turn.text === SERVER_SENSITIVE_MASK ? t("custodian.sensitiveReply") - : turn.text, - at: turn.at, - question: null, - step: null, - })); + : turn.text; + if (turn.role === "user" && turn.wizardAction) { + const previous = messages.at(-1); + const supportingText = previous?.role === "assistant" ? previous.text : ""; + if (supportingText) { + messages.pop(); + } + messages.push({ + id: nextMessageId++, + role: "assistant", + text: supportingText, + at: turn.at, + question: null, + step: null, + structuredResponse: { + display, + kind: turn.wizardAction.kind, + state: "submitted", + ...(turn.wizardAction.prompt ? { prompt: turn.wizardAction.prompt } : {}), + }, + }); + continue; + } + messages.push({ + id: nextMessageId++, + role: turn.role, + text: display, + at: turn.at, + question: null, + step: null, + structuredResponse: null, + }); + } + if (activeWizard) { + const activePrompt = messages.findLast( + (message) => message.role === "assistant" && message.structuredResponse === null, + ); + if (activePrompt) { + activePrompt.step = activeWizard.step; + } else { + messages.push({ + id: nextMessageId++, + role: "assistant", + text: "", + at: Date.now(), + question: null, + step: activeWizard.step, + structuredResponse: null, + }); + } + } return { messages, nextMessageId }; } @@ -137,21 +192,15 @@ export async function loadCustodianTranscriptSnapshot( sessionId?: string, ): Promise { const history = await readCustodianTranscript(client, sessionId); - const transcript = createCustodianTranscriptMessages(history.turns, firstMessageId); + const transcript = createCustodianTranscriptMessages( + history.turns, + firstMessageId, + sessionId && history.activeWizard?.sessionId === sessionId ? history.activeWizard : undefined, + ); const earlierBoundaryAfterId = transcript.messages.at(-1)?.id ?? null; const activeWizard = history.activeWizard; const recoveredStep = sessionId && activeWizard?.sessionId === sessionId ? activeWizard.step : null; - if (recoveredStep) { - transcript.messages.push({ - id: transcript.nextMessageId++, - role: "assistant", - text: "", - at: Date.now(), - question: null, - step: recoveredStep, - }); - } return { ...transcript, earlierBoundaryAfterId, @@ -170,6 +219,49 @@ function renderCustodianEarlierDivider(message: CustodianMessage, boundaryAfterI : nothing; } +function structuredPrompt(message: CustodianMessage): string { + return ( + message.structuredResponse?.prompt ?? + message.step?.title ?? + message.step?.message ?? + message.question?.question ?? + t("custodian.structured.response") + ); +} + +function renderStructuredResponse(message: CustodianMessage) { + const response = message.structuredResponse; + if (!response) { + return nothing; + } + const cancelled = response.kind === "cancel"; + const status = cancelled + ? response.state === "submitting" + ? t("custodian.structured.cancelling") + : t("custodian.structured.cancelled") + : response.state === "submitting" + ? t("custodian.structured.submitting") + : t("custodian.structured.submitted"); + return html`
+ + + ${structuredPrompt(message)} + ${response.display} + ${status} + +
`; +} + export function renderCustodianTranscriptEntry(params: { message: CustodianMessage; boundaryAfterId: number | null; @@ -190,8 +282,11 @@ export function renderCustodianTranscriptEntry(params: { }) { const question = params.message.question; const step = params.message.step; + const hasStructuredResponse = params.message.structuredResponse !== null; + const hasActiveWizardStep = params.showWizardStep && step !== null; + const showTranscriptMessage = params.message.text; return html` - ${params.message.text + ${showTranscriptMessage ? renderMessageGroup(toCustodianMessageGroup(params.message), { showReasoning: false, showToolCalls: false, @@ -200,15 +295,17 @@ export function renderCustodianTranscriptEntry(params: { }) : nothing} ${renderCustodianEarlierDivider(params.message, params.boundaryAfterId)} - ${params.showQuestion && question - ? renderCustodianQuestionCard({ - question, - disabled: params.questionDisabled, - onSelect: params.onSelect, - onSkip: params.onSkip, - }) - : nothing} - ${params.showWizardStep && step + ${hasStructuredResponse + ? renderStructuredResponse(params.message) + : params.showQuestion && question + ? renderCustodianQuestionCard({ + question, + disabled: params.questionDisabled, + onSelect: params.onSelect, + onSkip: params.onSkip, + }) + : nothing} + ${hasActiveWizardStep && !hasStructuredResponse && step ? html`