From 4b05d83035cc7b466f155645603ce87d5b9fdf1a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Jul 2026 04:39:53 -0400 Subject: [PATCH] fix: generated images do not appear in native chat (#115042) * fix(chat): render managed images in native clients Co-authored-by: Francesco Giannicola * chore: keep release changelog owned * refactor(macos): split managed image loading * test(apple): prove managed image header policy * test(native): stabilize managed image proof * test(apple): satisfy strict concurrency checks --------- Co-authored-by: Francesco Giannicola --- apps/.i18n/native-source.json | 634 ++++++++++-------- .../java/ai/openclaw/app/MainViewModel.kt | 2 + .../main/java/ai/openclaw/app/NodeRuntime.kt | 2 + .../ai/openclaw/app/chat/ChatController.kt | 44 ++ .../java/ai/openclaw/app/chat/ChatModels.kt | 9 +- .../openclaw/app/chat/ChatTranscriptCache.kt | 80 ++- .../ai/openclaw/app/gateway/GatewaySession.kt | 79 +++ .../ai/openclaw/app/ui/chat/ChatImageCodec.kt | 15 +- .../openclaw/app/ui/chat/ChatMessageViews.kt | 189 ++++-- .../ai/openclaw/app/ui/chat/ChatScreen.kt | 39 +- .../chat/ChatControllerMessageIdentityTest.kt | 24 + .../app/chat/ChatMessageContentParsingTest.kt | 30 +- .../app/chat/RoomChatTranscriptCacheTest.kt | 47 +- .../GatewaySessionCustomHeadersTest.kt | 111 +++ .../app/ui/chat/ChatMessageViewsTest.kt | 56 ++ .../Chat/IOSGatewayChatTransport.swift | 30 +- .../Sources/Chat/IOSImageArtifactLoader.swift | 111 +++ apps/ios/Sources/Model/NodeAppModel.swift | 11 +- .../Tests/IOSImageArtifactLoaderTests.swift | 91 +++ apps/ios/Tests/SwiftUIRenderSmokeTests.swift | 62 +- .../GatewayConnection+ManagedImages.swift | 77 +++ .../Sources/OpenClaw/GatewayConnection.swift | 8 +- .../Sources/OpenClaw/WebChatSwiftUI.swift | 15 + .../OpenClawChatUI/ChatGatewayRequest.swift | 16 + .../ChatMediaImageAttachment.swift | 149 ++++ .../OpenClawChatUI/ChatMessageViews.swift | 75 ++- .../Sources/OpenClawChatUI/ChatModels.swift | 59 ++ .../OpenClawChatUI/ChatTranscriptCache.swift | 7 + .../OpenClawChatUI/ChatTransport.swift | 18 + .../Sources/OpenClawChatUI/ChatView.swift | 7 + .../ChatViewModel+HistoryReconciliation.swift | 26 +- .../OpenClawKit/GatewayNodeSession.swift | 7 + .../OpenClawKit/GatewayTLSPinning.swift | 40 ++ .../OpenClawProtocol/GatewayModels.swift | 6 +- .../ChatMessageMediaAttachmentTests.swift | 96 +++ docs/docs_map.md | 1 + docs/gateway/clients.md | 24 + docs/platforms/android.md | 1 + docs/platforms/ios.md | 1 + docs/platforms/macos.md | 5 + docs/web/control-ui.md | 9 +- .../gateway-protocol/src/schema/artifacts.ts | 1 + src/gateway/managed-image-attachments.test.ts | 50 ++ src/gateway/managed-image-attachments.ts | 264 +++++++- src/gateway/server-methods/artifacts.test.ts | 57 ++ src/gateway/server-methods/artifacts.ts | 68 +- ui/src/api/types.ts | 1 + ui/src/e2e/chat-flow.media-files.e2e.test.ts | 77 +++ ui/src/lib/observer-digest.test.ts | 27 +- ui/src/lib/observer-digest.ts | 18 + ui/src/pages/chat/chat-pane-deps.ts | 2 + ui/src/pages/chat/chat-pane-render.ts | 21 +- ui/src/pages/chat/chat-pane-state.test.ts | 44 +- ui/src/pages/chat/chat-pane-state.ts | 22 +- ui/src/pages/chat/chat-view.ts | 3 + .../chat/components/chat-message-bubble.ts | 3 + .../chat/components/chat-message-group.ts | 3 + .../chat/components/chat-message-images.ts | 47 +- .../chat/components/chat-message-media.ts | 8 + .../chat/components/chat-message.test.ts | 46 ++ ui/src/pages/chat/components/chat-thread.ts | 3 + 61 files changed, 2638 insertions(+), 440 deletions(-) create mode 100644 apps/android/app/src/test/java/ai/openclaw/app/ui/chat/ChatMessageViewsTest.kt create mode 100644 apps/ios/Sources/Chat/IOSImageArtifactLoader.swift create mode 100644 apps/ios/Tests/IOSImageArtifactLoaderTests.swift create mode 100644 apps/macos/Sources/OpenClaw/GatewayConnection+ManagedImages.swift create mode 100644 apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift create mode 100644 apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMessageMediaAttachmentTests.swift diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 9b2c93fd74ec..d2e56c2f6b39 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -1067,7 +1067,7 @@ }, { "kind": "ui-call", - "line": 5578, + "line": 5580, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load provider catalog.", "surface": "android", @@ -1075,7 +1075,7 @@ }, { "kind": "ui-call", - "line": 5609, + "line": 5611, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Update your Gateway to view provider model config.", "surface": "android", @@ -1083,7 +1083,7 @@ }, { "kind": "ui-call", - "line": 5611, + "line": 5613, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load provider model config.", "surface": "android", @@ -1091,7 +1091,7 @@ }, { "kind": "ui-call", - "line": 5627, + "line": 5629, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Provider models loaded, but readiness is unavailable.", "surface": "android", @@ -1099,7 +1099,7 @@ }, { "kind": "ui-call", - "line": 5714, + "line": 5716, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load automations.", "surface": "android", @@ -1107,7 +1107,7 @@ }, { "kind": "ui-call", - "line": 5800, + "line": 5802, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to inspect automations.", "surface": "android", @@ -1115,7 +1115,7 @@ }, { "kind": "ui-call", - "line": 5810, + "line": 5812, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Gateway returned an invalid automation.", "surface": "android", @@ -1123,7 +1123,7 @@ }, { "kind": "ui-call", - "line": 5814, + "line": 5816, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load automation.", "surface": "android", @@ -1131,7 +1131,7 @@ }, { "kind": "ui-call", - "line": 5826, + "line": 5828, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to inspect automation run history.", "surface": "android", @@ -1139,7 +1139,7 @@ }, { "kind": "ui-call", - "line": 5857, + "line": 5859, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load automation run history.", "surface": "android", @@ -1147,7 +1147,7 @@ }, { "kind": "ui-call", - "line": 5874, + "line": 5876, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Cron changes require operator.admin access.", "surface": "android", @@ -1155,7 +1155,7 @@ }, { "kind": "ui-call", - "line": 5883, + "line": 5885, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to manage automations.", "surface": "android", @@ -1163,7 +1163,7 @@ }, { "kind": "ui-call", - "line": 5895, + "line": 5897, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Another cron action is still finishing.", "surface": "android", @@ -1171,7 +1171,7 @@ }, { "kind": "ui-call", - "line": 5942, + "line": 5944, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Cron action failed.", "surface": "android", @@ -1179,7 +1179,7 @@ }, { "kind": "ui-call", - "line": 6054, + "line": 6056, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load usage.", "surface": "android", @@ -1187,7 +1187,7 @@ }, { "kind": "ui-call", - "line": 6069, + "line": 6071, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load skills.", "surface": "android", @@ -1195,7 +1195,7 @@ }, { "kind": "ui-call", - "line": 6089, + "line": 6091, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to update skills.", "surface": "android", @@ -1203,7 +1203,7 @@ }, { "kind": "ui-call", - "line": 6093, + "line": 6095, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "This gateway connection needs operator.admin to update skills.", "surface": "android", @@ -1211,7 +1211,7 @@ }, { "kind": "conditional-branch", - "line": 6108, + "line": 6110, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not disable skill.", "surface": "android", @@ -1219,7 +1219,7 @@ }, { "kind": "conditional-branch", - "line": 6108, + "line": 6110, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not enable skill.", "surface": "android", @@ -1227,7 +1227,7 @@ }, { "kind": "ui-call", - "line": 6126, + "line": 6128, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to search ClawHub skills.", "surface": "android", @@ -1235,7 +1235,7 @@ }, { "kind": "ui-call", - "line": 6172, + "line": 6174, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not search ClawHub skills.", "surface": "android", @@ -1243,7 +1243,7 @@ }, { "kind": "ui-call", - "line": 6185, + "line": 6187, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to inspect ClawHub skills.", "surface": "android", @@ -1251,7 +1251,7 @@ }, { "kind": "conditional-branch", - "line": 6218, + "line": 6220, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "ClawHub did not return an installable version for ${skill.slug}.", "surface": "android", @@ -1259,7 +1259,7 @@ }, { "kind": "ui-call", - "line": 6234, + "line": 6236, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load ClawHub details for ${skill.slug}.", "surface": "android", @@ -1267,7 +1267,7 @@ }, { "kind": "ui-call", - "line": 6250, + "line": 6252, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to install ClawHub skills.", "surface": "android", @@ -1275,7 +1275,7 @@ }, { "kind": "ui-call", - "line": 6266, + "line": 6268, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "This gateway connection needs operator.admin to install ClawHub skills.", "surface": "android", @@ -1283,7 +1283,7 @@ }, { "kind": "conditional-branch", - "line": 6356, + "line": 6358, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Installed $slug.", "surface": "android", @@ -1291,7 +1291,7 @@ }, { "kind": "ui-call", - "line": 6363, + "line": 6365, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not install ${slug} from ClawHub.", "surface": "android", @@ -1299,7 +1299,7 @@ }, { "kind": "ui-call", - "line": 6403, + "line": 6405, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to load Skill Workshop proposals.", "surface": "android", @@ -1307,7 +1307,7 @@ }, { "kind": "ui-call", - "line": 6443, + "line": 6445, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load Skill Workshop proposals.", "surface": "android", @@ -1315,7 +1315,7 @@ }, { "kind": "ui-call", - "line": 6463, + "line": 6465, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to inspect Skill Workshop proposals.", "surface": "android", @@ -1323,7 +1323,7 @@ }, { "kind": "ui-call", - "line": 6515, + "line": 6517, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not inspect Skill Workshop proposal.", "surface": "android", @@ -1331,7 +1331,7 @@ }, { "kind": "ui-call", - "line": 6535, + "line": 6537, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Skill Workshop proposal actions require operator.admin scope.", "surface": "android", @@ -1339,7 +1339,7 @@ }, { "kind": "ui-call", - "line": 6540, + "line": 6542, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect the gateway to update Skill Workshop proposals.", "surface": "android", @@ -1347,7 +1347,7 @@ }, { "kind": "ui-call", - "line": 6730, + "line": 6732, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not verify the device pairing change. Refresh and try again.", "surface": "android", @@ -1355,7 +1355,7 @@ }, { "kind": "ui-named-argument", - "line": 6812, + "line": 6814, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connected", "surface": "android", @@ -1363,7 +1363,7 @@ }, { "kind": "ui-call", - "line": 6842, + "line": 6844, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load nodes and devices.", "surface": "android", @@ -1371,7 +1371,7 @@ }, { "kind": "ui-call", - "line": 7492, + "line": 7494, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load channels.", "surface": "android", @@ -1379,7 +1379,7 @@ }, { "kind": "ui-call", - "line": 7510, + "line": 7512, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load dreaming.", "surface": "android", @@ -1387,7 +1387,7 @@ }, { "kind": "ui-call", - "line": 7525, + "line": 7527, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Could not load gateway logs.", "surface": "android", @@ -1395,7 +1395,7 @@ }, { "kind": "ui-call", - "line": 8008, + "line": 8010, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "One time", "surface": "android", @@ -1403,7 +1403,7 @@ }, { "kind": "ui-call", - "line": 8017, + "line": 8019, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Cron", "surface": "android", @@ -1411,7 +1411,7 @@ }, { "kind": "ui-call", - "line": 8018, + "line": 8020, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Scheduled", "surface": "android", @@ -1419,7 +1419,7 @@ }, { "kind": "ui-call", - "line": 8026, + "line": 8028, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Every ${days}d", "surface": "android", @@ -1427,7 +1427,7 @@ }, { "kind": "ui-call", - "line": 8027, + "line": 8029, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Every ${hours}h", "surface": "android", @@ -1435,7 +1435,7 @@ }, { "kind": "ui-call", - "line": 8028, + "line": 8030, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Every ${minutes}m", "surface": "android", @@ -1443,7 +1443,7 @@ }, { "kind": "ui-call", - "line": 8029, + "line": 8031, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Repeating", "surface": "android", @@ -1451,7 +1451,7 @@ }, { "kind": "ui-call", - "line": 8045, + "line": 8047, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "No prompt", "surface": "android", @@ -1459,7 +1459,7 @@ }, { "kind": "ui-call", - "line": 8062, + "line": 8064, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Gateway", "surface": "android", @@ -1467,7 +1467,7 @@ }, { "kind": "ui-call", - "line": 8070, + "line": 8072, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connected to $gatewayLabel", "surface": "android", @@ -1475,7 +1475,7 @@ }, { "kind": "ui-call", - "line": 8071, + "line": 8073, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Your agents are ready", "surface": "android", @@ -1483,7 +1483,7 @@ }, { "kind": "ui-call", - "line": 8073, + "line": 8075, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "This phone stays dormant until the gateway needs it, then wakes, syncs, and goes back to sleep.", "surface": "android", @@ -1491,7 +1491,7 @@ }, { "kind": "ui-call", - "line": 8077, + "line": 8079, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Selected on this phone", "surface": "android", @@ -1499,7 +1499,7 @@ }, { "kind": "ui-call", - "line": 8080, + "line": 8082, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "The overview refreshes on reconnect and when this screen opens.", "surface": "android", @@ -1507,7 +1507,7 @@ }, { "kind": "ui-call", - "line": 8085, + "line": 8087, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Reconnecting", "surface": "android", @@ -1515,7 +1515,7 @@ }, { "kind": "ui-call", - "line": 8086, + "line": 8088, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "OpenClaw is syncing back up", "surface": "android", @@ -1523,7 +1523,7 @@ }, { "kind": "ui-call", - "line": 8088, + "line": 8090, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "The gateway session is coming back online. Agent shortcuts should settle automatically in a moment.", "surface": "android", @@ -1531,7 +1531,7 @@ }, { "kind": "ui-call", - "line": 8092, + "line": 8094, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Gateway session in progress", "surface": "android", @@ -1539,7 +1539,7 @@ }, { "kind": "ui-call", - "line": 8095, + "line": 8097, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "If the gateway is reachable, reconnect should complete without intervention.", "surface": "android", @@ -1547,7 +1547,7 @@ }, { "kind": "ui-call", - "line": 8100, + "line": 8102, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Welcome to OpenClaw", "surface": "android", @@ -1555,7 +1555,7 @@ }, { "kind": "ui-call", - "line": 8101, + "line": 8103, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Your phone stays quiet until it is needed", "surface": "android", @@ -1563,7 +1563,7 @@ }, { "kind": "ui-call", - "line": 8103, + "line": 8105, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Pair this device to your gateway to wake it only for real work, keep a live agent overview handy, and avoid battery-draining background loops.", "surface": "android", @@ -1571,7 +1571,7 @@ }, { "kind": "ui-call", - "line": 8107, + "line": 8109, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Connect to load your agents", "surface": "android", @@ -1579,7 +1579,7 @@ }, { "kind": "ui-call", - "line": 8110, + "line": 8112, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "When connected, the gateway can wake the phone with a silent push instead of holding an always-on session.", "surface": "android", @@ -1587,7 +1587,7 @@ }, { "kind": "ui-call", - "line": 8142, + "line": 8144, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Main", "surface": "android", @@ -1595,7 +1595,7 @@ }, { "kind": "ui-call", - "line": 8157, + "line": 8159, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Active on this phone", "surface": "android", @@ -1603,7 +1603,7 @@ }, { "kind": "ui-call", - "line": 8158, + "line": 8160, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Default agent", "surface": "android", @@ -1611,7 +1611,7 @@ }, { "kind": "ui-call", - "line": 8159, + "line": 8161, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Ready", "surface": "android", @@ -1619,7 +1619,7 @@ }, { "kind": "ui-call", - "line": 8814, + "line": 8816, "path": "apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt", "source": "Dream", "surface": "android", @@ -1795,7 +1795,7 @@ }, { "kind": "ui-call", - "line": 1802, + "line": 1825, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Wait for the current response to finish before starting a new chat.", "surface": "android", @@ -1803,7 +1803,7 @@ }, { "kind": "ui-call", - "line": 1935, + "line": 1958, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Could not update model.", "surface": "android", @@ -1811,7 +1811,7 @@ }, { "kind": "ui-call", - "line": 2000, + "line": 2023, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Could not update thinking level.", "surface": "android", @@ -1819,7 +1819,7 @@ }, { "kind": "ui-call", - "line": 2513, + "line": 2536, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Chat failed before the run started; try again.", "surface": "android", @@ -1827,7 +1827,7 @@ }, { "kind": "ui-call", - "line": 4209, + "line": 4232, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Could not stage an attachment for sending.", "surface": "android", @@ -1835,7 +1835,7 @@ }, { "kind": "ui-call", - "line": 4242, + "line": 4265, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Offline queue is full ($OUTBOX_MAX_QUEUED messages); delete queued items first.", "surface": "android", @@ -1843,7 +1843,7 @@ }, { "kind": "ui-call", - "line": 4248, + "line": 4271, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Attachments are too large to queue for one message; remove some and try again.", "surface": "android", @@ -1851,7 +1851,7 @@ }, { "kind": "ui-call", - "line": 4254, + "line": 4277, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Offline attachment storage is full; delete queued items first.", "surface": "android", @@ -1859,7 +1859,7 @@ }, { "kind": "ui-call", - "line": 4259, + "line": 4282, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Gateway health not OK; cannot send", "surface": "android", @@ -1867,7 +1867,7 @@ }, { "kind": "ui-call", - "line": 4266, + "line": 4289, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Could not queue message for later delivery.", "surface": "android", @@ -1875,7 +1875,7 @@ }, { "kind": "ui-call", - "line": 5167, + "line": 5190, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Chat failed", "surface": "android", @@ -1883,7 +1883,7 @@ }, { "kind": "ui-call", - "line": 5398, + "line": 5421, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Event stream interrupted; try refreshing.", "surface": "android", @@ -1891,7 +1891,7 @@ }, { "kind": "ui-call", - "line": 5539, + "line": 5562, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Timed out waiting for a reply; try again or refresh.", "surface": "android", @@ -1899,7 +1899,7 @@ }, { "kind": "ui-call", - "line": 5747, + "line": 5770, "path": "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", "source": "Timed out confirming the sent message; refresh to check delivery.", "surface": "android", @@ -1913,9 +1913,17 @@ "surface": "android", "id": "native.android.93ecb3b3e1f02b63" }, + { + "kind": "ui-call", + "line": 883, + "path": "apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt", + "source": "Accept", + "surface": "android", + "id": "native.android.60062377f4798a11" + }, { "kind": "conditional-branch", - "line": 1563, + "line": 1642, "path": "apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt", "source": "Connecting…", "surface": "android", @@ -1923,7 +1931,7 @@ }, { "kind": "conditional-branch", - "line": 1563, + "line": 1642, "path": "apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt", "source": "Reconnecting…", "surface": "android", @@ -12331,7 +12339,7 @@ }, { "kind": "ui-call", - "line": 180, + "line": 190, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Preparing audio…", "surface": "android", @@ -12339,7 +12347,7 @@ }, { "kind": "ui-call", - "line": 180, + "line": 190, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Speaking…", "surface": "android", @@ -12347,7 +12355,15 @@ }, { "kind": "ui-call", - "line": 287, + "line": 252, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", + "source": "Image", + "surface": "android", + "id": "native.android.010f1a1fdd158ba0" + }, + { + "kind": "ui-call", + "line": 307, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Preview · $domain", "surface": "android", @@ -12355,7 +12371,7 @@ }, { "kind": "ui-call", - "line": 296, + "line": 316, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Expand link preview", "surface": "android", @@ -12363,7 +12379,7 @@ }, { "kind": "ui-call", - "line": 339, + "line": 359, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Loading preview…", "surface": "android", @@ -12371,7 +12387,7 @@ }, { "kind": "ui-call", - "line": 340, + "line": 360, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "No preview available", "surface": "android", @@ -12379,7 +12395,7 @@ }, { "kind": "ui-call", - "line": 387, + "line": 407, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Working", "surface": "android", @@ -12387,7 +12403,7 @@ }, { "kind": "ui-call", - "line": 397, + "line": 417, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "· $phrase", "surface": "android", @@ -12395,7 +12411,7 @@ }, { "kind": "ui-named-argument", - "line": 414, + "line": 434, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Tools", "surface": "android", @@ -12403,7 +12419,7 @@ }, { "kind": "ui-call", - "line": 417, + "line": 437, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Running tools...", "surface": "android", @@ -12411,7 +12427,7 @@ }, { "kind": "ui-call", - "line": 421, + "line": 441, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "${display.emoji} ${display.label}", "surface": "android", @@ -12419,7 +12435,7 @@ }, { "kind": "ui-call", - "line": 438, + "line": 458, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "... +${toolCalls.size - 6} more", "surface": "android", @@ -12427,7 +12443,7 @@ }, { "kind": "ui-call", - "line": 459, + "line": 479, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Queued — sends when reconnected", "surface": "android", @@ -12435,7 +12451,7 @@ }, { "kind": "ui-call", - "line": 460, + "line": 480, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Sending…", "surface": "android", @@ -12443,7 +12459,7 @@ }, { "kind": "ui-call", - "line": 461, + "line": 481, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Sent — confirming delivery…", "surface": "android", @@ -12451,7 +12467,7 @@ }, { "kind": "ui-call", - "line": 469, + "line": 489, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Session branch changed; review and retry this message.", "surface": "android", @@ -12459,7 +12475,7 @@ }, { "kind": "ui-call", - "line": 473, + "line": 493, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Failed — $it", "surface": "android", @@ -12467,7 +12483,7 @@ }, { "kind": "ui-call", - "line": 474, + "line": 494, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Failed", "surface": "android", @@ -12475,7 +12491,7 @@ }, { "kind": "ui-call", - "line": 478, + "line": 498, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "user", "surface": "android", @@ -12483,7 +12499,7 @@ }, { "kind": "ui-call", - "line": 486, + "line": 506, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "📎 ${attachment.fileName}", "surface": "android", @@ -12491,7 +12507,7 @@ }, { "kind": "ui-call", - "line": 502, + "line": 522, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Retry", "surface": "android", @@ -12499,7 +12515,7 @@ }, { "kind": "ui-call", - "line": 507, + "line": 527, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Delete", "surface": "android", @@ -12507,7 +12523,7 @@ }, { "kind": "ui-call", - "line": 538, + "line": 558, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "assistant", "surface": "android", @@ -12515,7 +12531,7 @@ }, { "kind": "ui-named-argument", - "line": 539, + "line": 559, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "OpenClaw · Live", "surface": "android", @@ -12523,7 +12539,7 @@ }, { "kind": "ui-call", - "line": 575, + "line": 595, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "You", "surface": "android", @@ -12531,7 +12547,7 @@ }, { "kind": "ui-call", - "line": 576, + "line": 596, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "System", "surface": "android", @@ -12539,7 +12555,7 @@ }, { "kind": "ui-call", - "line": 577, + "line": 597, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "OpenClaw", "surface": "android", @@ -12547,7 +12563,7 @@ }, { "kind": "ui-call", - "line": 600, + "line": 609, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Attachment", "surface": "android", @@ -12555,7 +12571,31 @@ }, { "kind": "ui-call", - "line": 613, + "line": 611, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", + "source": "Unsupported attachment", + "surface": "android", + "id": "native.android.09720189ba712747" + }, + { + "kind": "ui-call", + "line": 653, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", + "source": "Image unavailable · Tap to retry", + "surface": "android", + "id": "native.android.df7c0abff7557f23" + }, + { + "kind": "ui-call", + "line": 661, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", + "source": "Loading image…", + "surface": "android", + "id": "native.android.3fdc2798b6eada4a" + }, + { + "kind": "ui-call", + "line": 699, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Open image preview", "surface": "android", @@ -12563,7 +12603,7 @@ }, { "kind": "ui-call", - "line": 631, + "line": 717, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Image preview", "surface": "android", @@ -12571,20 +12611,12 @@ }, { "kind": "ui-call", - "line": 645, + "line": 731, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", "source": "Close image preview", "surface": "android", "id": "native.android.7f8cf7bee5cc491f" }, - { - "kind": "ui-call", - "line": 654, - "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt", - "source": "Unsupported attachment", - "surface": "android", - "id": "native.android.09720189ba712747" - }, { "kind": "ui-call", "line": 173, @@ -12659,7 +12691,7 @@ }, { "kind": "ui-call", - "line": 395, + "line": 396, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Model", "surface": "android", @@ -12667,7 +12699,7 @@ }, { "kind": "ui-call", - "line": 518, + "line": 519, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Some shared images were omitted or could not be added.", "surface": "android", @@ -12675,7 +12707,7 @@ }, { "kind": "ui-call", - "line": 672, + "line": 673, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Chat needs attention", "surface": "android", @@ -12683,7 +12715,7 @@ }, { "kind": "ui-call", - "line": 1010, + "line": 1012, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "All", "surface": "android", @@ -12691,7 +12723,7 @@ }, { "kind": "ui-call", - "line": 1097, + "line": 1099, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Working", "surface": "android", @@ -12699,7 +12731,7 @@ }, { "kind": "ui-call", - "line": 1098, + "line": 1100, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Ready", "surface": "android", @@ -12707,7 +12739,7 @@ }, { "kind": "ui-call", - "line": 1099, + "line": 1101, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Offline", "surface": "android", @@ -12715,7 +12747,7 @@ }, { "kind": "ui-call", - "line": 1112, + "line": 1114, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Chat actions", "surface": "android", @@ -12723,7 +12755,7 @@ }, { "kind": "ui-call", - "line": 1117, + "line": 1119, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Refresh chat", "surface": "android", @@ -12731,7 +12763,7 @@ }, { "kind": "ui-call", - "line": 1136, + "line": 1138, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Dashboard", "surface": "android", @@ -12739,7 +12771,7 @@ }, { "kind": "ui-call", - "line": 1144, + "line": 1146, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Background tasks", "surface": "android", @@ -12747,7 +12779,7 @@ }, { "kind": "ui-call", - "line": 1165, + "line": 1167, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Chat", "surface": "android", @@ -12755,7 +12787,7 @@ }, { "kind": "ui-call", - "line": 1342, + "line": 1346, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Messages to recover", "surface": "android", @@ -12763,7 +12795,7 @@ }, { "kind": "ui-call", - "line": 1344, + "line": 1348, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "${item.count} message(s) need recovery. Re-enter anything you want to keep, then delete these rows.", "surface": "android", @@ -12771,7 +12803,7 @@ }, { "kind": "ui-call", - "line": 1382, + "line": 1387, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Loading thread", "surface": "android", @@ -12779,7 +12811,7 @@ }, { "kind": "ui-call", - "line": 1415, + "line": 1420, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Jump to latest", "surface": "android", @@ -12787,7 +12819,7 @@ }, { "kind": "ui-call", - "line": 1494, + "line": 1499, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Ready when you are", "surface": "android", @@ -12795,7 +12827,7 @@ }, { "kind": "ui-call", - "line": 1498, + "line": 1503, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Start with a prompt, or use voice.", "surface": "android", @@ -12803,7 +12835,7 @@ }, { "kind": "ui-call", - "line": 1500, + "line": 1505, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Use the recovery options below to reconnect.", "surface": "android", @@ -12811,7 +12843,7 @@ }, { "kind": "ui-call", - "line": 1502, + "line": 1507, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Chat is checking Gateway health.", "surface": "android", @@ -12819,7 +12851,7 @@ }, { "kind": "ui-call", - "line": 1525, + "line": 1530, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Fix connection", "surface": "android", @@ -12827,7 +12859,7 @@ }, { "kind": "ui-call", - "line": 1526, + "line": 1531, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Copy diagnostics", "surface": "android", @@ -12835,7 +12867,7 @@ }, { "kind": "ui-call", - "line": 1585, + "line": 1590, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Catch me up", "surface": "android", @@ -12843,7 +12875,7 @@ }, { "kind": "ui-call", - "line": 1586, + "line": 1591, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Summarize recent threads and next steps.", "surface": "android", @@ -12851,7 +12883,7 @@ }, { "kind": "ui-call", - "line": 1587, + "line": 1592, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Catch me up on my recent OpenClaw threads and suggest next steps.", "surface": "android", @@ -12859,7 +12891,7 @@ }, { "kind": "ui-call", - "line": 1591, + "line": 1596, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Plan the work", "surface": "android", @@ -12867,7 +12899,7 @@ }, { "kind": "ui-call", - "line": 1592, + "line": 1597, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Turn a goal into an actionable checklist.", "surface": "android", @@ -12875,7 +12907,7 @@ }, { "kind": "ui-call", - "line": 1593, + "line": 1598, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Help me turn this goal into a practical checklist: ", "surface": "android", @@ -12883,7 +12915,7 @@ }, { "kind": "ui-call", - "line": 1597, + "line": 1602, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Use this phone", "surface": "android", @@ -12891,7 +12923,7 @@ }, { "kind": "ui-call", - "line": 1598, + "line": 1603, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Ask OpenClaw to use Android capabilities.", "surface": "android", @@ -12899,7 +12931,7 @@ }, { "kind": "ui-call", - "line": 1599, + "line": 1604, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "What can you help me do from this phone right now?", "surface": "android", @@ -12907,7 +12939,7 @@ }, { "kind": "ui-call", - "line": 1673, + "line": 1686, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "OpenClaw · Live", "surface": "android", @@ -12915,7 +12947,7 @@ }, { "kind": "ui-call", - "line": 1674, + "line": 1687, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "You", "surface": "android", @@ -12923,7 +12955,7 @@ }, { "kind": "ui-call", - "line": 1675, + "line": 1688, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "System", "surface": "android", @@ -12931,7 +12963,7 @@ }, { "kind": "ui-call", - "line": 1676, + "line": 1689, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "OpenClaw", "surface": "android", @@ -12939,7 +12971,23 @@ }, { "kind": "ui-call", - "line": 1758, + "line": 1716, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", + "source": "Image", + "surface": "android", + "id": "native.android.b18964167dbb3c02" + }, + { + "kind": "ui-call", + "line": 1732, + "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", + "source": "Additional images hidden: ${omittedImageCount}", + "surface": "android", + "id": "native.android.952a88e71b4aaee4" + }, + { + "kind": "ui-call", + "line": 1787, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Preparing audio…", "surface": "android", @@ -12947,7 +12995,7 @@ }, { "kind": "ui-call", - "line": 1758, + "line": 1787, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Speaking…", "surface": "android", @@ -12955,7 +13003,7 @@ }, { "kind": "ui-call", - "line": 1789, + "line": 1818, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Close", "surface": "android", @@ -12963,7 +13011,7 @@ }, { "kind": "ui-call", - "line": 1789, + "line": 1818, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "View all", "surface": "android", @@ -12971,7 +13019,7 @@ }, { "kind": "ui-call", - "line": 1819, + "line": 1848, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Tools running", "surface": "android", @@ -12979,7 +13027,7 @@ }, { "kind": "ui-call", - "line": 1821, + "line": 1850, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "OpenClaw is working", "surface": "android", @@ -12987,7 +13035,7 @@ }, { "kind": "ui-call", - "line": 1824, + "line": 1853, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "+${toolCalls.size - 4} more", "surface": "android", @@ -12995,7 +13043,7 @@ }, { "kind": "ui-named-argument", - "line": 1890, + "line": 1919, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "$completedCount/${steps.size}", "surface": "android", @@ -13003,7 +13051,7 @@ }, { "kind": "ui-call", - "line": 1897, + "line": 1926, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Collapse plan checklist", "surface": "android", @@ -13011,7 +13059,7 @@ }, { "kind": "ui-call", - "line": 1897, + "line": 1926, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Expand plan checklist", "surface": "android", @@ -13019,7 +13067,7 @@ }, { "kind": "ui-call", - "line": 2029, + "line": 2058, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Dismiss shared-image warning", "surface": "android", @@ -13027,7 +13075,7 @@ }, { "kind": "ui-call", - "line": 2132, + "line": 2161, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Stop", "surface": "android", @@ -13035,7 +13083,7 @@ }, { "kind": "ui-call", - "line": 2219, + "line": 2248, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Switch branch", "surface": "android", @@ -13043,7 +13091,7 @@ }, { "kind": "ui-call", - "line": 2243, + "line": 2272, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Untitled branch", "surface": "android", @@ -13051,7 +13099,7 @@ }, { "kind": "ui-call", - "line": 2258, + "line": 2287, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Current branch", "surface": "android", @@ -13059,7 +13107,7 @@ }, { "kind": "ui-call", - "line": 2270, + "line": 2299, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Messages: $count", "surface": "android", @@ -13067,7 +13115,7 @@ }, { "kind": "ui-call", - "line": 2278, + "line": 2307, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "$count · $updated", "surface": "android", @@ -13075,7 +13123,7 @@ }, { "kind": "ui-call", - "line": 2307, + "line": 2336, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Default", "surface": "android", @@ -13083,7 +13131,7 @@ }, { "kind": "ui-call", - "line": 2373, + "line": 2402, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Pin model", "surface": "android", @@ -13091,7 +13139,7 @@ }, { "kind": "ui-call", - "line": 2373, + "line": 2402, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Unpin model", "surface": "android", @@ -13099,7 +13147,7 @@ }, { "kind": "ui-call", - "line": 2390, + "line": 2419, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "No commands found", "surface": "android", @@ -13107,7 +13155,7 @@ }, { "kind": "ui-call", - "line": 2432, + "line": 2461, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Command", "surface": "android", @@ -13115,7 +13163,7 @@ }, { "kind": "ui-call", - "line": 2452, + "line": 2481, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Gateway offline", "surface": "android", @@ -13123,7 +13171,7 @@ }, { "kind": "ui-call", - "line": 2498, + "line": 2527, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Close thinking level selector", "surface": "android", @@ -13131,7 +13179,7 @@ }, { "kind": "ui-call", - "line": 2498, + "line": 2527, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Open thinking level selector", "surface": "android", @@ -13139,7 +13187,7 @@ }, { "kind": "ui-call", - "line": 2563, + "line": 2592, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Attach image", "surface": "android", @@ -13147,7 +13195,7 @@ }, { "kind": "ui-call", - "line": 2568, + "line": 2597, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Attachment", "surface": "android", @@ -13155,7 +13203,7 @@ }, { "kind": "ui-call", - "line": 2599, + "line": 2628, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Message OpenClaw", "surface": "android", @@ -13163,7 +13211,7 @@ }, { "kind": "ui-call", - "line": 2628, + "line": 2657, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "End Talk", "surface": "android", @@ -13171,7 +13219,7 @@ }, { "kind": "ui-call", - "line": 2628, + "line": 2657, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Start Talk", "surface": "android", @@ -13179,7 +13227,7 @@ }, { "kind": "ui-call", - "line": 2713, + "line": 2742, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Voice note · ${formatVoiceNoteDuration(duration)}", "surface": "android", @@ -13187,7 +13235,7 @@ }, { "kind": "ui-call", - "line": 2722, + "line": 2751, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Remove attachment", "surface": "android", @@ -13195,7 +13243,7 @@ }, { "kind": "ui-call", - "line": 2734, + "line": 2763, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "New chat", "surface": "android", @@ -13203,7 +13251,7 @@ }, { "kind": "ui-call", - "line": 2743, + "line": 2772, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Main", "surface": "android", @@ -13211,7 +13259,7 @@ }, { "kind": "ui-call", - "line": 2744, + "line": 2773, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Current", "surface": "android", @@ -13219,7 +13267,7 @@ }, { "kind": "ui-call", - "line": 2751, + "line": 2780, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "$emoji $name", "surface": "android", @@ -13227,7 +13275,7 @@ }, { "kind": "ui-call", - "line": 2810, + "line": 2839, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Send", "surface": "android", @@ -13235,7 +13283,7 @@ }, { "kind": "ui-call", - "line": 2821, + "line": 2850, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Chat is still checking Gateway health.", "surface": "android", @@ -13243,7 +13291,7 @@ }, { "kind": "ui-call", - "line": 2822, + "line": 2851, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Gateway is offline. Fix the connection below or copy diagnostics.", "surface": "android", @@ -13251,7 +13299,7 @@ }, { "kind": "ui-call", - "line": 2823, + "line": 2852, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Gateway authentication needs attention.", "surface": "android", @@ -13259,7 +13307,7 @@ }, { "kind": "ui-call", - "line": 2842, + "line": 2871, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Context ${(it * 100).roundToInt()}%", "surface": "android", @@ -13267,7 +13315,7 @@ }, { "kind": "ui-call", - "line": 2843, + "line": 2872, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Context --", "surface": "android", @@ -13275,7 +13323,7 @@ }, { "kind": "ui-call", - "line": 2844, + "line": 2873, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "$contextLabel · ${contextMeterThinkingLabel(thinkingLevel)}", "surface": "android", @@ -13283,7 +13331,7 @@ }, { "kind": "ui-call", - "line": 2883, + "line": 2912, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Off", "surface": "android", @@ -13291,7 +13339,7 @@ }, { "kind": "ui-call", - "line": 2884, + "line": 2913, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Minimal", "surface": "android", @@ -13299,7 +13347,7 @@ }, { "kind": "ui-call", - "line": 2885, + "line": 2914, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Low", "surface": "android", @@ -13307,7 +13355,7 @@ }, { "kind": "ui-call", - "line": 2886, + "line": 2915, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Medium", "surface": "android", @@ -13315,7 +13363,7 @@ }, { "kind": "ui-call", - "line": 2887, + "line": 2916, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "High", "surface": "android", @@ -13323,7 +13371,7 @@ }, { "kind": "ui-call", - "line": 2888, + "line": 2917, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Xhigh", "surface": "android", @@ -13331,7 +13379,7 @@ }, { "kind": "ui-call", - "line": 2889, + "line": 2918, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Adaptive", "surface": "android", @@ -13339,7 +13387,7 @@ }, { "kind": "ui-call", - "line": 2890, + "line": 2919, "path": "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", "source": "Max", "surface": "android", @@ -25795,7 +25843,7 @@ }, { "kind": "ui-localized-call", - "line": 4165, + "line": 4174, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval needed", "surface": "apple", @@ -25803,7 +25851,7 @@ }, { "kind": "ui-localized-call", - "line": 4166, + "line": 4175, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Action required", "surface": "apple", @@ -25811,7 +25859,7 @@ }, { "kind": "ui-localized-call", - "line": 4881, + "line": 4890, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connecting...", "surface": "apple", @@ -25819,7 +25867,7 @@ }, { "kind": "ui-localized-call", - "line": 4882, + "line": 4891, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Reconnecting...", "surface": "apple", @@ -25827,7 +25875,7 @@ }, { "kind": "conditional-branch", - "line": 5237, + "line": 5246, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connected", "surface": "apple", @@ -25835,7 +25883,7 @@ }, { "kind": "conditional-branch", - "line": 5237, + "line": 5246, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Offline", "surface": "apple", @@ -25843,7 +25891,7 @@ }, { "kind": "conditional-branch", - "line": 6407, + "line": 6416, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "No chat messages yet", "surface": "apple", @@ -25851,7 +25899,7 @@ }, { "kind": "conditional-branch", - "line": 6469, + "line": 6478, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Connecting…", "surface": "apple", @@ -25859,7 +25907,7 @@ }, { "kind": "conditional-branch", - "line": 6469, + "line": 6478, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Reconnecting…", "surface": "apple", @@ -25867,7 +25915,7 @@ }, { "kind": "conditional-branch", - "line": 8816, + "line": 8825, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval", "surface": "apple", @@ -25875,7 +25923,7 @@ }, { "kind": "conditional-branch", - "line": 8816, + "line": 8825, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "This approval was already", "surface": "apple", @@ -25883,7 +25931,7 @@ }, { "kind": "conditional-branch", - "line": 8822, + "line": 8831, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "This approval was already set to Always Allow.", "surface": "apple", @@ -25891,7 +25939,7 @@ }, { "kind": "conditional-branch", - "line": 8823, + "line": 8832, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "Approval set to Always Allow.", "surface": "apple", @@ -25899,7 +25947,7 @@ }, { "kind": "conditional-branch", - "line": 10193, + "line": 10202, "path": "apps/ios/Sources/Model/NodeAppModel.swift", "source": "\\(urlText.prefix(500))…", "surface": "apple", @@ -39315,7 +39363,7 @@ }, { "kind": "ui-localized-call", - "line": 931, + "line": 946, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Talk mode uses the primary Gateway window", "surface": "apple", @@ -39323,7 +39371,7 @@ }, { "kind": "ui-localized-call", - "line": 933, + "line": 948, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Talk mode off", "surface": "apple", @@ -39331,7 +39379,7 @@ }, { "kind": "ui-localized-call", - "line": 935, + "line": 950, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Talk mode paused", "surface": "apple", @@ -39339,7 +39387,7 @@ }, { "kind": "ui-localized-call", - "line": 938, + "line": 953, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Talk mode ready", "surface": "apple", @@ -39347,7 +39395,7 @@ }, { "kind": "ui-localized-call", - "line": 939, + "line": 954, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Listening", "surface": "apple", @@ -39355,7 +39403,7 @@ }, { "kind": "ui-localized-call", - "line": 940, + "line": 955, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Thinking", "surface": "apple", @@ -39363,7 +39411,7 @@ }, { "kind": "ui-localized-call", - "line": 941, + "line": 956, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Speaking", "surface": "apple", @@ -39371,7 +39419,7 @@ }, { "kind": "ui-localized-call", - "line": 945, + "line": 960, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "What would you like to work on?", "surface": "apple", @@ -39379,7 +39427,7 @@ }, { "kind": "ui-localized-call", - "line": 949, + "line": 964, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Check OpenClaw status", "surface": "apple", @@ -39387,7 +39435,7 @@ }, { "kind": "ui-localized-call", - "line": 950, + "line": 965, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Summarize the current OpenClaw status and tell me what needs attention.", "surface": "apple", @@ -39395,7 +39443,7 @@ }, { "kind": "ui-localized-call", - "line": 953, + "line": 968, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "What can you do?", "surface": "apple", @@ -39403,7 +39451,7 @@ }, { "kind": "ui-localized-call", - "line": 954, + "line": 969, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Show me what you can help with on this Mac right now.", "surface": "apple", @@ -39411,7 +39459,7 @@ }, { "kind": "ui-localized-call", - "line": 957, + "line": 972, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Catch me up", "surface": "apple", @@ -39419,7 +39467,7 @@ }, { "kind": "ui-localized-call", - "line": 958, + "line": 973, "path": "apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift", "source": "Summarize what happened in my threads since yesterday.", "surface": "apple", @@ -40089,6 +40137,46 @@ "surface": "apple", "id": "native.apple.769b7dcea4708c40" }, + { + "kind": "ui-localized-call", + "line": 57, + "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift", + "source": "Loading image…", + "surface": "apple", + "id": "native.apple.8b0faa2f4f38e339" + }, + { + "kind": "ui-localized-call", + "line": 78, + "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift", + "source": "Opens image preview", + "surface": "apple", + "id": "native.apple.e776a9e0e3ef1ae7" + }, + { + "kind": "ui-localized-call", + "line": 98, + "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift", + "source": "Close image preview", + "surface": "apple", + "id": "native.apple.cc336983da1cd070" + }, + { + "kind": "ui-localized-call", + "line": 104, + "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift", + "source": "Image unavailable", + "surface": "apple", + "id": "native.apple.4f97faec43e8e366" + }, + { + "kind": "ui-localized-call", + "line": 110, + "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift", + "source": "Retry", + "surface": "apple", + "id": "native.apple.2c50b099878d7c96" + }, { "kind": "ui-localized-call", "line": 39, @@ -40105,9 +40193,17 @@ "surface": "apple", "id": "native.apple.fccc8a9ba3da1426" }, + { + "kind": "ui-localized-call", + "line": 391, + "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", + "source": "Additional images hidden: %lld", + "surface": "apple", + "id": "native.apple.2bf58bad40fc4ae6" + }, { "kind": "conditional-branch", - "line": 413, + "line": 431, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Show less", "surface": "apple", @@ -40115,7 +40211,7 @@ }, { "kind": "conditional-branch", - "line": 413, + "line": 431, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Show more", "surface": "apple", @@ -40123,7 +40219,7 @@ }, { "kind": "conditional-branch", - "line": 427, + "line": 445, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Collapsed", "surface": "apple", @@ -40131,7 +40227,7 @@ }, { "kind": "conditional-branch", - "line": 427, + "line": 445, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Expanded", "surface": "apple", @@ -40139,7 +40235,7 @@ }, { "kind": "ui-localized-call", - "line": 447, + "line": 465, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Message usage", "surface": "apple", @@ -40147,15 +40243,23 @@ }, { "kind": "conditional-branch", - "line": 627, + "line": 672, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Voice note", "surface": "apple", "id": "native.apple.3c8c3679fd99c074" }, + { + "kind": "ui-localized-call", + "line": 704, + "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", + "source": "Attachment", + "surface": "apple", + "id": "native.apple.eca473edeb132a44" + }, { "kind": "ui-call", - "line": 684, + "line": 749, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Writing", "surface": "apple", @@ -40163,7 +40267,7 @@ }, { "kind": "ui-call", - "line": 718, + "line": 783, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Preparing audio…", "surface": "apple", @@ -40171,7 +40275,7 @@ }, { "kind": "ui-call", - "line": 721, + "line": 786, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Speaking…", "surface": "apple", @@ -40179,7 +40283,7 @@ }, { "kind": "conditional-branch", - "line": 729, + "line": 794, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Preparing audio, tap to cancel", "surface": "apple", @@ -40187,7 +40291,7 @@ }, { "kind": "conditional-branch", - "line": 730, + "line": 795, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift", "source": "Speaking, tap to stop", "surface": "apple", @@ -41371,7 +41475,7 @@ }, { "kind": "ui-call", - "line": 571, + "line": 578, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Retry Send", "surface": "apple", @@ -41379,7 +41483,7 @@ }, { "kind": "ui-call", - "line": 585, + "line": 592, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Delete", "surface": "apple", @@ -41387,7 +41491,7 @@ }, { "kind": "ui-call", - "line": 621, + "line": 628, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Stop Listening", "surface": "apple", @@ -41395,7 +41499,7 @@ }, { "kind": "ui-call", - "line": 624, + "line": 631, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Listen", "surface": "apple", @@ -41403,7 +41507,7 @@ }, { "kind": "ui-modifier", - "line": 721, + "line": 728, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Jump to latest reply", "surface": "apple", @@ -41411,7 +41515,7 @@ }, { "kind": "ui-named-argument", - "line": 741, + "line": 748, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Refresh", "surface": "apple", @@ -41419,7 +41523,7 @@ }, { "kind": "ui-call", - "line": 1157, + "line": 1164, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Copy Message", "surface": "apple", @@ -41427,7 +41531,7 @@ }, { "kind": "ui-call", - "line": 1180, + "line": 1187, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Open Full Message", "surface": "apple", @@ -41435,7 +41539,7 @@ }, { "kind": "ui-call", - "line": 1199, + "line": 1206, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Rewind to Here", "surface": "apple", @@ -41443,7 +41547,7 @@ }, { "kind": "ui-call", - "line": 1219, + "line": 1226, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Fork from Here", "surface": "apple", @@ -41451,7 +41555,7 @@ }, { "kind": "ui-localized-call", - "line": 1245, + "line": 1252, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Reply", "surface": "apple", @@ -41459,7 +41563,7 @@ }, { "kind": "ui-localized-call", - "line": 1255, + "line": 1262, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "You", "surface": "apple", @@ -41467,7 +41571,7 @@ }, { "kind": "ui-localized-call", - "line": 1257, + "line": 1264, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Assistant", "surface": "apple", @@ -41475,7 +41579,7 @@ }, { "kind": "ui-call", - "line": 1323, + "line": 1330, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Loading chat", "surface": "apple", @@ -41483,7 +41587,7 @@ }, { "kind": "ui-modifier", - "line": 1403, + "line": 1410, "path": "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift", "source": "Dismiss", "surface": "apple", diff --git a/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt b/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt index 6e3fe88d3cbb..7c3149b4ba11 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt @@ -1305,6 +1305,8 @@ class MainViewModel private constructor( failedResource: ChatWidgetResource?, ) = ensureRuntime().resolveInlineWidgetResource(path, failedResource) + internal suspend fun loadChatImageArtifact(artifactId: String) = ensureRuntime().loadChatImageArtifact(artifactId) + fun requestCanvasRehydrate(source: String = "screen_tab") { ensureRuntime().requestCanvasRehydrate(source = source, force = true) } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt b/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt index 1d018a982492..abc6981ab0f1 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt @@ -4910,6 +4910,8 @@ class NodeRuntime private constructor( } } + internal suspend fun loadChatImageArtifact(artifactId: String) = chat.loadImageArtifact(artifactId) + fun loadChat( sessionKey: String, ownerAgentId: String? = null, diff --git a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt index 705974edf704..300dcfc5dd2f 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt @@ -1,6 +1,7 @@ package ai.openclaw.app.chat import ai.openclaw.app.GatewayModelSummary +import ai.openclaw.app.gateway.GatewayLoadedImage import ai.openclaw.app.gateway.GatewayRequestDefinitiveFailure import ai.openclaw.app.gateway.GatewayRequestNotEnqueued import ai.openclaw.app.gateway.GatewayRequestOutcomeUnknown @@ -56,6 +57,8 @@ internal const val SESSION_LIST_FETCH_LIMIT = 200 private val QUESTION_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L) private val SWARM_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L) private const val SESSION_EDITOR_MAX_BASE64_CHARS = ((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 2) / 3) * 4 +private val MANAGED_IMAGE_PATH_REGEX = + Regex("^/api/chat/media/outgoing/[^/]+/([0-9a-fA-F-]{36})/full(?:\\?.*)?$") internal fun chatOutboxQueueFailureText(): NativeText = ChatController.queueFailureText() @@ -111,6 +114,12 @@ class ChatController internal constructor( private val cacheScope: () -> ChatCacheScope? = { null }, private val currentDefaultAgentId: () -> String? = { "main" }, private val currentDefaultAgentRevision: () -> Long = { 0L }, + private val loadGatewayImageArtifact: suspend ( + gatewayId: String?, + sessionKey: String, + agentId: String?, + artifactId: String, + ) -> GatewayLoadedImage? = { _, _, _, _ -> null }, private val commandOutbox: ChatCommandOutbox? = null, private val recordModelRecent: (String) -> Unit = {}, private val onSessionDeleted: (ChatSessionDeletion) -> Unit = {}, @@ -142,12 +151,26 @@ class ChatController internal constructor( cacheScope = cacheScope, currentDefaultAgentId = currentDefaultAgentId, currentDefaultAgentRevision = currentDefaultAgentRevision, + loadGatewayImageArtifact = { gatewayId, sessionKey, agentId, artifactId -> + session.loadImageArtifact(gatewayId, sessionKey, agentId, artifactId) + }, commandOutbox = commandOutbox, recordModelRecent = recordModelRecent, onSessionDeleted = onSessionDeleted, onOfflineDefaultAgentRestored = onOfflineDefaultAgentRestored, ) + suspend fun loadImageArtifact(artifactId: String): GatewayLoadedImage? { + val normalizedArtifactId = artifactId.trim().takeIf(String::isNotEmpty) ?: return null + val sessionKey = normalizeRequestedSessionKey(_sessionKey.value) + return loadGatewayImageArtifact( + currentCacheScope()?.gatewayId, + sessionKey, + resolveAgentIdForSessionKey(sessionKey), + normalizedArtifactId, + ) + } + private var appliedMainSessionKey = "main" private val cacheMutationMutex = Mutex() private val defaultAgentPersistenceMutex = Mutex() @@ -6368,10 +6391,18 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? { "image", "audio" -> { val type = obj["type"].asStringOrNull() ?: "image" val inlineContent = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() } + val url = obj["url"].asStringOrNull() ChatMessageContent( type = type, mimeType = obj["mimeType"].asStringOrNull(), fileName = obj["fileName"].asStringOrNull(), + artifactId = obj["artifactId"].asStringOrNull() ?: managedImageArtifactId(url), + url = url, + openUrl = obj["openUrl"].asStringOrNull(), + alt = obj["alt"].asStringOrNull(), + width = obj["width"].asLongOrNull()?.toInt(), + height = obj["height"].asLongOrNull()?.toInt(), + sizeBytes = obj["sizeBytes"].asLongOrNull(), base64 = inlineContent?.takeIf { type != "image" || it.length <= CHAT_IMAGE_MAX_BASE64_CHARS }, ) } @@ -6415,6 +6446,16 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? { } } +internal fun managedImageArtifactId(rawUrl: String?): String? { + val match = + rawUrl + ?.trim() + ?.let(MANAGED_IMAGE_PATH_REGEX::matchEntire) + ?: return null + val attachmentId = runCatching { UUID.fromString(match.groupValues[1]).toString() }.getOrNull() ?: return null + return "artifact_managed_image_$attachmentId" +} + internal fun parseChatMessageContents(obj: JsonObject): List { val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseChatMessageContent) @@ -6655,6 +6696,9 @@ private fun messageContentIdentityKey(message: ChatMessage): String? { ?.lowercase() .orEmpty(), part.fileName?.trim().orEmpty(), + part.artifactId?.trim().orEmpty(), + part.url?.trim().orEmpty(), + part.openUrl?.trim().orEmpty(), part.base64 ?.hashCode() ?.toString() diff --git a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt index 4b50e44a7b85..a0b9154a1b98 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatModels.kt @@ -63,13 +63,20 @@ data class ChatTranscriptAnchorState( ) /** - * One content part in a chat message; binary parts carry base64 plus their MIME metadata. + * One content part in a chat message; images carry either bounded base64 or a managed artifact reference. */ data class ChatMessageContent( val type: String = "text", val text: String? = null, val mimeType: String? = null, val fileName: String? = null, + val artifactId: String? = null, + val url: String? = null, + val openUrl: String? = null, + val alt: String? = null, + val width: Int? = null, + val height: Int? = null, + val sizeBytes: Long? = null, val base64: String? = null, val durationMs: Long? = null, val widget: ChatWidgetPreview? = null, diff --git a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatTranscriptCache.kt b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatTranscriptCache.kt index fe79cc4e3e50..8cadf6005400 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatTranscriptCache.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/chat/ChatTranscriptCache.kt @@ -6,6 +6,7 @@ import androidx.room.Insert import androidx.room.OnConflictStrategy import androidx.room.Query import androidx.room.withTransaction +import kotlinx.serialization.Serializable import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.builtins.serializer import kotlinx.serialization.json.Json @@ -17,6 +18,21 @@ internal const val MAX_CACHED_SESSIONS = 50 /** Upper bound of cached transcript rows per session; only the newest messages are kept. */ internal const val MAX_CACHED_MESSAGES_PER_SESSION = 200 +@Serializable +private data class CachedMessageContent( + val type: String, + val text: String? = null, + val mimeType: String? = null, + val fileName: String? = null, + val artifactId: String? = null, + val url: String? = null, + val openUrl: String? = null, + val alt: String? = null, + val width: Int? = null, + val height: Int? = null, + val sizeBytes: Long? = null, +) + /** * Read-only offline cache of chat sessions and transcripts. * @@ -92,7 +108,7 @@ internal data class CachedMessageEntity( val sessionKey: String, val rowOrder: Int, val role: String, - // JSON array of text part strings; attachments/binary parts are never persisted. + // JSON array of text and managed-image references; attachment bytes are never persisted. val textPartsJson: String, val timestampMs: Long?, // Kept so live history reconciliation can match cached rows by identity key. @@ -232,7 +248,8 @@ class RoomChatTranscriptCache internal constructor( private val database: GatewayCacheDatabase, ) : ChatTranscriptCache { private val json = Json - private val textPartsSerializer = ListSerializer(String.serializer()) + private val cachedContentSerializer = ListSerializer(CachedMessageContent.serializer()) + private val legacyTextPartsSerializer = ListSerializer(String.serializer()) override suspend fun loadLastDefaultAgentId(gatewayId: String): String? { val gateway = scopedGatewayId(gatewayId) ?: return null @@ -287,7 +304,22 @@ class RoomChatTranscriptCache internal constructor( ChatMessage( id = UUID.randomUUID().toString(), role = role, - content = decodeTextParts(row.textPartsJson).map { ChatMessageContent(type = "text", text = it) }, + content = + decodeCachedContent(row.textPartsJson).map { part -> + ChatMessageContent( + type = part.type, + text = part.text, + mimeType = part.mimeType, + fileName = part.fileName, + artifactId = part.artifactId, + url = part.url, + openUrl = part.openUrl, + alt = part.alt, + width = part.width, + height = part.height, + sizeBytes = part.sizeBytes, + ) + }, timestampMs = row.timestampMs, idempotencyKey = row.idempotencyKey, // Canonical tree ids stay live-only; cached rows regain actions after history refresh. @@ -367,23 +399,44 @@ class RoomChatTranscriptCache internal constructor( val gateway = scopedGatewayId(gatewayId) ?: return val agent = scopedAgentId(agentId) ?: return val key = sessionKey.trim().takeIf { it.isNotEmpty() } ?: return - // Text rows only: attachment/binary parts are dropped, and messages without any text are skipped. + // Persist small managed-image references, never attachment bytes. This keeps generated images + // visible offline without turning the disposable transcript cache into a binary store. val rows = messages .mapNotNull { message -> val role = normalizeVisibleChatMessageRole(message.role) ?: return@mapNotNull null - val textParts = message.content.filter { it.type == "text" }.mapNotNull { it.text } - if (textParts.isEmpty()) return@mapNotNull null - Triple(message, role, textParts) + val content = + message.content.mapNotNull { part -> + when { + part.type == "text" && !part.text.isNullOrBlank() -> + CachedMessageContent(type = "text", text = part.text) + part.type == "image" && !part.artifactId.isNullOrBlank() && !part.url.isNullOrBlank() -> + CachedMessageContent( + type = "image", + mimeType = part.mimeType, + fileName = part.fileName, + artifactId = part.artifactId, + url = part.url, + openUrl = part.openUrl, + alt = part.alt, + width = part.width, + height = part.height, + sizeBytes = part.sizeBytes, + ) + else -> null + } + } + if (content.isEmpty()) return@mapNotNull null + Triple(message, role, content) }.takeLast(MAX_CACHED_MESSAGES_PER_SESSION) - .mapIndexed { index, (message, role, textParts) -> + .mapIndexed { index, (message, role, content) -> CachedMessageEntity( gatewayId = gateway, agentId = agent, sessionKey = key, rowOrder = index, role = role, - textPartsJson = json.encodeToString(textPartsSerializer, textParts), + textPartsJson = json.encodeToString(cachedContentSerializer, content), timestampMs = message.timestampMs, idempotencyKey = message.idempotencyKey, ) @@ -452,5 +505,12 @@ class RoomChatTranscriptCache internal constructor( private fun scopedAgentId(agentId: String): String? = agentId.trim().takeIf { it.isNotEmpty() } - private fun decodeTextParts(encoded: String): List = runCatching { json.decodeFromString(textPartsSerializer, encoded) }.getOrDefault(emptyList()) + private fun decodeCachedContent(encoded: String): List = + runCatching { json.decodeFromString(cachedContentSerializer, encoded) }.getOrElse { + // Offline transcript browsing is shipped behavior. Keep the previous string-array rows + // readable until a live history refresh naturally rewrites this disposable cache entry. + runCatching { json.decodeFromString(legacyTextPartsSerializer, encoded) } + .getOrDefault(emptyList()) + .map { CachedMessageContent(type = "text", text = it) } + } } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt index 439398d5b1db..fc0b0de66ccd 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt @@ -33,6 +33,8 @@ import okhttp3.Request import okhttp3.Response import okhttp3.WebSocket import okhttp3.WebSocketListener +import okio.Buffer +import java.net.URI import java.util.Locale import java.util.UUID import java.util.concurrent.ConcurrentHashMap @@ -53,6 +55,11 @@ data class GatewayClientInfo( val modelIdentifier: String?, ) +data class GatewayLoadedImage( + val bytes: ByteArray, + val mimeType: String, +) + /** * Role, scopes, commands, and permission snapshot sent with the connect frame. */ @@ -616,6 +623,38 @@ class GatewaySession( throw GatewayRequestRejected(res.error ?: ErrorShape("UNAVAILABLE", "request failed")) } + suspend fun loadImageArtifact( + expectedEndpointStableId: String?, + sessionKey: String, + agentId: String?, + artifactId: String, + ): GatewayLoadedImage? { + val conn = readyConnection(expectedEndpointStableId) ?: return null + val params = + buildJsonObject { + put("sessionKey", JsonPrimitive(sessionKey)) + agentId?.trim()?.takeIf(String::isNotEmpty)?.let { put("agentId", JsonPrimitive(it)) } + put("artifactId", JsonPrimitive(artifactId)) + } + val response = conn.request(GatewayMethod.ArtifactsDownload.rawValue, params, timeoutMs = 15_000) + if (!response.ok) { + throw GatewayRequestRejected(response.error ?: ErrorShape("UNAVAILABLE", "artifact download failed")) + } + val ticketedPath = + response.payloadJson + ?.let(::parseJsonOrNull) + .asObjectOrNull() + ?.get("url") + .asStringOrNull() + ?.trim() + ?.takeIf(String::isNotEmpty) + ?: return null + val loaded = conn.loadTicketedImage(ticketedPath) ?: return null + return synchronized(lifecycleLock) { + loaded.takeIf { currentConnection === conn && conn.isReady() } + } + } + internal suspend fun requestForEndpoint( expectedEndpointStableId: String, method: String, @@ -830,6 +869,46 @@ class GatewaySession( } } + suspend fun loadTicketedImage(ticketedPath: String): GatewayLoadedImage? = + withContext(Dispatchers.IO) { + val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return@withContext null + val rawPath = uri.rawPath ?: return@withContext null + val rawQuery = uri.rawQuery ?: return@withContext null + if (uri.isAbsolute || uri.rawAuthority != null || uri.rawFragment != null) return@withContext null + if (!rawPath.startsWith("/api/chat/media/outgoing/") || !rawQuery.contains("mediaTicket=")) { + return@withContext null + } + val scheme = if (tlsConfig != null) "https" else "http" + val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath" + val request = Request.Builder().url(url).header("Accept", "image/*") + if (tlsConfig != null) { + for ((name, value) in GatewayCustomHeaders.sanitized(customHeadersProvider?.invoke(endpoint.stableId).orEmpty())) { + request.header(name, value) + } + } + val call = client.newCall(request.build()) + call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS) + call.execute().use { response -> + if (!response.isSuccessful) return@withContext null + val body = response.body + val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null + if (!mimeType.startsWith("image/")) return@withContext null + val maximumBytes = 12L * 1024L * 1024L + val declaredLength = body.contentLength() + if (declaredLength > maximumBytes) return@withContext null + val buffer = Buffer() + val source = body.source() + var total = 0L + while (true) { + val read = source.read(buffer, minOf(8192L, maximumBytes + 1L - total)) + if (read == -1L) break + total += read + if (total > maximumBytes) return@withContext null + } + GatewayLoadedImage(bytes = buffer.readByteArray(), mimeType = mimeType) + } + } + @OptIn(DelicateCoroutinesApi::class) suspend fun sendRequestFrame( method: String, diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt index b0ce920260e8..ba36a65095e1 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatImageCodec.kt @@ -171,11 +171,18 @@ internal fun decodeBase64Bitmap( maxDimension: Int = CHAT_DECODE_MAX_DIMENSION, ): Bitmap? { if (base64.length > CHAT_IMAGE_MAX_BASE64_CHARS) return null - val cacheKey = "$maxDimension:${base64.length}:${base64.hashCode()}" - decodedBitmapCache.get(cacheKey)?.let { return it } - val bytes = Base64.decode(base64, Base64.DEFAULT) - if (bytes.isEmpty()) return null + return decodeImageBytes(bytes, maxDimension) +} + +/** Decodes already-authorized image bytes without base64 expansion. */ +internal fun decodeImageBytes( + bytes: ByteArray, + maxDimension: Int = CHAT_DECODE_MAX_DIMENSION, +): Bitmap? { + if (bytes.isEmpty() || bytes.size > 12 * 1024 * 1024) return null + val cacheKey = "$maxDimension:${bytes.size}:${bytes.contentHashCode()}" + decodedBitmapCache.get(cacheKey)?.let { return it } val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds) diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt index 6b0db49a2a2a..bf8229c2ffa8 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageViews.kt @@ -10,6 +10,7 @@ import ai.openclaw.app.chat.MessageSpeechState import ai.openclaw.app.chat.OUTBOX_BRANCH_CHANGED_ERROR import ai.openclaw.app.chat.chatOutboxDisplayError import ai.openclaw.app.chat.normalizeVisibleChatMessageRole +import ai.openclaw.app.gateway.GatewayLoadedImage import ai.openclaw.app.i18n.nativeString import ai.openclaw.app.i18n.nativeStringResource import ai.openclaw.app.tools.ToolDisplayRegistry @@ -83,6 +84,8 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.compose.ui.window.Dialog import androidx.compose.ui.window.DialogProperties +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import java.util.Locale private data class ChatBubbleStyle( @@ -102,6 +105,8 @@ internal fun ChatMessageBubble( onForkMessage: (String) -> Unit = {}, speechState: MessageSpeechState? = null, onToggleListen: ((String, String) -> Unit)? = null, + imageResolverReady: Boolean = false, + loadImageArtifact: suspend (String) -> GatewayLoadedImage? = { null }, ) { val role = normalizeVisibleChatMessageRole(message.role) ?: return val style = bubbleStyle(role) @@ -111,7 +116,7 @@ internal fun ChatMessageBubble( message.content.filter { part -> when (part.type) { "text" -> !part.text.isNullOrBlank() - "image" -> !part.base64.isNullOrBlank() + "image" -> !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank() else -> part.isAudioAttachment() } } @@ -138,7 +143,12 @@ internal fun ChatMessageBubble( modifier = Modifier.fillMaxWidth(), ) { ChatBubbleContainer(style = style, roleLabel = roleLabel(role)) { - ChatMessageBody(content = displayableContent, textColor = mobileText) + ChatMessageBody( + content = displayableContent, + textColor = mobileText, + imageResolverReady = imageResolverReady, + loadImageArtifact = loadImageArtifact, + ) ChatMessageLinkPreview(messageId = message.id, role = role, content = displayableContent) messageSpeech?.let { speech -> MessageSpeechIndicator( @@ -223,6 +233,8 @@ private fun ChatBubbleContainer( private fun ChatMessageBody( content: List, textColor: Color, + imageResolverReady: Boolean, + loadImageArtifact: suspend (String) -> GatewayLoadedImage?, ) { Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { for (part in content) { @@ -232,9 +244,17 @@ private fun ChatMessageBody( ChatMarkdown(text = text, textColor = textColor) } part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs) + part.type == "image" && !part.base64.isNullOrBlank() -> + ChatBase64Image(base64 = part.base64, mimeType = part.mimeType) + part.type == "image" && !part.artifactId.isNullOrBlank() -> + ChatManagedImage( + artifactId = part.artifactId, + label = part.alt?.takeIf(String::isNotBlank) ?: part.fileName ?: nativeString("Image"), + resolverReady = imageResolverReady, + loadImage = loadImageArtifact, + ) else -> { - val b64 = part.base64 ?: continue - ChatBase64Image(base64 = b64, mimeType = part.mimeType) + Text(part.fileName ?: nativeString("Attachment"), style = mobileCaption1, color = mobileTextSecondary) } } } @@ -583,75 +603,138 @@ internal fun ChatBase64Image( mimeType: String?, ) { val imageState = rememberBase64ImageState(base64) - var previewVisible by rememberSaveable(base64) { mutableStateOf(false) } val image = imageState.image if (image != null) { - Surface( - onClick = { previewVisible = true }, - shape = RoundedCornerShape(10.dp), - border = BorderStroke(1.dp, mobileBorder), - color = mobileCardSurface, - modifier = Modifier.fillMaxWidth(), + ChatImagePreview(image = image, description = mimeType ?: nativeString("Attachment"), stateKey = base64) + } else if (imageState.failed) { + Text(nativeString("Unsupported attachment"), style = mobileCaption1, color = mobileTextSecondary) + } +} + +@Composable +internal fun ChatManagedImage( + artifactId: String, + label: String, + resolverReady: Boolean, + loadImage: suspend (String) -> GatewayLoadedImage?, +) { + var image by remember(artifactId) { mutableStateOf(null) } + var failed by remember(artifactId) { mutableStateOf(false) } + var retryGeneration by rememberSaveable(artifactId) { mutableStateOf(0) } + + LaunchedEffect(artifactId, resolverReady, retryGeneration) { + if (!resolverReady) { + failed = true + image = null + return@LaunchedEffect + } + failed = false + image = null + val loaded = runCatching { loadImage(artifactId) }.getOrNull() + image = + loaded?.let { value -> + withContext(Dispatchers.Default) { decodeImageBytes(value.bytes)?.asImageBitmap() } + } + failed = image == null + } + + when { + image != null -> ChatImagePreview(image = checkNotNull(image), description = label, stateKey = artifactId) + failed -> + Surface( + onClick = { retryGeneration += 1 }, + shape = RoundedCornerShape(10.dp), + border = BorderStroke(1.dp, mobileBorder), + color = mobileCardSurface, + modifier = Modifier.fillMaxWidth(), + ) { + Text( + nativeString("Image unavailable · Tap to retry"), + modifier = Modifier.padding(12.dp), + style = mobileCaption1, + color = mobileTextSecondary, + ) + } + else -> + Text( + nativeString("Loading image…"), + modifier = Modifier.padding(12.dp), + style = mobileCaption1, + color = mobileTextSecondary, + ) + } +} + +@Composable +private fun ChatImagePreview( + image: ImageBitmap, + description: String, + stateKey: String, +) { + var previewVisible by rememberSaveable(stateKey) { mutableStateOf(false) } + Surface( + onClick = { previewVisible = true }, + shape = RoundedCornerShape(10.dp), + border = BorderStroke(1.dp, mobileBorder), + color = mobileCardSurface, + modifier = Modifier.fillMaxWidth(), + ) { + Box { + Image( + bitmap = image, + contentDescription = description, + contentScale = ContentScale.Fit, + modifier = Modifier.fillMaxWidth(), + ) + Surface( + modifier = Modifier.align(Alignment.BottomEnd).padding(8.dp).size(32.dp), + shape = CircleShape, + color = Color.Black.copy(alpha = 0.62f), + contentColor = Color.White, + ) { + Box(contentAlignment = Alignment.Center) { + Icon( + imageVector = Icons.Default.OpenInFull, + contentDescription = nativeString("Open image preview"), + modifier = Modifier.size(17.dp), + ) + } + } + } + } + if (previewVisible) { + Dialog( + onDismissRequest = { previewVisible = false }, + properties = DialogProperties(usePlatformDefaultWidth = false), ) { - Box { + Box( + modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.96f)).clickable { previewVisible = false }, + contentAlignment = Alignment.Center, + ) { Image( bitmap = image, - contentDescription = mimeType ?: nativeString("Attachment"), + contentDescription = nativeString("Image preview"), contentScale = ContentScale.Fit, - modifier = Modifier.fillMaxWidth(), + modifier = Modifier.fillMaxSize().padding(20.dp), ) Surface( - modifier = Modifier.align(Alignment.BottomEnd).padding(8.dp).size(32.dp), + onClick = { previewVisible = false }, + modifier = Modifier.align(Alignment.TopEnd).padding(16.dp).size(44.dp), shape = CircleShape, color = Color.Black.copy(alpha = 0.62f), contentColor = Color.White, ) { Box(contentAlignment = Alignment.Center) { Icon( - imageVector = Icons.Default.OpenInFull, - contentDescription = nativeString("Open image preview"), - modifier = Modifier.size(17.dp), + imageVector = Icons.Default.Close, + contentDescription = nativeString("Close image preview"), + modifier = Modifier.size(22.dp), ) } } } } - if (previewVisible) { - Dialog( - onDismissRequest = { previewVisible = false }, - properties = DialogProperties(usePlatformDefaultWidth = false), - ) { - Box( - modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.96f)).clickable { previewVisible = false }, - contentAlignment = Alignment.Center, - ) { - Image( - bitmap = image, - contentDescription = nativeString("Image preview"), - contentScale = ContentScale.Fit, - modifier = Modifier.fillMaxSize().padding(20.dp), - ) - Surface( - onClick = { previewVisible = false }, - modifier = Modifier.align(Alignment.TopEnd).padding(16.dp).size(44.dp), - shape = CircleShape, - color = Color.Black.copy(alpha = 0.62f), - contentColor = Color.White, - ) { - Box(contentAlignment = Alignment.Center) { - Icon( - imageVector = Icons.Default.Close, - contentDescription = nativeString("Close image preview"), - modifier = Modifier.size(22.dp), - ) - } - } - } - } - } - } else if (imageState.failed) { - Text(nativeString("Unsupported attachment"), style = mobileCaption1, color = mobileTextSecondary) } } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt index ccc17eaf7b16..35153b40656a 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt @@ -32,6 +32,7 @@ import ai.openclaw.app.chat.questionsForSession import ai.openclaw.app.chat.resolveChatComposerOwner import ai.openclaw.app.chat.resolveGatewayDefaultAgentId import ai.openclaw.app.currentAppLanguage +import ai.openclaw.app.gateway.GatewayLoadedImage import ai.openclaw.app.i18n.NativeText import ai.openclaw.app.i18n.joinedNativeText import ai.openclaw.app.i18n.nativeString @@ -739,6 +740,7 @@ fun ChatScreen( speechState = messageSpeechState, onToggleListen = viewModel::toggleChatMessageSpeech, resolveInlineWidgetResource = viewModel::resolveInlineWidgetResource, + loadImageArtifact = viewModel::loadChatImageArtifact, modifier = Modifier.weight(1f), ) @@ -1255,6 +1257,7 @@ private fun ChatMessageList( speechState: MessageSpeechState?, onToggleListen: (String, String) -> Unit, resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?, + loadImageArtifact: suspend (String) -> GatewayLoadedImage?, modifier: Modifier = Modifier, ) { val baseTimeline = @@ -1323,6 +1326,7 @@ private fun ChatMessageList( onToggleListen = onToggleListen, inlineWidgetResolverReady = healthOk, resolveInlineWidgetResource = resolveInlineWidgetResource, + loadImageArtifact = loadImageArtifact, ) is ChatTimelineItem.OutboxCommand -> ChatOutboxBubble( @@ -1366,6 +1370,7 @@ private fun ChatMessageList( onToggleListen = onToggleListen, inlineWidgetResolverReady = healthOk, resolveInlineWidgetResource = resolveInlineWidgetResource, + loadImageArtifact = loadImageArtifact, ) ChatTimelineItem.Thinking -> { val run = workingRun @@ -1616,18 +1621,26 @@ private fun ChatBubble( onToggleListen: (String, String) -> Unit, inlineWidgetResolverReady: Boolean, resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?, + loadImageArtifact: suspend (String) -> GatewayLoadedImage?, ) { val normalizedRole = role.trim().lowercase(Locale.US) val isUser = normalizedRole == "user" + var visibleImageCount = 0 val displayableContent = content.filter { part -> when (part.type) { "text" -> !part.text.isNullOrBlank() - "image" -> !part.base64.isNullOrBlank() + "image" -> { + val displayable = !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank() + val visible = displayable && visibleImageCount < 4 + if (displayable) visibleImageCount += 1 + visible + } "canvas" -> normalizedRole == "assistant" && part.widget != null else -> part.isAudioAttachment() } } + val omittedImageCount = (visibleImageCount - 4).coerceAtLeast(0) if (displayableContent.isEmpty()) return val messageText = chatMessagePlainText(displayableContent) @@ -1692,10 +1705,19 @@ private fun ChatBubble( part.type == "text" -> Unit part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs) part.type == "image" -> - ChatBase64Image( - base64 = checkNotNull(part.base64), - mimeType = part.mimeType, - ) + if (!part.base64.isNullOrBlank()) { + ChatBase64Image( + base64 = part.base64, + mimeType = part.mimeType, + ) + } else { + ChatManagedImage( + artifactId = checkNotNull(part.artifactId), + label = part.alt?.takeIf(String::isNotBlank) ?: part.fileName ?: nativeString("Image"), + resolverReady = inlineWidgetResolverReady, + loadImage = loadImageArtifact, + ) + } part.type == "canvas" && normalizedRole == "assistant" -> ChatInlineWidget( preview = checkNotNull(part.widget), @@ -1705,6 +1727,13 @@ private fun ChatBubble( else -> Text(text = part.fileName ?: nativeString("Attachment"), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted) } } + if (omittedImageCount > 0) { + Text( + text = nativeString("Additional images hidden: \${omittedImageCount}", omittedImageCount), + style = ClawTheme.type.caption, + color = ClawTheme.colors.textMuted, + ) + } if (messageId != null) { ChatMessageLinkPreview(messageId = messageId, role = normalizedRole, content = displayableContent) } diff --git a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt index 3f275c23b8d6..e46fd6956440 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatControllerMessageIdentityTest.kt @@ -60,6 +60,30 @@ class ChatControllerMessageIdentityTest { assertEquals(listOf(ChatMessageContent(type = "text", text = "Hi there")), content) } + @Test + fun managedImagesParticipateInMessageIdentity() { + fun message(artifactId: String) = + ChatMessage( + id = artifactId, + role = "assistant", + content = + listOf( + ChatMessageContent( + type = "image", + artifactId = artifactId, + url = "/api/chat/media/outgoing/main/$artifactId/full", + mimeType = "image/png", + ), + ), + timestampMs = 1, + ) + + assertNotEquals( + messageIdentityKey(message("artifact_managed_image_11111111-1111-4111-8111-111111111111")), + messageIdentityKey(message("artifact_managed_image_22222222-2222-4222-8222-222222222222")), + ) + } + @Test @OptIn(ExperimentalCoroutinesApi::class) fun liveHistoryDropsInternalRoleRows() = diff --git a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt index ed5d055bf208..76b96f208d97 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/chat/ChatMessageContentParsingTest.kt @@ -276,14 +276,14 @@ class ChatMessageContentParsingTest { } @Test - fun parsesImageBlocksOnlyWhenInlineContentExists() { + fun parsesInlineAndManagedImageBlocks() { val image = Json.parseToJsonElement( """{"type":"image","mimeType":"image/png","fileName":"chart.png","content":"abc123"}""", ) val managedImage = Json.parseToJsonElement( - """{"type":"image","mimeType":"image/png","fileName":"chart.png","url":"/api/chat/media/outgoing/main/id"}""", + """{"type":"image","artifactId":"artifact_managed_image_11111111-1111-4111-8111-111111111111","mimeType":"image/png","fileName":"chart.png","url":"/api/chat/media/outgoing/main/id","openUrl":"/api/chat/media/outgoing/main/id","alt":"Chart","width":1200,"height":800,"sizeBytes":2048}""", ) assertEquals( @@ -291,11 +291,35 @@ class ChatMessageContentParsingTest { parseChatMessageContent(image), ) assertEquals( - ChatMessageContent(type = "image", mimeType = "image/png", fileName = "chart.png", base64 = null), + ChatMessageContent( + type = "image", + mimeType = "image/png", + fileName = "chart.png", + artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111", + url = "/api/chat/media/outgoing/main/id", + openUrl = "/api/chat/media/outgoing/main/id", + alt = "Chart", + width = 1200, + height = 800, + sizeBytes = 2048, + ), parseChatMessageContent(managedImage), ) } + @Test + fun derivesArtifactIdentityForShippedManagedImageBlocks() { + val image = + Json.parseToJsonElement( + """{"type":"image","mimeType":"image/png","url":"/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full"}""", + ) + + assertEquals( + "artifact_managed_image_11111111-1111-4111-8111-111111111111", + parseChatMessageContent(image)?.artifactId, + ) + } + @Test fun dropsOversizedInlineImageContentBeforeRendering() { val oversized = "A".repeat(CHAT_IMAGE_MAX_BASE64_CHARS + 1) diff --git a/apps/android/app/src/test/java/ai/openclaw/app/chat/RoomChatTranscriptCacheTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/chat/RoomChatTranscriptCacheTest.kt index f1362a3476d0..dae2e1e9809c 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/chat/RoomChatTranscriptCacheTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/chat/RoomChatTranscriptCacheTest.kt @@ -40,10 +40,18 @@ class RoomChatTranscriptCacheTest { ) @Test - fun transcriptRoundTripKeepsTextRowsOnly() = + fun transcriptRoundTripKeepsTextAndManagedReferencesWithoutBinaryParts() = runTest { val store = cache() val imagePart = ChatMessageContent(type = "image", mimeType = "image/png", fileName = "a.png", base64 = "AAAA") + val managedImage = + ChatMessageContent( + type = "image", + mimeType = "image/png", + artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111", + url = "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full", + alt = "Managed image", + ) store.saveTranscript( gatewayId = "gateway-a", agentId = "main", @@ -51,19 +59,44 @@ class RoomChatTranscriptCacheTest { messages = listOf( message("hello", role = "user", timestampMs = 10, idempotencyKey = "run-1:user", extraParts = listOf(imagePart)), - // Attachment-only messages have no cacheable text and are skipped entirely. + // Inline binary-only messages remain disposable and are skipped entirely. ChatMessage(id = "img", role = "user", content = listOf(imagePart), timestampMs = 11), + ChatMessage(id = "managed", role = "assistant", content = listOf(managedImage), timestampMs = 11), message("world", role = "assistant", timestampMs = 12), ), ) val loaded = store.loadTranscript("gateway-a", "main", "main") - assertEquals(listOf("hello", "world"), loaded.map { it.content.single().text }) - assertTrue(loaded.all { message -> message.content.all { part -> part.type == "text" && part.base64 == null } }) - assertEquals(listOf("user", "assistant"), loaded.map { it.role }) - assertEquals(listOf(10L, 12L), loaded.map { it.timestampMs }) - assertEquals(listOf("run-1:user", null), loaded.map { it.idempotencyKey }) + assertEquals(listOf("hello", null, "world"), loaded.map { it.content.single().text }) + assertTrue(loaded.all { message -> message.content.all { part -> part.base64 == null } }) + assertEquals(managedImage.artifactId, loaded[1].content.single().artifactId) + assertEquals(listOf("user", "assistant", "assistant"), loaded.map { it.role }) + assertEquals(listOf(10L, 11L, 12L), loaded.map { it.timestampMs }) + assertEquals(listOf("run-1:user", null, null), loaded.map { it.idempotencyKey }) + } + + @Test + fun legacyStringArrayTranscriptRowsRemainReadable() = + runTest { + database.dao().insertMessages( + listOf( + CachedMessageEntity( + gatewayId = "gateway-a", + agentId = "main", + sessionKey = "main", + rowOrder = 0, + role = "assistant", + textPartsJson = """["legacy one","legacy two"]""", + timestampMs = 10, + idempotencyKey = null, + ), + ), + ) + + val loaded = cache().loadTranscript("gateway-a", "main", "main").single() + + assertEquals(listOf("legacy one", "legacy two"), loaded.content.map { it.text }) } @Test diff --git a/apps/android/app/src/test/java/ai/openclaw/app/gateway/GatewaySessionCustomHeadersTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/gateway/GatewaySessionCustomHeadersTest.kt index 61847a5a5334..c0f5216c730a 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/gateway/GatewaySessionCustomHeadersTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/gateway/GatewaySessionCustomHeadersTest.kt @@ -19,6 +19,8 @@ import okhttp3.mockwebserver.Dispatcher import okhttp3.mockwebserver.MockResponse import okhttp3.mockwebserver.MockWebServer import okhttp3.mockwebserver.RecordedRequest +import okio.Buffer +import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals import org.junit.Assert.assertNull import org.junit.Assert.assertTrue @@ -60,6 +62,115 @@ private class NoopDeviceAuthStore : DeviceAuthTokenStore { @RunWith(RobolectricTestRunner::class) @Config(sdk = [34]) class GatewaySessionCustomHeadersTest { + @Test + fun managedImageDownload_usesArtifactTicketWithoutGatewayBearer() = + runBlocking { + val app = RuntimeEnvironment.getApplication() + val json = Json { ignoreUnknownKeys = true } + val connected = CompletableDeferred() + val imageRequest = CompletableDeferred() + val imageBytes = byteArrayOf(1, 2, 3, 4) + val attachmentId = "11111111-1111-4111-8111-111111111111" + val artifactId = "artifact_managed_image_$attachmentId" + val imagePath = "/api/chat/media/outgoing/main/$attachmentId/full?mediaTicket=ticket" + val server = + MockWebServer().apply { + dispatcher = + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + if (request.path == imagePath) { + imageRequest.complete(request) + return MockResponse() + .setHeader("Content-Type", "image/png") + .setBody(Buffer().write(imageBytes)) + } + return MockResponse().withWebSocketUpgrade( + object : WebSocketListener() { + override fun onOpen( + webSocket: WebSocket, + response: Response, + ) { + webSocket.send(CONNECT_CHALLENGE_FRAME) + } + + override fun onMessage( + webSocket: WebSocket, + text: String, + ) { + val frame = json.parseToJsonElement(text).jsonObject + if (frame["type"]?.jsonPrimitive?.content != "req") return + val id = frame["id"]?.jsonPrimitive?.content ?: return + when (frame["method"]?.jsonPrimitive?.content) { + "connect" -> + webSocket.send( + """{"type":"res","id":"$id","ok":true,"payload":{"snapshot":{"sessionDefaults":{"mainSessionKey":"main"}}}}""", + ) + "artifacts.download" -> + webSocket.send( + """{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""", + ) + } + } + }, + ) + } + } + start() + } + val stableId = "manual|127.0.0.1|${server.port}" + val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + val session = + GatewaySession( + scope = scope, + identityStore = testDeviceIdentityStore(app), + deviceAuthStore = NoopDeviceAuthStore(), + onConnected = { if (!connected.isCompleted) connected.complete(Unit) }, + onDisconnected = {}, + onEvent = { _, _ -> }, + ) + + try { + session.connect( + endpoint = GatewayEndpoint(stableId, "test", "127.0.0.1", server.port, tlsEnabled = false), + token = "bootstrap-token", + bootstrapToken = null, + password = null, + options = + GatewayConnectOptions( + role = "operator", + scopes = listOf("operator.read"), + caps = emptyList(), + commands = emptyList(), + permissions = emptyMap(), + client = + GatewayClientInfo( + id = "openclaw-android-test", + displayName = "Android Test", + version = "1.0.0-test", + platform = "android", + mode = "ui", + instanceId = "android-test-instance", + deviceFamily = "android", + modelIdentifier = "test", + ), + ), + tls = null, + ) + withTimeout(TEST_TIMEOUT_MS) { connected.await() } + + val loaded = session.loadImageArtifact(stableId, "main", "main", artifactId) + assertArrayEquals(imageBytes, loaded?.bytes) + assertEquals("image/png", loaded?.mimeType) + val request = withTimeout(TEST_TIMEOUT_MS) { imageRequest.await() } + assertNull(request.getHeader("Authorization")) + assertEquals("image/*", request.getHeader("Accept")) + } finally { + session.disconnectAndJoin() + scope.cancel() + server.shutdown() + } + } + @Test fun tlsUpgradeRequest_carriesLatestSanitizedHeadersForOnlyThisGateway() { val app = RuntimeEnvironment.getApplication() diff --git a/apps/android/app/src/test/java/ai/openclaw/app/ui/chat/ChatMessageViewsTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/ui/chat/ChatMessageViewsTest.kt new file mode 100644 index 000000000000..8a352500aac6 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/app/ui/chat/ChatMessageViewsTest.kt @@ -0,0 +1,56 @@ +package ai.openclaw.app.ui.chat + +import ai.openclaw.app.chat.ChatMessage +import ai.openclaw.app.chat.ChatMessageContent +import android.os.Looper +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf + +@RunWith(RobolectricTestRunner::class) +class ChatMessageViewsTest { + @Test + fun managedImageCompositionRequestsItsArtifact() { + val artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111" + val requested = mutableListOf() + val controller = Robolectric.buildActivity(ComponentActivity::class.java).setup() + + try { + controller.get().setContent { + ChatMessageBubble( + message = + ChatMessage( + id = "managed-image", + role = "assistant", + content = + listOf( + ChatMessageContent( + type = "image", + mimeType = "image/png", + artifactId = artifactId, + alt = "Managed image", + ), + ), + timestampMs = 1, + ), + imageResolverReady = true, + loadImageArtifact = { requestedArtifactId -> + requested += requestedArtifactId + null + }, + ) + } + shadowOf(Looper.getMainLooper()).idle() + + assertEquals(listOf(artifactId), requested) + } finally { + controller.pause().stop().destroy() + shadowOf(Looper.getMainLooper()).idle() + } + } +} diff --git a/apps/ios/Sources/Chat/IOSGatewayChatTransport.swift b/apps/ios/Sources/Chat/IOSGatewayChatTransport.swift index 253872d3415b..dad97033bfe1 100644 --- a/apps/ios/Sources/Chat/IOSGatewayChatTransport.swift +++ b/apps/ios/Sources/Chat/IOSGatewayChatTransport.swift @@ -11,6 +11,7 @@ struct IOSGatewayChatTransport: OpenClawChatTransport { private let globalAgentId: String? private let outboxGatewayID: String? private let sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? + private let imageArtifactLoader: IOSImageArtifactLoader? var outboxRequiresSessionRoutingContract: Bool { true @@ -21,7 +22,8 @@ struct IOSGatewayChatTransport: OpenClawChatTransport { widgetGateway: GatewayNodeSession? = nil, globalAgentId: String? = nil, outboxGatewayID: String? = nil, - sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? = nil) + sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? = nil, + imageArtifactLoader: IOSImageArtifactLoader? = nil) { self.gateway = gateway self.widgetGateway = widgetGateway @@ -30,6 +32,7 @@ struct IOSGatewayChatTransport: OpenClawChatTransport { let normalizedGatewayID = outboxGatewayID?.trimmingCharacters(in: .whitespacesAndNewlines) self.outboxGatewayID = normalizedGatewayID?.isEmpty == false ? normalizedGatewayID : nil self.sessionMutationRequest = sessionMutationRequest + self.imageArtifactLoader = imageArtifactLoader } func acquireOutboxRouteLease() async -> OpenClawChatTransportRouteLeaseResult { @@ -562,6 +565,31 @@ struct IOSGatewayChatTransport: OpenClawChatTransport { }) } + func loadImageArtifact( + sessionKey: String, + artifactId: String) async throws -> OpenClawChatLoadedImage? + { + guard let imageArtifactLoader, + let route = await gateway.currentRoute(), + let gatewayID = await gateway.currentGatewayID(ifCurrentRoute: route) + else { return nil } + let target = self.sessionTarget(for: sessionKey) + let request = OpenClawChatGatewayRequests.artifactDownload( + sessionKey: target.sessionKey, + agentID: target.agentID, + artifactId: artifactId) + let data = try await gateway.request(request, ifCurrentRoute: route) + let response = try JSONDecoder().decode(ArtifactsDownloadResult.self, from: data) + guard let url = response.url?.trimmingCharacters(in: .whitespacesAndNewlines), !url.isEmpty + else { return nil } + guard await self.gateway.currentRoute() == route else { throw CancellationError() } + let loaded = try await imageArtifactLoader.load( + ticketedPath: url, + expectedGatewayID: gatewayID) + guard await self.gateway.currentRoute() == route else { throw CancellationError() } + return loaded + } + func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL? { await self.resolveInlineWidgetResource( path: path, diff --git a/apps/ios/Sources/Chat/IOSImageArtifactLoader.swift b/apps/ios/Sources/Chat/IOSImageArtifactLoader.swift new file mode 100644 index 000000000000..3a179a9378e0 --- /dev/null +++ b/apps/ios/Sources/Chat/IOSImageArtifactLoader.swift @@ -0,0 +1,111 @@ +import Foundation +import OpenClawChatUI +import OpenClawKit + +struct IOSImageArtifactLoader: Sendable { + struct Connection: Sendable { + let config: GatewayConnectConfig + let gatewayID: String + let customHeaders: [String: String] + } + + enum LoadError: Error, Equatable { + case invalidSource + case invalidResponse + case requestFailed(statusCode: Int) + case unsupportedMediaType + case payloadTooLarge + } + + typealias Request = @Sendable (URLRequest) async throws -> (Data, URLResponse) + typealias RequestFactory = @Sendable (GatewayTLSParams, Int) -> Request + typealias ConnectionProvider = @MainActor @Sendable () -> Connection? + + static let maximumImageBytes = 12 * 1024 * 1024 + private static let managedImagePathPrefix = "/api/chat/media/outgoing/" + private let connectionProvider: ConnectionProvider + private let requestFactory: RequestFactory + + init(connectionProvider: @escaping ConnectionProvider) { + self.init(connectionProvider: connectionProvider) { tls, maximumBytes in + let session = GatewayTLSPinningSession(params: tls) + return { request in + defer { session.finishTasksAndInvalidate() } + return try await session.data(for: request, maximumBytes: maximumBytes) + } + } + } + + init( + connectionProvider: @escaping ConnectionProvider, + requestFactory: @escaping RequestFactory) + { + self.connectionProvider = connectionProvider + self.requestFactory = requestFactory + } + + func load( + ticketedPath rawPath: String, + expectedGatewayID: String) async throws -> OpenClawChatLoadedImage + { + let path = rawPath.trimmingCharacters(in: .whitespacesAndNewlines) + guard let connection = await self.connectionProvider(), + connection.gatewayID == expectedGatewayID, + let url = Self.managedImageURL(config: connection.config, path: path) + else { throw LoadError.invalidSource } + + var request = URLRequest(url: url) + request.timeoutInterval = 20 + request.setValue("image/*", forHTTPHeaderField: "Accept") + if url.scheme?.lowercased() == "https" { + for (name, value) in GatewayCustomHeaders.sanitized(connection.customHeaders) { + request.setValue(value, forHTTPHeaderField: name) + } + } + let tls = connection.config.tls ?? GatewayTLSParams( + required: false, + expectedFingerprint: nil, + allowTOFU: false, + storeKey: nil) + let data: Data + let response: URLResponse + do { + (data, response) = try await self.requestFactory(tls, Self.maximumImageBytes)(request) + } catch is GatewayBoundedDataError { + throw LoadError.payloadTooLarge + } + guard let http = response as? HTTPURLResponse else { throw LoadError.invalidResponse } + guard (200..<300).contains(http.statusCode) else { + throw LoadError.requestFailed(statusCode: http.statusCode) + } + guard let mimeType = http.mimeType?.lowercased(), mimeType.hasPrefix("image/") else { + throw LoadError.unsupportedMediaType + } + guard data.count <= Self.maximumImageBytes else { throw LoadError.payloadTooLarge } + return OpenClawChatLoadedImage(data: data, mimeType: mimeType) + } + + private static func managedImageURL(config: GatewayConnectConfig, path: String) -> URL? { + guard path.hasPrefix(self.managedImagePathPrefix), + let relative = URLComponents(string: path), + relative.scheme == nil, + relative.host == nil, + relative.fragment == nil, + relative.percentEncodedPath.hasPrefix(Self.managedImagePathPrefix), + relative.queryItems?.contains(where: { + $0.name == "mediaTicket" && $0.value?.isEmpty == false + }) == true, + var base = URLComponents(url: config.url, resolvingAgainstBaseURL: false), + base.host != nil + else { return nil } + switch base.scheme?.lowercased() { + case "wss", "https": base.scheme = "https" + case "ws", "http": base.scheme = "http" + default: return nil + } + base.percentEncodedPath = relative.percentEncodedPath + base.percentEncodedQuery = relative.percentEncodedQuery + base.fragment = nil + return base.url + } +} diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift index ee73dee33ae9..a9925dbf58ed 100644 --- a/apps/ios/Sources/Model/NodeAppModel.swift +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -619,11 +619,20 @@ final class NodeAppModel { if self.isAppleReviewDemoModeEnabled { return AppleReviewDemoChatTransport() } + let imageArtifactLoader = IOSImageArtifactLoader { [weak self] in + guard let config = self?.activeGatewayConnectConfig else { return nil } + return IOSImageArtifactLoader.Connection( + config: config, + gatewayID: config.nodeOptions.deviceAuthGatewayID ?? config.effectiveStableID, + customHeaders: GatewaySettingsStore.loadGatewayCustomHeaders( + gatewayStableID: config.effectiveStableID)) + } return IOSGatewayChatTransport( gateway: self.operatorSession, widgetGateway: self.nodeGateway, globalAgentId: self.chatDeliveryAgentId, - outboxGatewayID: outboxGatewayID) + outboxGatewayID: outboxGatewayID, + imageArtifactLoader: imageArtifactLoader) } /// Gateway identity the transcript cache is scoped to: the active diff --git a/apps/ios/Tests/IOSImageArtifactLoaderTests.swift b/apps/ios/Tests/IOSImageArtifactLoaderTests.swift new file mode 100644 index 000000000000..e0aa77016221 --- /dev/null +++ b/apps/ios/Tests/IOSImageArtifactLoaderTests.swift @@ -0,0 +1,91 @@ +import Foundation +import OpenClawKit +import Testing +@testable import OpenClaw + +@Suite("iOS managed image artifact loader") +struct IOSImageArtifactLoaderTests { + @Test @MainActor func `loads ticketed image with proxy headers and without a gateway bearer`() async throws { + let gatewayURL = try #require(URL(string: "wss://gateway.example")) + let config = Self.config(url: gatewayURL) + let loader = IOSImageArtifactLoader( + connectionProvider: { + IOSImageArtifactLoader.Connection( + config: config, + gatewayID: config.effectiveStableID, + customHeaders: ["X-Proxy-Token": "proxy"]) + }, + requestFactory: { _, maximumBytes in + #expect(maximumBytes == 12 * 1024 * 1024) + return { request in + #expect(request.url?.absoluteString == + "https://gateway.example/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full?mediaTicket=ticket") + #expect(request.value(forHTTPHeaderField: "Authorization") == nil) + #expect(request.value(forHTTPHeaderField: "X-Proxy-Token") == "proxy") + let responseURL = try #require(request.url) + let response = try #require(HTTPURLResponse( + url: responseURL, + statusCode: 200, + httpVersion: nil, + headerFields: ["Content-Type": "image/png"])) + return (Data([1, 2, 3]), response) + } + }) + + let loaded = try await loader.load( + ticketedPath: + "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full?mediaTicket=ticket", + expectedGatewayID: config.effectiveStableID) + + #expect(loaded.data == Data([1, 2, 3])) + #expect(loaded.mimeType == "image/png") + } + + @Test @MainActor func `rejects absolute and unticketed paths before fetching`() async { + let config = Self.config() + let loader = IOSImageArtifactLoader( + connectionProvider: { + IOSImageArtifactLoader.Connection( + config: config, + gatewayID: config.effectiveStableID, + customHeaders: [:]) + }, + requestFactory: { _, _ in + Issue.record("invalid paths must not reach the network") + return { _ in throw CancellationError() } + }) + + await #expect(throws: IOSImageArtifactLoader.LoadError.invalidSource) { + try await loader.load( + ticketedPath: "https://example.com/image.png?mediaTicket=ticket", + expectedGatewayID: config.effectiveStableID) + } + await #expect(throws: IOSImageArtifactLoader.LoadError.invalidSource) { + try await loader.load( + ticketedPath: + "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full", + expectedGatewayID: config.effectiveStableID) + } + } + + private static func config( + url: URL = URL(string: "ws://127.0.0.1:18789")!) -> GatewayConnectConfig + { + GatewayConnectConfig( + url: url, + stableID: "manual|127.0.0.1|18789", + tls: nil, + token: nil, + bootstrapToken: nil, + password: nil, + nodeOptions: GatewayConnectOptions( + role: "node", + scopes: [], + caps: [], + commands: [], + permissions: [:], + clientId: "ios", + clientMode: "node", + clientDisplayName: "Phone")) + } +} diff --git a/apps/ios/Tests/SwiftUIRenderSmokeTests.swift b/apps/ios/Tests/SwiftUIRenderSmokeTests.swift index fdd7ca204ce5..951d60bac16b 100644 --- a/apps/ios/Tests/SwiftUIRenderSmokeTests.swift +++ b/apps/ios/Tests/SwiftUIRenderSmokeTests.swift @@ -269,13 +269,65 @@ struct SwiftUIRenderSmokeTests { userMessageExpanded: false, onToggleUserMessageExpanded: {}, inlineWidgetResolverReady: true, - inlineWidgetResourceResolver: { _, _ in nil }) + inlineWidgetResourceResolver: { _, _ in nil }, + imageArtifactResolverReady: false, + loadImageArtifact: { _ in nil }) .environment(\.dynamicTypeSize, typeSize) _ = Self.host(root, size: CGSize(width: 320, height: 420)) } } + @Test @MainActor func `managed assistant image starts its artifact load`() async throws { + let artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111" + let message = OpenClawChatMessage( + role: "assistant", + content: [OpenClawChatMessageContent( + type: "image", + text: nil, + mimeType: "image/png", + fileName: nil, + artifactId: artifactId, + url: "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full", + alt: "Managed preview", + content: nil)], + timestamp: 1) + var requestedArtifactId: String? + let root = ChatMessageBubble( + message: message, + style: .standard, + markdownVariant: .standard, + userAccent: nil, + displayOptions: [], + assistantName: "OpenClaw", + assistantAvatarText: "OC", + assistantAvatarTint: nil, + showsAssistantAvatar: true, + isClean: false, + contextWindowTokens: nil, + userMessageExpanded: false, + onToggleUserMessageExpanded: {}, + inlineWidgetResolverReady: true, + inlineWidgetResourceResolver: { _, _ in nil }, + imageArtifactResolverReady: true, + loadImageArtifact: { requested in + requestedArtifactId = requested + return OpenClawChatLoadedImage( + data: Data(base64Encoded: + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=")!, + mimeType: "image/png") + }) + let window = Self.host(root, size: CGSize(width: 393, height: 420)) + defer { window.isHidden = true } + + let deadline = ContinuousClock().now.advanced(by: .seconds(2)) + while requestedArtifactId == nil, ContinuousClock().now < deadline { + try await Task.sleep(for: .milliseconds(10)) + } + + #expect(requestedArtifactId == artifactId) + } + @Test @MainActor func `streaming assistant bubble builds mixed prose and code`() { let text = """ Earlier prose stays visible. @@ -336,7 +388,9 @@ struct SwiftUIRenderSmokeTests { userMessageExpanded: false, onToggleUserMessageExpanded: {}, inlineWidgetResolverReady: true, - inlineWidgetResourceResolver: { _, _ in nil }) + inlineWidgetResourceResolver: { _, _ in nil }, + imageArtifactResolverReady: false, + loadImageArtifact: { _ in nil }) ChatStreamingAssistantBubble( text: text, markdownVariant: .standard, @@ -390,7 +444,9 @@ struct SwiftUIRenderSmokeTests { userMessageExpanded: false, onToggleUserMessageExpanded: {}, inlineWidgetResolverReady: true, - inlineWidgetResourceResolver: { _, _ in nil }) + inlineWidgetResourceResolver: { _, _ in nil }, + imageArtifactResolverReady: false, + loadImageArtifact: { _ in nil }) .environment(\.dynamicTypeSize, typeSize) _ = Self.host(root, size: CGSize(width: 320, height: 280)) diff --git a/apps/macos/Sources/OpenClaw/GatewayConnection+ManagedImages.swift b/apps/macos/Sources/OpenClaw/GatewayConnection+ManagedImages.swift new file mode 100644 index 000000000000..cd432ea39077 --- /dev/null +++ b/apps/macos/Sources/OpenClaw/GatewayConnection+ManagedImages.swift @@ -0,0 +1,77 @@ +import Foundation +import OpenClawChatUI +import OpenClawKit +import OpenClawProtocol + +private let gatewayManagedImagePathPrefix = "/api/chat/media/outgoing/" + +extension GatewayConnection { + func loadImageArtifact( + sessionKey: String, + agentID: String?, + artifactId: String, + ifCurrentServerLease lease: ServerLease) async throws -> OpenClawChatLoadedImage? + { + let request = OpenClawChatGatewayRequests.artifactDownload( + sessionKey: sessionKey, + agentID: agentID, + artifactId: artifactId) + let responseData = try await self.request( + method: request.method, + params: request.params, + timeoutMs: request.timeoutMs, + ifCurrentServerLease: lease) + let response = try JSONDecoder().decode(ArtifactsDownloadResult.self, from: responseData) + guard let ticketedPath = response.url?.trimmingCharacters(in: .whitespacesAndNewlines), + let url = Self.managedImageURL(gatewayURL: lease.route.url, ticketedPath: ticketedPath) + else { return nil } + + var urlRequest = URLRequest(url: url) + urlRequest.timeoutInterval = 20 + urlRequest.setValue("image/*", forHTTPHeaderField: "Accept") + // Native macOS has no per-Gateway proxy-header configuration surface today. If one is + // added, carry its immutable snapshot on Route so the socket and ticket GET cannot diverge. + let tls = lease.route.tls?.params ?? GatewayTLSParams( + required: false, + expectedFingerprint: nil, + allowTOFU: false, + storeKey: nil) + let session = GatewayTLSPinningSession(params: tls) + defer { session.finishTasksAndInvalidate() } + let (data, urlResponse) = try await session.data( + for: urlRequest, + maximumBytes: 12 * 1024 * 1024) + guard await self.isCurrentServerLease(lease) else { + throw OpenClawChatTransportSendError.notDispatched + } + guard let http = urlResponse as? HTTPURLResponse, + (200..<300).contains(http.statusCode), + let mimeType = http.mimeType?.lowercased(), + mimeType.hasPrefix("image/") + else { return nil } + return OpenClawChatLoadedImage(data: data, mimeType: mimeType) + } + + private static func managedImageURL(gatewayURL: URL, ticketedPath: String) -> URL? { + guard ticketedPath.hasPrefix(gatewayManagedImagePathPrefix), + let relative = URLComponents(string: ticketedPath), + relative.scheme == nil, + relative.host == nil, + relative.fragment == nil, + relative.queryItems?.contains(where: { + $0.name == "mediaTicket" && $0.value?.isEmpty == false + }) == true, + var base = URLComponents(url: gatewayURL, resolvingAgainstBaseURL: false), + base.host != nil + else { return nil } + switch base.scheme?.lowercased() { + case "wss", "https": base.scheme = "https" + case "ws", "http": base.scheme = "http" + default: return nil + } + base.percentEncodedPath = relative.percentEncodedPath + base.percentEncodedQuery = relative.percentEncodedQuery + base.fragment = nil + return base.url + } +} diff --git a/apps/macos/Sources/OpenClaw/GatewayConnection.swift b/apps/macos/Sources/OpenClaw/GatewayConnection.swift index ba19c2b2dd15..a668793c603f 100644 --- a/apps/macos/Sources/OpenClaw/GatewayConnection.swift +++ b/apps/macos/Sources/OpenClaw/GatewayConnection.swift @@ -47,10 +47,10 @@ actor GatewayConnection { struct Route: Equatable, Sendable { fileprivate let generation: UInt64 fileprivate let authority: UInt64? - fileprivate let url: URL + let url: URL fileprivate let token: String? fileprivate let password: String? - fileprivate let tls: GatewayTLSRoute? + let tls: GatewayTLSRoute? fileprivate let deviceAuthGatewayID: String? let activationOwnershipFingerprint: String? @@ -67,7 +67,9 @@ actor GatewayConnection { /// One connected Gateway server, not merely an endpoint configuration. /// A reconnect at the same URL creates a different lease. struct ServerLease: Sendable { - fileprivate let route: Route + // Managed-image HTTP reuses this captured route from its focused extension file. + // Carrying the snapshot forward prevents endpoint or TLS rediscovery after suspension. + let route: Route fileprivate let socketGeneration: UInt64 fileprivate let client: GatewayChannelActor } diff --git a/apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift b/apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift index a952817b0ec9..3f4b3bed393b 100644 --- a/apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift +++ b/apps/macos/Sources/OpenClaw/WebChatSwiftUI.swift @@ -566,6 +566,21 @@ struct MacGatewayChatTransport: OpenClawChatTransport { connection: self.connection) } + func loadImageArtifact( + sessionKey: String, + artifactId: String) async throws -> OpenClawChatLoadedImage? + { + guard let serverLease = await connection.captureServerLease() else { + throw OpenClawChatTransportSendError.notDispatched + } + let target = self.sessionTarget(for: sessionKey) + return try await self.connection.loadImageArtifact( + sessionKey: target.sessionKey, + agentID: target.agentID, + artifactId: artifactId, + ifCurrentServerLease: serverLease) + } + var supportsSlashCommandCatalog: Bool { true } diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatGatewayRequest.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatGatewayRequest.swift index bf795f6c8c56..e83e5789c184 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatGatewayRequest.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatGatewayRequest.swift @@ -80,6 +80,22 @@ public enum OpenClawChatGatewayRequests { OpenClawChatGatewayRequest(method: "models.list", timeoutMs: self.defaultTimeoutMs) } + public static func artifactDownload( + sessionKey: String, + agentID: String?, + artifactId: String) -> OpenClawChatGatewayRequest + { + var params: [String: AnyCodable] = [ + "sessionKey": AnyCodable(sessionKey), + "artifactId": AnyCodable(artifactId), + ] + self.add(agentID, to: ¶ms, key: "agentId") + return OpenClawChatGatewayRequest( + method: "artifacts.download", + params: params, + timeoutMs: self.defaultTimeoutMs) + } + public static func chatMetadata( sessionKey: String, fallbackAgentID: String?) -> OpenClawChatGatewayRequest diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift new file mode 100644 index 000000000000..cec221a0fdf9 --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMediaImageAttachment.swift @@ -0,0 +1,149 @@ +import Foundation +import ImageIO +import SwiftUI + +#if canImport(AppKit) +import AppKit +#elseif canImport(UIKit) +import UIKit +#endif + +enum ChatMediaImageDecoder { + static let maximumThumbnailPixels = 2048 + + static func decode(_ data: Data) -> OpenClawPlatformImage? { + guard let source = CGImageSourceCreateWithData(data as CFData, nil), + CGImageSourceGetCount(source) > 0 + else { return nil } + let options: [CFString: Any] = [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceCreateThumbnailWithTransform: true, + kCGImageSourceThumbnailMaxPixelSize: self.maximumThumbnailPixels, + kCGImageSourceShouldCacheImmediately: true, + ] + guard let cgImage = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) + else { return nil } + #if canImport(AppKit) + return NSImage(cgImage: cgImage, size: .zero) + #elseif canImport(UIKit) + return UIImage(cgImage: cgImage) + #endif + } +} + +@MainActor +struct ChatMediaImageAttachment: View { + private enum LoadState { + case loading + case loaded(OpenClawPlatformImage) + case unavailable + } + + let artifactId: String + let label: String + let resolverReady: Bool + let load: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage? + + @State private var state: LoadState = .loading + @State private var retryGeneration = 0 + @State private var showsFullImage = false + + var body: some View { + Group { + switch self.state { + case .loading: + HStack(spacing: 8) { + ProgressView() + Text(String(localized: "Loading image…")) + .font(OpenClawChatTypography.footnote) + .foregroundStyle(.secondary) + } + .frame(minHeight: 88) + .frame(maxWidth: .infinity) + case let .loaded(image): + Button { + self.showsFullImage = true + } label: { + OpenClawPlatformImageFactory.image(image) + .resizable() + .scaledToFit() + .frame(maxHeight: 320) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(Color.white.opacity(0.12), lineWidth: 1)) + } + .buttonStyle(.plain) + .accessibilityLabel(self.label) + .accessibilityHint(String(localized: "Opens image preview")) + .sheet(isPresented: self.$showsFullImage) { + ZStack(alignment: .topTrailing) { + Color.black.ignoresSafeArea() + ScrollView([.horizontal, .vertical]) { + OpenClawPlatformImageFactory.image(image) + .resizable() + .scaledToFit() + .padding(20) + } + Button { + self.showsFullImage = false + } label: { + Image(systemName: "xmark.circle.fill") + .font(.title2) + .symbolRenderingMode(.hierarchical) + } + .buttonStyle(.plain) + .foregroundStyle(.white) + .padding(16) + .accessibilityLabel(String(localized: "Close image preview")) + } + } + case .unavailable: + HStack(spacing: 8) { + Image(systemName: "photo.badge.exclamationmark") + Text(String(localized: "Image unavailable")) + .font(OpenClawChatTypography.footnote) + Spacer() + Button { + self.retryGeneration &+= 1 + } label: { + Text(String(localized: "Retry")) + .font(OpenClawChatTypography.footnote) + } + .buttonStyle(.plain) + } + .foregroundStyle(.secondary) + .padding(10) + .background(Color.black.opacity(0.04)) + .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) + } + } + .task(id: "\(self.artifactId):\(self.resolverReady):\(self.retryGeneration)") { + await self.loadImage() + } + } + + private func loadImage() async { + guard self.resolverReady else { + self.state = .unavailable + return + } + self.state = .loading + do { + guard let loaded = try await self.load(self.artifactId), !Task.isCancelled else { + if !Task.isCancelled { self.state = .unavailable } + return + } + let image = await Task.detached(priority: .userInitiated) { + ChatMediaImageDecoder.decode(loaded.data) + }.value + guard !Task.isCancelled else { return } + self.state = image.map(LoadState.loaded) ?? .unavailable + } catch is CancellationError { + return + } catch { + guard !Task.isCancelled else { return } + self.state = .unavailable + } + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift index 618c3075ef53..9de95d3a3172 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMessageViews.swift @@ -219,6 +219,8 @@ struct ChatMessageBubble: View { let inlineWidgetResourceResolver: @MainActor @Sendable ( String, OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource? + let imageArtifactResolverReady: Bool + let loadImageArtifact: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage? var body: some View { if self.isUser { @@ -261,7 +263,9 @@ struct ChatMessageBubble: View { userMessageExpanded: self.userMessageExpanded, onToggleUserMessageExpanded: self.onToggleUserMessageExpanded, inlineWidgetResolverReady: self.inlineWidgetResolverReady, - inlineWidgetResourceResolver: self.inlineWidgetResourceResolver) + inlineWidgetResourceResolver: self.inlineWidgetResourceResolver, + imageArtifactResolverReady: self.imageArtifactResolverReady, + loadImageArtifact: self.loadImageArtifact) } } @@ -314,6 +318,8 @@ private struct ChatMessageBody: View { let inlineWidgetResourceResolver: @MainActor @Sendable ( String, OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource? + let imageArtifactResolverReady: Bool + let loadImageArtifact: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage? var body: some View { let text = self.primaryText @@ -370,12 +376,24 @@ private struct ChatMessageBody: View { ChatLinkPreview(url: previewURL) } - if !self.inlineAttachments.isEmpty { - ForEach(self.inlineAttachments.indices, id: \.self) { idx in - AttachmentRow(att: self.inlineAttachments[idx], isUser: self.isUser) + if !self.visibleInlineAttachments.isEmpty { + ForEach(self.visibleInlineAttachments.indices, id: \.self) { idx in + AttachmentRow( + att: self.visibleInlineAttachments[idx], + isUser: self.isUser, + resolverReady: self.imageArtifactResolverReady, + loadImage: self.loadImageArtifact) } } + if self.omittedImageAttachmentCount > 0 { + Text(String( + format: String(localized: "Additional images hidden: %lld"), + Int64(self.omittedImageAttachmentCount))) + .font(OpenClawChatTypography.footnote) + .foregroundStyle(.secondary) + } + ForEach(self.inlineWidgets.indices, id: \.self) { idx in ChatInlineWidgetView( preview: self.inlineWidgets[idx], @@ -503,6 +521,19 @@ private struct ChatMessageBody: View { self.message.content.filter(\.isInlineAttachment) } + private var visibleInlineAttachments: [OpenClawChatMessageContent] { + var imageCount = 0 + return self.inlineAttachments.filter { attachment in + guard attachment.isImageAttachment else { return true } + defer { imageCount += 1 } + return imageCount < 4 + } + } + + private var omittedImageAttachmentCount: Int { + max(0, self.inlineAttachments.filter(\.isImageAttachment).count - 4) + } + private var inlineWidgets: [OpenClawChatCanvasPreview] { guard self.message.role.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "assistant" else { return [] } @@ -620,11 +651,25 @@ private struct ChatMessageBody: View { private struct AttachmentRow: View { let att: OpenClawChatMessageContent let isUser: Bool + let resolverReady: Bool + let loadImage: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage? var body: some View { + if self.att.isImageAttachment, let artifactId = self.normalizedArtifactId { + ChatMediaImageAttachment( + artifactId: artifactId, + label: self.attachmentLabel, + resolverReady: self.resolverReady, + load: self.loadImage) + } else { + self.fallbackRow + } + } + + private var fallbackRow: some View { HStack(spacing: 8) { Image(systemName: self.isAudio ? "waveform" : "paperclip") - Text(self.isAudio ? "Voice note" : (self.att.fileName ?? "Attachment")) + Text(self.isAudio ? "Voice note" : self.attachmentLabel) .font(OpenClawChatTypography.footnote) .lineLimit(1) .foregroundStyle(self.isUser ? OpenClawChatTheme.userText : OpenClawChatTheme.assistantText) @@ -646,6 +691,26 @@ private struct AttachmentRow: View { private var isAudio: Bool { self.att.mimeType?.hasPrefix("audio/") == true } + + private var normalizedArtifactId: String? { + let value = self.att.artifactId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return value.isEmpty ? nil : value + } + + private var attachmentLabel: String { + let values = [self.att.alt, self.att.fileName] + return values.lazy + .compactMap { $0?.trimmingCharacters(in: .whitespacesAndNewlines) } + .first { !$0.isEmpty } ?? String(localized: "Attachment") + } +} + +extension OpenClawChatMessageContent { + fileprivate var isImageAttachment: Bool { + self.type?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "image" || + self.mimeType?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + .hasPrefix("image/") == true + } } @MainActor diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift index eac1bbca672c..22b4577c9d66 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatModels.swift @@ -116,6 +116,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { public let thinkingSignature: String? public let mimeType: String? public let fileName: String? + public let artifactId: String? + public let url: String? + public let openUrl: String? + public let alt: String? + public let width: Int? + public let height: Int? + public let sizeBytes: Int? public let durationSeconds: Double? public let content: AnyCodable? public let preview: OpenClawChatCanvasPreview? @@ -144,6 +151,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { thinkingSignature: String? = nil, mimeType: String?, fileName: String?, + artifactId: String? = nil, + url: String? = nil, + openUrl: String? = nil, + alt: String? = nil, + width: Int? = nil, + height: Int? = nil, + sizeBytes: Int? = nil, durationSeconds: Double? = nil, content: AnyCodable?, preview: OpenClawChatCanvasPreview? = nil, @@ -159,6 +173,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { self.thinkingSignature = thinkingSignature self.mimeType = mimeType self.fileName = fileName + self.artifactId = artifactId + self.url = url + self.openUrl = openUrl + self.alt = alt + self.width = width + self.height = height + self.sizeBytes = sizeBytes self.durationSeconds = durationSeconds self.content = content self.preview = preview @@ -176,6 +197,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { case thinkingSignature case mimeType case fileName + case artifactId + case url + case openUrl + case alt + case width + case height + case sizeBytes case durationSeconds case content case preview @@ -195,6 +223,15 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { self.thinkingSignature = try container.decodeIfPresent(String.self, forKey: .thinkingSignature) self.mimeType = try container.decodeIfPresent(String.self, forKey: .mimeType) self.fileName = try container.decodeIfPresent(String.self, forKey: .fileName) + let decodedURL = try container.decodeIfPresent(String.self, forKey: .url) + self.url = decodedURL + self.openUrl = try container.decodeIfPresent(String.self, forKey: .openUrl) + self.artifactId = try container.decodeIfPresent(String.self, forKey: .artifactId) + ?? Self.managedImageArtifactId(from: decodedURL) + self.alt = try container.decodeIfPresent(String.self, forKey: .alt) + self.width = try container.decodeIfPresent(Int.self, forKey: .width) + self.height = try container.decodeIfPresent(Int.self, forKey: .height) + self.sizeBytes = try container.decodeIfPresent(Int.self, forKey: .sizeBytes) self.durationSeconds = try container.decodeIfPresent(Double.self, forKey: .durationSeconds) self.id = try container.decodeIfPresent(String.self, forKey: .id) self.name = try container.decodeIfPresent(String.self, forKey: .name) @@ -221,6 +258,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { try container.encodeIfPresent(self.thinkingSignature, forKey: .thinkingSignature) try container.encodeIfPresent(self.mimeType, forKey: .mimeType) try container.encodeIfPresent(self.fileName, forKey: .fileName) + try container.encodeIfPresent(self.artifactId, forKey: .artifactId) + try container.encodeIfPresent(self.url, forKey: .url) + try container.encodeIfPresent(self.openUrl, forKey: .openUrl) + try container.encodeIfPresent(self.alt, forKey: .alt) + try container.encodeIfPresent(self.width, forKey: .width) + try container.encodeIfPresent(self.height, forKey: .height) + try container.encodeIfPresent(self.sizeBytes, forKey: .sizeBytes) try container.encodeIfPresent(self.durationSeconds, forKey: .durationSeconds) try container.encodeIfPresent(self.content, forKey: .content) try container.encodeIfPresent(self.preview, forKey: .preview) @@ -230,6 +274,21 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable { try container.encodeIfPresent(self.details, forKey: .details) try container.encodeIfPresent(self.isError, forKey: .isError) } + + private static func managedImageArtifactId(from rawURL: String?) -> String? { + guard let rawURL, + let components = URLComponents(string: rawURL), + components.scheme == nil, + components.host == nil + else { return nil } + let segments = components.percentEncodedPath.split(separator: "/", omittingEmptySubsequences: true) + guard segments.count == 7, + segments[0...3] == ["api", "chat", "media", "outgoing"], + segments[6] == "full", + let attachmentId = UUID(uuidString: String(segments[5]))?.uuidString.lowercased() + else { return nil } + return "artifact_managed_image_\(attachmentId)" + } } public struct OpenClawChatCanvasPreview: Codable, Hashable, Sendable { diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTranscriptCache.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTranscriptCache.swift index a539b8cea200..75de226d2b9d 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTranscriptCache.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTranscriptCache.swift @@ -1405,6 +1405,13 @@ extension OpenClawChatSQLiteTranscriptCache { thinkingSignature: nil, mimeType: item.mimeType, fileName: item.fileName, + artifactId: item.artifactId, + url: item.url, + openUrl: item.openUrl, + alt: item.alt, + width: item.width, + height: item.height, + sizeBytes: item.sizeBytes, durationSeconds: item.durationSeconds, content: nil, id: item.id, diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift index 084109ac115a..8c9e8900cfe4 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatTransport.swift @@ -548,6 +548,16 @@ public struct OpenClawChatMetadataCapabilities: Codable, Sendable, Equatable { } } +public struct OpenClawChatLoadedImage: Sendable { + public let data: Data + public let mimeType: String + + public init(data: Data, mimeType: String) { + self.data = data + self.mimeType = mimeType + } +} + /// One physical Gateway route for Swarm capability discovery and child paging. /// All pages use the captured route so a reconnect cannot combine two servers. public struct OpenClawChatSwarmRouteLease: Sendable { @@ -671,6 +681,7 @@ public protocol OpenClawChatTransport: Sendable { path: String, replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource? func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL? + func loadImageArtifact(sessionKey: String, artifactId: String) async throws -> OpenClawChatLoadedImage? func setActiveSessionKey(_ sessionKey: String) async throws func resetSession(sessionKey: String) async throws @@ -678,6 +689,13 @@ public protocol OpenClawChatTransport: Sendable { } extension OpenClawChatTransport { + public func loadImageArtifact( + sessionKey _: String, + artifactId _: String) async throws -> OpenClawChatLoadedImage? + { + nil + } + public func isSwarmEnabled(sessionKey _: String) async throws -> Bool { false } diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift index a7e86edfe41b..3311430d0359 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatView.swift @@ -544,6 +544,13 @@ public struct OpenClawChatView: View { inlineWidgetResolverReady: self.viewModel.healthOK, inlineWidgetResourceResolver: { [weak viewModel] path, failedResource in await viewModel?.resolveInlineWidgetResource(path: path, replacing: failedResource) + }, + imageArtifactResolverReady: self.viewModel.healthOK, + loadImageArtifact: { [weak viewModel] artifactId in + guard let viewModel else { return nil } + return try await viewModel.transport.loadImageArtifact( + sessionKey: viewModel.sessionKey, + artifactId: artifactId) }) .frame( maxWidth: .infinity, diff --git a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel+HistoryReconciliation.swift b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel+HistoryReconciliation.swift index 18336ce9300e..08dedc173b78 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel+HistoryReconciliation.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel+HistoryReconciliation.swift @@ -25,6 +25,13 @@ extension OpenClawChatViewModel { thinkingSignature: content.thinkingSignature, mimeType: content.mimeType, fileName: content.fileName, + artifactId: content.artifactId, + url: content.url, + openUrl: content.openUrl, + alt: content.alt, + width: content.width, + height: content.height, + sizeBytes: content.sizeBytes, durationSeconds: content.durationSeconds, content: content.content, id: content.id, @@ -56,7 +63,12 @@ extension OpenClawChatViewModel { let id = (item.id ?? "").trimmingCharacters(in: .whitespacesAndNewlines) let name = (item.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines) let fileName = (item.fileName ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - return [type, text, id, name, fileName].joined(separator: "\\u{001F}") + let artifactId = (item.artifactId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let url = (item.url ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let openUrl = (item.openUrl ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let mimeType = (item.mimeType ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + return [type, text, id, name, fileName, artifactId, url, openUrl, mimeType] + .joined(separator: "\\u{001F}") }.joined(separator: "\\u{001E}") } @@ -64,7 +76,10 @@ extension OpenClawChatViewModel { message.content.map { item in let type = (item.type ?? "text").trimmingCharacters(in: .whitespacesAndNewlines).lowercased() let text = (item.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - return [type, text].joined(separator: "\\u{001F}") + let artifactId = (item.artifactId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let url = (item.url ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let openUrl = (item.openUrl ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + return [type, text, artifactId, url, openUrl].joined(separator: "\\u{001F}") }.joined(separator: "\\u{001E}") } @@ -176,6 +191,13 @@ extension OpenClawChatViewModel { thinkingSignature: content.thinkingSignature, mimeType: content.mimeType, fileName: content.fileName, + artifactId: content.artifactId, + url: content.url, + openUrl: content.openUrl, + alt: content.alt, + width: content.width, + height: content.height, + sizeBytes: content.sizeBytes, durationSeconds: localDuration, content: content.content, id: content.id, diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayNodeSession.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayNodeSession.swift index 274bc097ccd3..40e7a324b75c 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayNodeSession.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayNodeSession.swift @@ -710,6 +710,13 @@ public actor GatewayNodeSession { socketGeneration: socketGeneration) } + public func currentGatewayID(ifCurrentRoute route: GatewayNodeSessionRoute) -> String? { + guard self.isCurrentRoute(route), self.channel != nil else { return nil } + // iOS operator routes normalize this to the effective stable ID before connect. + // Keep nil for unscoped clients rather than letting artifact HTTP bind to a guessed owner. + return self.connectOptions?.deviceAuthGatewayID + } + public func supportsServerCapability( _ capability: GatewayServerCapability, ifCurrentRoute expectedRoute: GatewayNodeSessionRoute) -> Bool? diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayTLSPinning.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayTLSPinning.swift index 42b154354c77..642151da41a6 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayTLSPinning.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayTLSPinning.swift @@ -81,6 +81,10 @@ public struct GatewayTLSValidationError: LocalizedError, Sendable { } } +public enum GatewayBoundedDataError: Error, Equatable, Sendable { + case responseTooLarge(maximumBytes: Int) +} + protocol GatewayTLSFailureProviding: AnyObject { func consumeLastTLSFailure() -> GatewayTLSValidationFailure? } @@ -764,6 +768,42 @@ public final class GatewayTLSPinningSession: NSObject, WebSocketSessioning, URLS return WebSocketTaskBox(task: task) } + public func data(for request: URLRequest, maximumBytes: Int) async throws -> (Data, URLResponse) { + self.registerExpectedAuthority(url: request.url) + guard maximumBytes >= 0 else { + throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes) + } + + let (bytes, response) = try await self.session.bytes(for: request) + let expectedLength = response.expectedContentLength + guard expectedLength < 0 || expectedLength <= Int64(maximumBytes) else { + bytes.task.cancel() + throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes) + } + + var data = Data() + if expectedLength > 0 { + data.reserveCapacity(Int(expectedLength)) + } + do { + for try await byte in bytes { + guard data.count < maximumBytes else { + bytes.task.cancel() + throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes) + } + data.append(byte) + } + } catch { + bytes.task.cancel() + throw error + } + return (data, response) + } + + public func finishTasksAndInvalidate() { + self.session.finishTasksAndInvalidate() + } + public func urlSession( _ session: URLSession, didReceive challenge: URLAuthenticationChallenge, diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 6a1ef14bad52..89a6c76fb881 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -11813,17 +11813,20 @@ public struct ArtifactsDownloadResult: Codable, Sendable { public let encoding: String? public let data: String? public let url: String? + public let expiresat: String? public init( artifact: ArtifactSummary, encoding: String? = nil, data: String? = nil, - url: String? = nil) + url: String? = nil, + expiresat: String? = nil) { self.artifact = artifact self.encoding = encoding self.data = data self.url = url + self.expiresat = expiresat } private enum CodingKeys: String, CodingKey { @@ -11831,6 +11834,7 @@ public struct ArtifactsDownloadResult: Codable, Sendable { case encoding case data case url + case expiresat = "expiresAt" } } diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMessageMediaAttachmentTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMessageMediaAttachmentTests.swift new file mode 100644 index 000000000000..09699cec1d1f --- /dev/null +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMessageMediaAttachmentTests.swift @@ -0,0 +1,96 @@ +import Foundation +import Testing +@testable import OpenClawChatUI + +@Suite("Managed chat image attachments") +struct ChatMessageMediaAttachmentTests { + @Test func `decodes canonical managed image fields`() throws { + let message = try JSONDecoder().decode( + OpenClawChatMessage.self, + from: Data( + """ + { + "role": "assistant", + "content": [{ + "type": "image", + "artifactId": "artifact_managed_image_11111111-1111-4111-8111-111111111111", + "url": "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full", + "openUrl": "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full", + "alt": "Chart", + "mimeType": "image/png", + "width": 1200, + "height": 800, + "sizeBytes": 2048 + }] + } + """.utf8)) + + let image = try #require(message.content.first) + #expect(image.artifactId == "artifact_managed_image_11111111-1111-4111-8111-111111111111") + #expect(image.alt == "Chart") + #expect(image.mimeType == "image/png") + #expect(image.width == 1200) + #expect(image.height == 800) + #expect(image.sizeBytes == 2048) + #expect(image.isInlineAttachment) + } + + @Test @MainActor func `distinct images never reconcile as the same final message`() { + let first = Self.message(artifactId: "artifact_managed_image_11111111-1111-4111-8111-111111111111") + let second = Self.message(artifactId: "artifact_managed_image_22222222-2222-4222-8222-222222222222") + + #expect( + OpenClawChatViewModel.finalMessageContentFingerprint(for: first) != + OpenClawChatViewModel.finalMessageContentFingerprint(for: second)) + #expect( + OpenClawChatViewModel.messageContentFingerprint(for: first) != + OpenClawChatViewModel.messageContentFingerprint(for: second)) + } + + @Test func `derives stable identity for shipped managed image blocks`() throws { + let message = try JSONDecoder().decode( + OpenClawChatMessage.self, + from: Data( + """ + { + "role": "assistant", + "content": [{ + "type": "image", + "url": "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full", + "mimeType": "image/png" + }] + } + """.utf8)) + + #expect( + message.content.first?.artifactId == + "artifact_managed_image_11111111-1111-4111-8111-111111111111") + } + + @Test func `transcript cache preserves references without image bytes`() throws { + let message = Self.message( + artifactId: "artifact_managed_image_11111111-1111-4111-8111-111111111111") + let cached = try #require(OpenClawChatSQLiteTranscriptCache.cacheableMessages([message]).first) + let image = try #require(cached.content.first) + + #expect(image.artifactId == message.content.first?.artifactId) + #expect(image.url == message.content.first?.url) + #expect(image.content == nil) + } + + private static func message(artifactId: String) -> OpenClawChatMessage { + OpenClawChatMessage( + role: "assistant", + content: [ + OpenClawChatMessageContent( + type: "image", + text: nil, + mimeType: "image/png", + fileName: nil, + artifactId: artifactId, + url: "/api/chat/media/outgoing/main/\(artifactId)/full", + content: nil), + ], + timestamp: nil) + } +} diff --git a/docs/docs_map.md b/docs/docs_map.md index f2f270b129e2..78dfa61cbfb9 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -3310,6 +3310,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Choose scopes and pair the device - H2: Advertise client capabilities - H2: Recover state after reconnect + - H2: Render generated image artifacts - H2: Use history metadata and stable anchors - H2: Subscribe instead of polling usage - H2: Backfill exec approvals diff --git a/docs/gateway/clients.md b/docs/gateway/clients.md index b3ba002ac72c..82bd0c129e47 100644 --- a/docs/gateway/clients.md +++ b/docs/gateway/clients.md @@ -133,6 +133,30 @@ current WebSocket connection. It resets with a new connection. The `seq` inside an `agent` event payload is assigned per run and orders that run's lifecycle, assistant, plan, tool, and other stream events. +## Render generated image artifacts + +Assistant-generated images arrive as canonical `type: "image"` content blocks. +Managed blocks include a stable `artifactId`, a Gateway-relative `url`, MIME +type, dimensions, size, and accessible alt text. Keep that reference in the +transcript cache; do not persist downloaded bytes or temporary download URLs. + +Resolve the image through the authenticated WebSocket connection: + +1. Call `artifacts.download` with the current `sessionKey`, optional `agentId`, + and the block's `artifactId`. +2. Use the returned short-lived `url` before `expiresAt`. The URL is scoped to + that exact transcript-backed artifact and does not contain a reusable Gateway + or device credential. +3. Fetch it from the Gateway origin using the same TLS pin and reverse-proxy + headers as the active connection. Validate the response as an image and + enforce a 12 MiB source limit plus a bounded decoded thumbnail. +4. If the URL expires, repeat `artifacts.download` once. Reconnect or route + changes cancel the old load rather than retargeting it to another Gateway. + +Older image blocks without `artifactId` remain displayable by existing Control +UI clients, but native clients should show a readable attachment fallback rather +than forward a shared owner credential. + ## Use history metadata and stable anchors Rows returned by `chat.history` can carry an `__openclaw` metadata envelope: diff --git a/docs/platforms/android.md b/docs/platforms/android.md index 0906cab1ccbd..b3537f0108b3 100644 --- a/docs/platforms/android.md +++ b/docs/platforms/android.md @@ -301,6 +301,7 @@ The Android Chat tab supports session selection (default `main`, plus other exis - History: `chat.history` (display-normalized — inline directive tags, plain-text tool-call XML payloads (``, ``, ``, ``, and truncated variants), and leaked ASCII/full-width model control tokens are stripped; silent-token assistant rows such as exact `NO_REPLY` / `no_reply` are omitted; oversized rows can be replaced with placeholders) - Send: `chat.send` - Durable sending: every send (text, picked images, and voice notes) is journaled to a per-gateway on-device outbox before any network attempt, so app termination cannot lose submitted input. Sends queued while offline deliver in order on reconnect with stable idempotency keys, and a send is retired only after the turn is visible in canonical `chat.history` — an acknowledgement alone is not treated as proof of delivery. Ambiguous outcomes (lost acknowledgement, app killed mid-send, gateway restart before the transcript write) surface as visible rows with explicit **Retry**/**Delete** instead of auto-resending. Slash commands never auto-replay across a reconnect; they park for explicit retry. The queue is bounded (50 messages and 48 MB of attachment bytes per gateway) and unsent rows expire after 48 hours. Composer drafts that were never submitted are not process-durable. +- Image input works through the picker and Android Sharesheet. Assistant-generated images resolve through the paired Gateway connection, render inline with a full-screen preview, and retain only their small artifact references in the offline transcript cache. Downloads are capped at 12 MiB and decoded to bounded display bitmaps. - Push updates (best-effort): `chat.subscribe` -> `event:"chat"` - Listen: long-press an assistant message and choose **Listen** to hear it; audio renders via gateway `tts.speak` with the configured TTS provider chain, and on-device system TTS is used when the gateway cannot render audio. Playback stops on session switch, new chat, app backgrounding, or chat close. diff --git a/docs/platforms/ios.md b/docs/platforms/ios.md index d4824733659c..38168455f434 100644 --- a/docs/platforms/ios.md +++ b/docs/platforms/ios.md @@ -19,6 +19,7 @@ Availability: iPhone app builds are distributed through Apple channels when enab - Keeps a small read-only offline cache of recent chat sessions and transcripts per paired gateway: cold opens paint the last known transcript immediately and refresh once the gateway responds, recent chats stay browsable while disconnected, and reset/forget purges the protected local cache. - Queues text messages sent while disconnected in a durable per-gateway outbox (up to 50): queued bubbles show in the transcript, flush in order on reconnect with idempotent retries, remain durable until canonical history confirms the send, retry with backoff before surfacing a retry/delete action, and expire instead of sending after 48 hours offline; reset/forget clears the queue with the cache. - Chat is the single text-and-voice surface. Chat actions can open the full Sessions screen without leaving Chat and can show or hide assistant reasoning and tool activity. Tap the microphone for draft dictation, open its menu to record a voice note, or use the inline Talk control for realtime voice; the Talk control animates from live microphone or playback level while listening or speaking. +- Chat accepts images from the photo picker, camera, Files, paste, and the iOS share sheet. Assistant-generated images render inline from short-lived Gateway artifact URLs, open in a full-screen preview, and remain available after reconnect or history reload without storing image bytes in the transcript cache. - **Settings -> OpenClaw** opens a dedicated Gateway settings assistant when the operator connection has `operator.admin` and the Gateway supports `openclaw.chat`. Its setup conversation stays separate from ordinary Chat, redacts secret replies locally, and moves to Chat only after you tap **Open Chat**. - Speaks assistant messages on demand: long-press a message in Chat and choose **Listen**. The app plays supported gateway `tts.speak` clips with the configured TTS provider and falls back to on-device speech when gateway audio is unavailable or unplayable. Playback stops on session switch or backgrounding. diff --git a/docs/platforms/macos.md b/docs/platforms/macos.md index 329422a51b87..27ec55056738 100644 --- a/docs/platforms/macos.md +++ b/docs/platforms/macos.md @@ -13,6 +13,11 @@ Mac-hosted node tools such as `system.run`. Use **Quick Chat** for a Spotlight-style main-session composer without opening a full window. Press Option-Space (⌥Space) by default, choose it from the menu bar menu, or record another shortcut in **Settings → General**. +The full native chat accepts image attachments through its picker, paste, and +drag and drop. Assistant-generated images render inline through short-lived +Gateway artifact URLs and open in a larger preview; iOS and macOS share the same +bounded image model and renderer. + Only need the CLI and Gateway? Start with [Getting started](/start/getting-started). ## Download diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index 59450014bc1d..bd50c460266c 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -417,7 +417,7 @@ The macOS app keeps its native link-browser sidebar for links clicked in the das - Re-sending with the same `idempotencyKey` returns `{ status: "in_flight" }` while running, and `{ status: "ok" }` after completion. - `chat.history` responses are size-bounded for UI safety. When transcript entries are too large, Gateway may truncate long text fields, omit heavy metadata blocks, and replace oversized messages with a placeholder (`[chat.history omitted: message too large]`). - When a visible assistant message was truncated in `chat.history`, the side reader can fetch the full display-normalized transcript entry on demand through `chat.message.get` by `sessionKey`, active `agentId` when needed, and transcript `messageId`. If the Gateway still cannot return more, the reader shows an explicit unavailable state instead of silently repeating the truncated preview. - - Assistant/generated images are persisted as managed media references and served back through authenticated Gateway media URLs, so reloads do not depend on raw base64 image payloads staying in the chat history response. + - Assistant/generated images are persisted as managed media references. New clients resolve their stable artifact ids through authenticated `artifacts.download` and receive short-lived, exact-resource media URLs, so reloads do not depend on raw base64 payloads or reusable credentials in image URLs. - When rendering `chat.history`, the Control UI strips display-only inline directive tags from visible assistant text (for example `[[reply_to_*]]` and `[[audio_as_voice]]`), plain-text tool-call XML payloads (including `...`, `...`, `...`, `...`, and truncated tool-call blocks), and leaked ASCII/full-width model control tokens. It omits assistant entries whose whole visible text is only the exact silent token `NO_REPLY` / `no_reply` or the heartbeat acknowledgement token `HEARTBEAT_OK`. - During an active send and the final history refresh, the chat view keeps local optimistic user/assistant messages visible if `chat.history` briefly returns an older snapshot; the canonical transcript replaces those local messages once the Gateway history catches up. - Live `chat` events are delivery state, while `chat.history` is rebuilt from the durable session transcript. After tool-final events the Control UI reloads history and merges only a small optimistic tail; the transcript boundary is documented in [WebChat](/web/webchat). @@ -663,6 +663,13 @@ When gateway auth is configured, assistant local-media previews use a two-step r This keeps media rendering compatible with browser-native media elements without putting reusable gateway credentials in visible media URLs. +Generated images under `/api/chat/media/outgoing/...` use the same capability +principle through `artifacts.download`. The authenticated WebSocket request +authorizes the transcript artifact and returns a short-lived URL. The HTTP media +route rechecks that the artifact still belongs to the transcript before serving +bytes. The previous shared-owner bearer path remains available for older Control +UI clients during the compatibility window. + ## Approval links Operator approval notifications can deep-link to a [standalone approval document](/web/urls#special-documents-and-startup-modes). The URL is stable for the lifetime of the approval and safe to forward between your own devices: it identifies the approval, never authorizes it. diff --git a/packages/gateway-protocol/src/schema/artifacts.ts b/packages/gateway-protocol/src/schema/artifacts.ts index 0df655ef6b6a..658d495dcc7f 100644 --- a/packages/gateway-protocol/src/schema/artifacts.ts +++ b/packages/gateway-protocol/src/schema/artifacts.ts @@ -68,6 +68,7 @@ export const ArtifactsDownloadResultSchema = closedObject({ encoding: Type.Optional(Type.Literal("base64")), data: Type.Optional(Type.String()), url: Type.Optional(NonEmptyString), + expiresAt: Type.Optional(NonEmptyString), }); // Wire types derive directly from local schema consts so public d.ts graphs never diff --git a/src/gateway/managed-image-attachments.test.ts b/src/gateway/managed-image-attachments.test.ts index c9e2b05b94e0..bf9d9b497e75 100644 --- a/src/gateway/managed-image-attachments.test.ts +++ b/src/gateway/managed-image-attachments.test.ts @@ -55,10 +55,12 @@ vi.mock("./session-transcript-readers.js", () => ({ const { DEFAULT_MANAGED_IMAGE_ATTACHMENT_LIMITS, + MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX, attachManagedOutgoingImagesToMessage, cleanupManagedOutgoingImageRecords, createManagedOutgoingImageBlocks, handleManagedOutgoingImageHttpRequest, + resolveManagedOutgoingImageArtifactDownload, resolveManagedImageAttachmentLimits, } = await import("./managed-image-attachments.js"); @@ -101,8 +103,10 @@ async function expectPathMissing(targetPath: string): Promise { type ManagedImageBlock = { type?: string; + artifactId?: string; alt?: string; mimeType?: string; + sizeBytes?: number; url?: string; openUrl?: string; }; @@ -319,6 +323,50 @@ describe("handleManagedOutgoingImageHttpRequest", () => { ); }); + it("serves an exact transcript image through a short-lived artifact ticket", async () => { + const { attachmentId, sessionKey } = await createFixture(stateDir); + const canonicalPath = `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/${attachmentId}/full`; + loadSessionEntryMock.mockReturnValue({ + storePath: path.join(stateDir, "gateway-sessions.json"), + entry: { sessionId: "sess-1", sessionFile: "session.jsonl" }, + }); + resolveSessionHistoryTranscriptPathMock.mockResolvedValue("session.jsonl"); + readSessionMessagesMock.mockResolvedValue([ + { + role: "assistant", + content: [{ type: "image", url: canonicalPath, openUrl: canonicalPath }], + __openclaw: { id: "msg-1" }, + }, + ]); + + const download = await resolveManagedOutgoingImageArtifactDownload({ + sessionKey, + artifactId: `${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`, + stateDir, + }); + expect(download?.url).toContain("mediaTicket="); + + vi.clearAllMocks(); + const { result } = await requestManagedImage({ + stateDir, + pathName: download?.url ?? "", + denyAuth: true, + }); + + expect(result.statusCode).toBe(200); + expect(result.body.toString("utf-8")).toBe("original-image"); + expect(authorizeGatewayHttpRequestOrReplyMock).not.toHaveBeenCalled(); + + const wrongAttachmentId = "22222222-2222-4222-8222-222222222222"; + const wrong = await requestManagedImage({ + stateDir, + pathName: (download?.url ?? "").replace(attachmentId, wrongAttachmentId), + denyAuth: true, + }); + expect(wrong.result.statusCode).toBe(401); + expect(authorizeGatewayHttpRequestOrReplyMock).toHaveBeenCalledTimes(1); + }); + it("keeps serving and deleting an original after the configured media root changes", async () => { const fixture = await createFixture(stateDir); const externalConfigDir = tempDirs.make("managed-image-moved-config-"); @@ -653,6 +701,8 @@ describe("createManagedOutgoingImageBlocks", () => { expect(String(block.url)).toMatch(/\/full$/); const attachmentId = requireAttachmentIdFromUrl(block.url); + expect(block.artifactId).toBe(`${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`); + expect(block.sizeBytes).toBe(Buffer.from(TINY_PNG_BASE64, "base64").byteLength); const record = readManagedImageRecord(attachmentId, stateDir); expect(record?.original.mediaSubdir).toBe(MANAGED_OUTGOING_ORIGINALS_SUBDIR); expect(record?.original.mediaId).toMatch(/\.png$/); diff --git a/src/gateway/managed-image-attachments.ts b/src/gateway/managed-image-attachments.ts index febfa7aa13b0..b00e16d4a360 100644 --- a/src/gateway/managed-image-attachments.ts +++ b/src/gateway/managed-image-attachments.ts @@ -1,10 +1,14 @@ // Gateway managed image attachment store. // Validates, stores, serves, and cleans up outgoing image attachments. -import { randomUUID } from "node:crypto"; +import { createHmac, randomBytes, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import type { IncomingMessage, ServerResponse } from "node:http"; import path from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; +import { + asDateTimestampMs, + resolveTimestampMsToIsoString, +} from "@openclaw/normalization-core/number-coercion"; import { resolveDefaultAgentId } from "../agents/agent-scope-config.js"; import { getRuntimeConfig } from "../config/config.js"; import { resolveStateDir } from "../config/paths.js"; @@ -17,6 +21,7 @@ import { readImageProbeFromHeader, } from "../media/media-services.js"; import { getMediaDir, MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "../media/store.js"; +import { safeEqualSecret } from "../security/secret-equal.js"; import type { AuthRateLimiter } from "./auth-rate-limit.js"; import type { ResolvedGatewayAuth } from "./auth.js"; import { sendJson, sendMethodNotAllowed, sendMissingScopeForbidden } from "./http-common.js"; @@ -44,8 +49,12 @@ import { const OUTGOING_IMAGE_ROUTE_PREFIX = "/api/chat/media/outgoing"; const DEFAULT_TRANSIENT_OUTGOING_IMAGE_TTL_MS = 15 * 60 * 1000; +const MANAGED_OUTGOING_IMAGE_TICKET_SCOPE = "managed-outgoing-image"; +export const MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS = 5 * 60 * 1000; +export const MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX = "artifact_managed_image_"; const MANAGED_OUTGOING_ATTACHMENT_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const managedOutgoingImageTicketSecret = randomBytes(32); export const DEFAULT_MANAGED_IMAGE_ATTACHMENT_LIMITS = { maxBytes: 12 * 1024 * 1024, @@ -86,6 +95,24 @@ type SessionManagedOutgoingAttachmentIndexCacheEntry = { size: number; index: SessionManagedOutgoingAttachmentIndex; }; + +type ManagedOutgoingImageTicketPayload = { + scope: typeof MANAGED_OUTGOING_IMAGE_TICKET_SCOPE; + sessionKey: string; + attachmentId: string; + variant: "full"; + exp: number; +}; + +export type ManagedOutgoingImageArtifactDownload = { + artifactId: string; + sessionKey: string; + title: string; + mimeType?: string; + sizeBytes?: number; + url: string; + expiresAt: string; +}; type SessionManagedOutgoingAttachmentTranscriptStat = Omit< SessionManagedOutgoingAttachmentIndexCacheEntry, "index" @@ -310,6 +337,91 @@ function buildOutgoingVariantUrl(sessionKey: string, attachmentId: string, varia return `${OUTGOING_IMAGE_ROUTE_PREFIX}/${encodeURIComponent(sessionKey)}/${attachmentId}/${variant}`; } +function buildManagedOutgoingImageArtifactId(attachmentId: string): string { + return `${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`; +} + +export function parseManagedOutgoingImageArtifactId(value: string): string | null { + if (!value.startsWith(MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX)) { + return null; + } + const attachmentId = value.slice(MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX.length); + return MANAGED_OUTGOING_ATTACHMENT_ID_RE.test(attachmentId) ? attachmentId : null; +} + +function signManagedOutgoingImageTicketPayload(encodedPayload: string): string { + return createHmac("sha256", managedOutgoingImageTicketSecret) + .update(encodedPayload) + .digest("base64url"); +} + +function createManagedOutgoingImageTicket(params: { + sessionKey: string; + attachmentId: string; + nowMs?: number; +}): { ticket: string; expiresAt: string } | null { + const now = asDateTimestampMs(params.nowMs ?? Date.now()); + if (now === undefined) { + return null; + } + const exp = asDateTimestampMs(now + MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS); + if (exp === undefined) { + return null; + } + const payload: ManagedOutgoingImageTicketPayload = { + scope: MANAGED_OUTGOING_IMAGE_TICKET_SCOPE, + sessionKey: params.sessionKey, + attachmentId: params.attachmentId, + variant: "full", + exp, + }; + const encodedPayload = Buffer.from(JSON.stringify(payload), "utf8").toString("base64url"); + const signature = signManagedOutgoingImageTicketPayload(encodedPayload); + return { + ticket: `v1.${encodedPayload}.${signature}`, + expiresAt: resolveTimestampMsToIsoString(exp), + }; +} + +function verifyManagedOutgoingImageTicket(params: { + ticket: string | null; + sessionKey: string; + attachmentId: string; + nowMs?: number; +}): boolean { + const now = asDateTimestampMs(params.nowMs ?? Date.now()); + if (now === undefined) { + return false; + } + const parts = params.ticket?.split("."); + if (!parts || parts.length !== 3 || parts[0] !== "v1") { + return false; + } + const [, encodedPayload, signature] = parts; + if (!encodedPayload || !signature) { + return false; + } + if (!safeEqualSecret(signature, signManagedOutgoingImageTicketPayload(encodedPayload))) { + return false; + } + try { + const payload = JSON.parse( + Buffer.from(encodedPayload, "base64url").toString("utf8"), + ) as Partial; + return ( + payload.scope === MANAGED_OUTGOING_IMAGE_TICKET_SCOPE && + payload.sessionKey === params.sessionKey && + payload.attachmentId === params.attachmentId && + payload.variant === "full" && + typeof payload.exp === "number" && + Number.isFinite(payload.exp) && + payload.exp >= now + ); + } catch { + return false; + } +} + function deriveAltText(source: string, index: number) { const fallback = `Generated image ${index + 1}`; try { @@ -508,12 +620,14 @@ function buildManagedImageBlock(record: ManagedImageRecord): ManagedImageBlock { const fullUrl = buildOutgoingVariantUrl(record.sessionKey, record.attachmentId, "full"); return { type: "image", + artifactId: buildManagedOutgoingImageArtifactId(record.attachmentId), url: fullUrl, openUrl: fullUrl, alt: record.alt, mimeType: record.original.contentType, width: record.original.width, height: record.original.height, + sizeBytes: record.original.sizeBytes, }; } @@ -565,7 +679,7 @@ function parseManagedOutgoingRoute(value: string) { sessionKey: decodeURIComponent( expectDefined(match[1], "managed image attachments regex capture 1"), ), - attachmentId: match[2], + attachmentId: expectDefined(match[2], "managed image attachments regex capture 2"), }; } catch { return null; @@ -792,6 +906,74 @@ async function recordMatchesTranscriptMessage( ); } +async function resolveManagedOutgoingImageArtifactDownloadForRecord( + record: ManagedImageRecord, +): Promise { + if (!(await recordMatchesTranscriptMessage(record))) { + return null; + } + const ticket = createManagedOutgoingImageTicket({ + sessionKey: record.sessionKey, + attachmentId: record.attachmentId, + }); + if (!ticket) { + return null; + } + try { + const stat = await fs.stat(resolveManagedImageOriginalPath(record)); + if (!stat.isFile()) { + return null; + } + } catch { + return null; + } + const canonicalUrl = buildOutgoingVariantUrl(record.sessionKey, record.attachmentId, "full"); + const params = new URLSearchParams({ mediaTicket: ticket.ticket }); + return { + artifactId: buildManagedOutgoingImageArtifactId(record.attachmentId), + sessionKey: record.sessionKey, + title: record.alt, + ...(record.original.contentType ? { mimeType: record.original.contentType } : {}), + ...(record.original.sizeBytes != null ? { sizeBytes: record.original.sizeBytes } : {}), + url: `${canonicalUrl}?${params.toString()}`, + expiresAt: ticket.expiresAt, + }; +} + +/** Resolve one transcript-backed image to a short-lived HTTP capability. */ +export async function resolveManagedOutgoingImageArtifactDownload(params: { + sessionKey: string; + artifactId: string; + stateDir?: string; +}): Promise { + const attachmentId = parseManagedOutgoingImageArtifactId(params.artifactId); + if (!attachmentId) { + return null; + } + const record = readManagedImageRecord(attachmentId, params.stateDir); + if (!record || record.sessionKey !== params.sessionKey) { + return null; + } + return await resolveManagedOutgoingImageArtifactDownloadForRecord(record); +} + +/** Upgrade legacy managed-image URLs that predate stable artifact ids. */ +export async function resolveManagedOutgoingImageUrlDownload(params: { + sessionKey: string; + url: string; + stateDir?: string; +}): Promise { + const parsed = parseManagedOutgoingRoute(params.url); + if (!parsed || parsed.sessionKey !== params.sessionKey) { + return null; + } + const record = readManagedImageRecord(parsed.attachmentId, params.stateDir); + if (!record || record.sessionKey !== params.sessionKey) { + return null; + } + return await resolveManagedOutgoingImageArtifactDownloadForRecord(record); +} + export async function attachManagedOutgoingImagesToMessage(params: { messageId: string; blocks?: readonly Record[]; @@ -1051,25 +1233,6 @@ export async function handleManagedOutgoingImageHttpRequest( return true; } - const requestAuth = await authorizeGatewayHttpRequestOrReply({ - req, - res, - auth: opts.auth, - trustedProxies: opts.trustedProxies, - allowRealIpFallback: opts.allowRealIpFallback, - rateLimiter: opts.rateLimiter, - }); - if (!requestAuth) { - return true; - } - - const requestedScopes = resolveOpenAiCompatibleHttpOperatorScopes(req, requestAuth); - const scopeAuth = authorizeOperatorScopesForMethod("chat.history", requestedScopes); - if (!scopeAuth.allowed) { - sendMissingScopeForbidden(res, scopeAuth.missingScope); - return true; - } - const encodedSessionKey = match[1]; const attachmentId = match[2]; if (!encodedSessionKey || !attachmentId) { @@ -1086,23 +1249,49 @@ export async function handleManagedOutgoingImageHttpRequest( sendStatus(res, 404, "not found"); return true; } + const hasValidMediaTicket = verifyManagedOutgoingImageTicket({ + ticket: requestUrl.searchParams.get("mediaTicket"), + sessionKey, + attachmentId, + }); + if (!hasValidMediaTicket) { + const requestAuth = await authorizeGatewayHttpRequestOrReply({ + req, + res, + auth: opts.auth, + trustedProxies: opts.trustedProxies, + allowRealIpFallback: opts.allowRealIpFallback, + rateLimiter: opts.rateLimiter, + }); + if (!requestAuth) { + return true; + } + + const requestedScopes = resolveOpenAiCompatibleHttpOperatorScopes(req, requestAuth); + const scopeAuth = authorizeOperatorScopesForMethod("chat.history", requestedScopes); + if (!scopeAuth.allowed) { + sendMissingScopeForbidden(res, scopeAuth.missingScope); + return true; + } + // The reusable shared-secret route remains for older Control UI clients. + // Ticketed clients prove the exact transcript attachment instead of + // forwarding an owner credential through another HTTP stack. + if (!resolveOpenAiCompatibleHttpSenderIsOwner(req, requestAuth)) { + sendJson(res, 403, { + ok: false, + error: { + type: "forbidden", + message: "owner access required", + }, + }); + return true; + } + } const record = readManagedImageRecord(attachmentId, opts.stateDir); if (!record || record.sessionKey !== sessionKey) { sendStatus(res, 404, "not found"); return true; } - // Requester-session headers are client-declared, so media bytes require - // authenticated owner/admin context rather than trusting a URL-scoped header. - if (!resolveOpenAiCompatibleHttpSenderIsOwner(req, requestAuth)) { - sendJson(res, 403, { - ok: false, - error: { - type: "forbidden", - message: "owner access required", - }, - }); - return true; - } if (!(await recordMatchesTranscriptMessage(record))) { sendStatus(res, 404, "not found"); return true; @@ -1123,7 +1312,14 @@ export async function handleManagedOutgoingImageHttpRequest( res.statusCode = 200; res.setHeader("content-type", record.original.contentType || "application/octet-stream"); res.setHeader("content-length", String(body.byteLength)); - res.setHeader("cache-control", "private, max-age=31536000, immutable"); + res.setHeader("x-content-type-options", "nosniff"); + res.setHeader("referrer-policy", "no-referrer"); + res.setHeader( + "cache-control", + hasValidMediaTicket + ? `private, max-age=${MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS / 1000}, immutable` + : "private, max-age=31536000, immutable", + ); res.setHeader( "content-disposition", `inline; filename="${safeAttachmentFilename(record.original.filename)}"`, diff --git a/src/gateway/server-methods/artifacts.test.ts b/src/gateway/server-methods/artifacts.test.ts index 331da807b198..0eb49ad097df 100644 --- a/src/gateway/server-methods/artifacts.test.ts +++ b/src/gateway/server-methods/artifacts.test.ts @@ -7,6 +7,8 @@ import { artifactsHandlers } from "./artifacts.js"; const hoisted = vi.hoisted(() => ({ getTaskSessionLookupByIdForStatus: vi.fn(), loadSessionEntry: vi.fn(), + resolveManagedArtifactDownload: vi.fn(), + resolveManagedUrlDownload: vi.fn(), visitSessionMessagesAsync: vi.fn(), resolveSessionKeyForRun: vi.fn(), })); @@ -44,6 +46,17 @@ vi.mock("../server-session-key.js", async () => { }; }); +vi.mock("../managed-image-attachments.js", async () => { + const actual = await vi.importActual( + "../managed-image-attachments.js", + ); + return { + ...actual, + resolveManagedOutgoingImageArtifactDownload: hoisted.resolveManagedArtifactDownload, + resolveManagedOutgoingImageUrlDownload: hoisted.resolveManagedUrlDownload, + }; +}); + function createResponder() { const calls: Array<{ ok: boolean; payload?: unknown; error?: unknown }> = []; return { @@ -211,6 +224,8 @@ describe("artifacts RPC handlers", () => { beforeEach(() => { vi.clearAllMocks(); hoisted.resolveSessionKeyForRun.mockReset(); + hoisted.resolveManagedArtifactDownload.mockResolvedValue(null); + hoisted.resolveManagedUrlDownload.mockResolvedValue(null); hoisted.getTaskSessionLookupByIdForStatus.mockReturnValue(undefined); hoisted.loadSessionEntry.mockReturnValue({ storePath: "/tmp/sessions.json", @@ -360,6 +375,48 @@ describe("artifacts RPC handlers", () => { expectFields(downloadPayload.artifact, { id: artifactId }); }); + it("preserves managed artifact identity and returns a ticketed download URL", async () => { + const artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111"; + mockedMessages([ + { + role: "assistant", + content: [ + { + type: "image", + artifactId, + url: "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full", + alt: "chart.png", + mimeType: "image/png", + sizeBytes: 14, + }, + ], + __openclaw: { seq: 2 }, + }, + ]); + hoisted.resolveManagedArtifactDownload.mockResolvedValue({ + artifactId, + sessionKey: "agent:main:main", + title: "chart.png", + mimeType: "image/png", + sizeBytes: 14, + url: "/api/chat/media/outgoing/agent%3Amain%3Amain/id/full?mediaTicket=ticket", + expiresAt: "2026-07-28T05:00:00.000Z", + }); + + const listed = await listArtifacts({ sessionKey: "agent:main:main" }); + expectFields(expectFirstArtifact(listed.calls), { id: artifactId, sizeBytes: 14 }); + + const downloaded = await downloadArtifact({ sessionKey: "agent:main:main", artifactId }); + expectFields(expectOkPayload(downloaded.calls), { + url: "/api/chat/media/outgoing/agent%3Amain%3Amain/id/full?mediaTicket=ticket", + expiresAt: "2026-07-28T05:00:00.000Z", + }); + expect(hoisted.resolveManagedArtifactDownload).toHaveBeenCalledWith({ + sessionKey: "agent:main:main", + artifactId, + }); + }); + it("can scan artifact summaries without retaining inline data", async () => { mockedMessages([ { diff --git a/src/gateway/server-methods/artifacts.ts b/src/gateway/server-methods/artifacts.ts index 1549e0311a75..99cdc4355582 100644 --- a/src/gateway/server-methods/artifacts.ts +++ b/src/gateway/server-methods/artifacts.ts @@ -22,6 +22,11 @@ import { toAgentStoreSessionKey, } from "../../routing/session-key.js"; import { getTaskSessionLookupByIdForStatus } from "../../tasks/task-status-access.js"; +import { + parseManagedOutgoingImageArtifactId, + resolveManagedOutgoingImageArtifactDownload, + resolveManagedOutgoingImageUrlDownload, +} from "../managed-image-attachments.js"; import { resolveSessionKeyForRun } from "../server-session-key.js"; import { resolveSessionStoreAgentId, @@ -411,13 +416,17 @@ function collectArtifactsFromMessage(params: { asNonEmptyString(block.filename) ?? asNonEmptyString(block.alt) ?? `${type} ${params.artifacts.length + 1}`; - const id = artifactId({ - sessionKey: params.sessionKey, - messageSeq, - contentIndex, - title, - type, - }); + const declaredArtifactId = asNonEmptyString(block.artifactId); + const id = + declaredArtifactId && parseManagedOutgoingImageArtifactId(declaredArtifactId) + ? declaredArtifactId + : artifactId({ + sessionKey: params.sessionKey, + messageSeq, + contentIndex, + title, + type, + }); const includeData = params.downloadArtifactId ? params.downloadArtifactId === id : params.includeDownloadData !== false; @@ -639,6 +648,37 @@ export const artifactsHandlers: GatewayRequestHandlers = { if (!requireQueryable(params, respond)) { return; } + if ( + params.sessionKey && + !params.runId && + !params.taskId && + parseManagedOutgoingImageArtifactId(params.artifactId) + ) { + const resolved = resolveQuerySession(params, context.getRuntimeConfig?.()); + const managed = resolved + ? await resolveManagedOutgoingImageArtifactDownload({ + sessionKey: resolved.sessionKey, + artifactId: params.artifactId, + }) + : null; + if (managed) { + respond(true, { + artifact: { + id: managed.artifactId, + type: "image", + title: managed.title, + ...(managed.mimeType ? { mimeType: managed.mimeType } : {}), + ...(managed.sizeBytes !== undefined ? { sizeBytes: managed.sizeBytes } : {}), + sessionKey: managed.sessionKey, + source: "session-transcript", + download: { mode: "url" as const }, + }, + url: managed.url, + expiresAt: managed.expiresAt, + }); + return; + } + } const { artifact } = await findArtifact(params, context.getRuntimeConfig?.(), { downloadArtifactId: params.artifactId, }); @@ -662,12 +702,24 @@ export const artifactsHandlers: GatewayRequestHandlers = { ); return; } + const managedUrl = + artifact.download.mode === "url" && artifact.url && artifact.sessionKey + ? await resolveManagedOutgoingImageUrlDownload({ + sessionKey: artifact.sessionKey, + url: artifact.url, + }) + : null; respond(true, { artifact: toSummary(artifact), ...(artifact.download.mode === "bytes" ? { encoding: "base64" as const, data: artifact.data } : {}), - ...(artifact.download.mode === "url" ? { url: artifact.url } : {}), + ...(artifact.download.mode === "url" + ? { + url: managedUrl?.url ?? artifact.url, + ...(managedUrl ? { expiresAt: managedUrl.expiresAt } : {}), + } + : {}), }); }, }; diff --git a/ui/src/api/types.ts b/ui/src/api/types.ts index e4b472169a3f..31d644a63b2f 100644 --- a/ui/src/api/types.ts +++ b/ui/src/api/types.ts @@ -464,6 +464,7 @@ export type ArtifactDownloadResult = { encoding?: "base64"; data?: string; url?: string; + expiresAt?: string; }; export type SessionRunStatus = "running" | "done" | "failed" | "killed" | "timeout"; diff --git a/ui/src/e2e/chat-flow.media-files.e2e.test.ts b/ui/src/e2e/chat-flow.media-files.e2e.test.ts index 072562cf41e8..d92fb9c92f52 100644 --- a/ui/src/e2e/chat-flow.media-files.e2e.test.ts +++ b/ui/src/e2e/chat-flow.media-files.e2e.test.ts @@ -109,6 +109,83 @@ suite.define(() => { } }); + it("renders a managed image through an artifact-scoped ticket", async () => { + const context = await suite.newBrowserContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, + }); + const page = await context.newPage(); + const attachmentId = crypto.randomUUID(); + const artifactId = `artifact_managed_image_${attachmentId}`; + const imageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${attachmentId}/full`; + const ticketedUrl = `${imageUrl}?mediaTicket=ticket-e2e`; + await page.route("**/api/chat/media/outgoing/**", async (route) => { + const request = route.request(); + expect(new URL(request.url()).searchParams.get("mediaTicket")).toBe("ticket-e2e"); + expect(request.headers().authorization).toBeUndefined(); + await route.fulfill({ + contentType: "image/png", + body: Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=", + "base64", + ), + }); + }); + const gateway = await installMockGateway(page, { + historyMessages: [ + { + role: "assistant", + content: [ + { + type: "image", + artifactId, + url: imageUrl, + alt: "Ticketed generated image", + mimeType: "image/png", + width: 1, + height: 1, + }, + ], + timestamp: Date.now(), + }, + ], + methodResponses: { + "artifacts.download": { + artifact: { + id: artifactId, + type: "image", + title: "Ticketed generated image", + mimeType: "image/png", + download: { mode: "url" }, + }, + url: ticketedUrl, + expiresAt: new Date(Date.now() + 5 * 60_000).toISOString(), + }, + }, + }); + + try { + await page.goto(`${suite.server.baseUrl}chat`); + const image = page.getByAltText("Ticketed generated image"); + await image.waitFor({ state: "visible", timeout: 10_000 }); + await expect + .poll(() => + image.evaluate((element) => + element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0, + ), + ) + .toBe(1); + const request = await gateway.waitForRequest("artifacts.download"); + expect(request.params).toMatchObject({ + sessionKey: "agent:main:main", + artifactId, + }); + } finally { + await suite.closeBrowserContext(context); + } + }); + it("renders a canonical inbound image through the ticketed media route", async () => { const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim(); const context = await suite.newBrowserContext({ diff --git a/ui/src/lib/observer-digest.test.ts b/ui/src/lib/observer-digest.test.ts index 2592a3bc00b2..e2b84e6232d4 100644 --- a/ui/src/lib/observer-digest.test.ts +++ b/ui/src/lib/observer-digest.test.ts @@ -1,6 +1,31 @@ import { describe, expect, it } from "vitest"; import type { SessionObserverDigest } from "../../../packages/gateway-protocol/src/schema/sessions.js"; -import { isCriticalObserverHealth, ObserverDigestHistory } from "./observer-digest.ts"; +import { + isCriticalObserverHealth, + ObserverDigestHistory, + projectSessionObserverDigest, +} from "./observer-digest.ts"; + +describe("projectSessionObserverDigest", () => { + it("binds a session-row projection to its owning session", () => { + expect( + projectSessionObserverDigest("agent:main:projected", { + runId: "run-1", + revision: 2, + updatedAt: 3, + headline: "Projected", + health: "on-track", + }), + ).toEqual({ + sessionKey: "agent:main:projected", + runId: "run-1", + revision: 2, + updatedAt: 3, + headline: "Projected", + health: "on-track", + }); + }); +}); describe("isCriticalObserverHealth", () => { it("recognizes only health states that require operator attention", () => { diff --git a/ui/src/lib/observer-digest.ts b/ui/src/lib/observer-digest.ts index 1f5b4ab37621..cfa62ac4fae7 100644 --- a/ui/src/lib/observer-digest.ts +++ b/ui/src/lib/observer-digest.ts @@ -13,6 +13,24 @@ type ProjectedObserverDigest = Pick< "agentId" | "runId" | "headline" | "health" | "updatedAt" | "revision" >; +export function projectSessionObserverDigest( + sessionKey: string, + digest: ProjectedObserverDigest | null | undefined, +): SessionObserverDigest | null { + if (!digest) { + return null; + } + return { + sessionKey, + ...(digest.agentId ? { agentId: digest.agentId } : {}), + runId: digest.runId, + revision: digest.revision, + updatedAt: digest.updatedAt, + headline: digest.headline, + health: digest.health, + }; +} + export function isCriticalObserverHealth(health: unknown): health is "stuck" | "waiting-on-user" { return health === "stuck" || health === "waiting-on-user"; } diff --git a/ui/src/pages/chat/chat-pane-deps.ts b/ui/src/pages/chat/chat-pane-deps.ts index f0b493c0e06e..2a246647d27e 100644 --- a/ui/src/pages/chat/chat-pane-deps.ts +++ b/ui/src/pages/chat/chat-pane-deps.ts @@ -115,6 +115,7 @@ export { export { ObserverDigestHistory, pickFreshestObserverDigest, + projectSessionObserverDigest, resolveChatPaneObserverRunId, } from "../../lib/observer-digest.ts"; export { isWorkboardEnabledInConfigSnapshot } from "../../lib/plugin-activation.ts"; @@ -175,6 +176,7 @@ export { sendSessionObserverVisibility } from "./chat-observer.ts"; export { applySelectedSessionProjection, dismissChatError, + resolveChatArtifactDownload, resolveAssistantAttachmentAuthToken, SessionParticipationTracker, } from "./chat-pane-state.ts"; diff --git a/ui/src/pages/chat/chat-pane-render.ts b/ui/src/pages/chat/chat-pane-render.ts index 62ccbb526e7c..1932bc8e61ea 100644 --- a/ui/src/pages/chat/chat-pane-render.ts +++ b/ui/src/pages/chat/chat-pane-render.ts @@ -21,6 +21,7 @@ import { openSessionWorkspaceFile, parseCatalogSessionKey, pickFreshestObserverDigest, + projectSessionObserverDigest, readPresenceEntries, refreshChatCommands, refreshPageChat, @@ -30,6 +31,7 @@ import { resolveActiveRunOutputTokens, resolveChatProjectionRunId, resolveAssistantAttachmentAuthToken, + resolveChatArtifactDownload, resolveChatAgentId, resolveChatAvatarUrl, resolveControlUiFollowUpMode, @@ -54,7 +56,6 @@ import { workspaceResultConflictFromPlacement, type BoardViewCallbacks, type ChatProps, - type SessionObserverDigest, type SidebarSide, type SidebarSlotId, } from "./chat-pane-deps.ts"; @@ -79,19 +80,10 @@ export class ChatPaneRender extends ChatPaneHeaderRender { return html`
`; } const selectedSession = selectedChatSessionRow(state); - const projectedObserverDigest: SessionObserverDigest | null = selectedSession?.observerDigest - ? { - sessionKey: selectedSession.key, - ...(selectedSession.observerDigest.agentId - ? { agentId: selectedSession.observerDigest.agentId } - : {}), - runId: selectedSession.observerDigest.runId, - revision: selectedSession.observerDigest.revision, - updatedAt: selectedSession.observerDigest.updatedAt, - headline: selectedSession.observerDigest.headline, - health: selectedSession.observerDigest.health, - } - : null; + const projectedObserverDigest = projectSessionObserverDigest( + selectedSession?.key ?? state.sessionKey, + selectedSession?.observerDigest, + ); const observerDigest = pickFreshestObserverDigest( state.observerDigest, projectedObserverDigest, @@ -559,6 +551,7 @@ export class ChatPaneRender extends ChatPaneHeaderRender { allowExternalEmbedUrls: state.allowExternalEmbedUrls, chatMessageMaxWidth: state.settings.chatMessageMaxWidth, assistantAttachmentAuthToken: resolveAssistantAttachmentAuthToken(state as never), + resolveArtifactDownload: (params) => resolveChatArtifactDownload(state, params), basePath: state.basePath, gatewayUrl: state.settings.gatewayUrl, }; diff --git a/ui/src/pages/chat/chat-pane-state.test.ts b/ui/src/pages/chat/chat-pane-state.test.ts index 82e4b36b54b9..988db8b7b227 100644 --- a/ui/src/pages/chat/chat-pane-state.test.ts +++ b/ui/src/pages/chat/chat-pane-state.test.ts @@ -2,7 +2,11 @@ import { describe, expect, it } from "vitest"; import type { SessionsListResult } from "../../api/types.ts"; import { reconcileSessionHistory } from "../../lib/sessions/reconcile.ts"; import { areUiSessionKeysEquivalent } from "../../lib/sessions/session-key.ts"; -import { applySelectedSessionProjection, SessionParticipationTracker } from "./chat-pane-state.ts"; +import { + applySelectedSessionProjection, + resolveChatArtifactDownload, + SessionParticipationTracker, +} from "./chat-pane-state.ts"; function projectionState(): Parameters[0] { return { @@ -80,6 +84,44 @@ describe("applySelectedSessionProjection", () => { }); }); +describe("resolveChatArtifactDownload", () => { + it("returns a trimmed ticket without exposing a gateway bearer credential", async () => { + const requests: Array<{ method: string; params: unknown }> = []; + const result = await resolveChatArtifactDownload( + { + connected: true, + client: { + request: async (method: string, params: unknown) => { + requests.push({ method, params }); + return { + artifact: { + id: "artifact-1", + type: "image", + title: "image", + download: { mode: "url" }, + }, + url: " /api/chat/media/outgoing/main/image/full?mediaTicket=ticket ", + expiresAt: " 2026-07-28T00:00:00.000Z ", + }; + }, + } as never, + }, + { sessionKey: "agent:main:main", artifactId: "artifact-1" }, + ); + + expect(requests).toEqual([ + { + method: "artifacts.download", + params: { sessionKey: "agent:main:main", artifactId: "artifact-1" }, + }, + ]); + expect(result).toEqual({ + url: "/api/chat/media/outgoing/main/image/full?mediaTicket=ticket", + expiresAt: "2026-07-28T00:00:00.000Z", + }); + }); +}); + describe("SessionParticipationTracker", () => { const resolve = ( tracker: SessionParticipationTracker, diff --git a/ui/src/pages/chat/chat-pane-state.ts b/ui/src/pages/chat/chat-pane-state.ts index 752fb267eb38..628cde3a8b1b 100644 --- a/ui/src/pages/chat/chat-pane-state.ts +++ b/ui/src/pages/chat/chat-pane-state.ts @@ -1,4 +1,5 @@ -import type { GatewaySessionRow } from "../../api/types.ts"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import type { ArtifactDownloadResult, GatewaySessionRow } from "../../api/types.ts"; import { resolveControlUiAuthToken } from "../../app/control-ui-auth.ts"; type SelectedSessionProjectionState = { @@ -83,6 +84,25 @@ export function resolveAssistantAttachmentAuthToken(state: { return resolveControlUiAuthToken(state); } +export async function resolveChatArtifactDownload( + state: { connected: boolean; client?: GatewayBrowserClient | null }, + params: { sessionKey: string; artifactId: string }, +): Promise<{ url: string; expiresAt?: string } | null> { + if (!state.connected || !state.client) { + return null; + } + const result = await state.client.request( + "artifacts.download", + params, + ); + const url = typeof result?.url === "string" ? result.url.trim() : ""; + if (!url) { + return null; + } + const expiresAt = typeof result?.expiresAt === "string" ? result.expiresAt.trim() : undefined; + return { url, ...(expiresAt ? { expiresAt } : {}) }; +} + export function dismissChatError(state: { chatError?: string | null; lastError: string | null; diff --git a/ui/src/pages/chat/chat-view.ts b/ui/src/pages/chat/chat-view.ts index 977fc0ead15d..82d1c7617f2d 100644 --- a/ui/src/pages/chat/chat-view.ts +++ b/ui/src/pages/chat/chat-view.ts @@ -43,6 +43,7 @@ import { import type { ChatComposerDisabledBanner } from "./components/chat-composer-types.ts"; import { isChatRunWorking, renderChatComposer } from "./components/chat-composer.ts"; import { inlineChatImageFromEvent, openInlineChatImage } from "./components/chat-image-lightbox.ts"; +import type { ArtifactDownloadResolver } from "./components/chat-message-media.ts"; import { renderChatPullRequests } from "./components/chat-pull-requests.ts"; import type { SessionRailMode } from "./components/chat-session-rail.ts"; import { renderChatSessionSuggestions } from "./components/chat-session-suggestions.ts"; @@ -176,6 +177,7 @@ export type ChatProps = { userAvatar?: string | null; localMediaPreviewRoots?: string[]; assistantAttachmentAuthToken?: string | null; + resolveArtifactDownload?: ArtifactDownloadResolver; autoExpandToolCalls?: boolean; attachments?: ChatAttachment[]; getAttachments?: () => ChatAttachment[]; @@ -331,6 +333,7 @@ export function renderChat(props: ChatProps) { fullMessageAgentId: props.fullMessageAgentId, localMediaPreviewRoots: props.localMediaPreviewRoots, assistantAttachmentAuthToken: props.assistantAttachmentAuthToken, + resolveArtifactDownload: props.resolveArtifactDownload, canvasPluginSurfaceUrl: props.canvasPluginSurfaceUrl, embedSandboxMode: props.embedSandboxMode, allowExternalEmbedUrls: props.allowExternalEmbedUrls, diff --git a/ui/src/pages/chat/components/chat-message-bubble.ts b/ui/src/pages/chat/components/chat-message-bubble.ts index 8ac4e080688e..83625d365f7b 100644 --- a/ui/src/pages/chat/components/chat-message-bubble.ts +++ b/ui/src/pages/chat/components/chat-message-bubble.ts @@ -51,6 +51,7 @@ import { extractTranscriptAttachments, schedulePairingQrExpiryRefresh, type AttachmentItem, + type ArtifactDownloadResolver, type PairingQrExpiryNotice, } from "./chat-message-media.ts"; import type { SidebarContent } from "./chat-sidebar.ts"; @@ -176,6 +177,7 @@ export function renderGroupedMessage( basePath?: string; localMediaPreviewRoots?: readonly string[]; assistantAttachmentAuthToken?: string | null; + resolveArtifactDownload?: ArtifactDownloadResolver; onAssistantAttachmentLoaded?: () => void; onRequestOpenImage?: () => number; onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void; @@ -209,6 +211,7 @@ export function renderGroupedMessage( onRequestUpdate: opts.onRequestUpdate, onRequestOpenImage: opts.onRequestOpenImage, onOpenImage: opts.onOpenImage, + resolveArtifactDownload: opts.resolveArtifactDownload, }; schedulePairingQrExpiryRefresh(messageKey, message, opts.onRequestUpdate); const images = resolveRenderableMessageImages(extractImages(message), imageRenderOptions); diff --git a/ui/src/pages/chat/components/chat-message-group.ts b/ui/src/pages/chat/components/chat-message-group.ts index 2516c0e40db6..860eb87cb7f4 100644 --- a/ui/src/pages/chat/components/chat-message-group.ts +++ b/ui/src/pages/chat/components/chat-message-group.ts @@ -24,6 +24,7 @@ import { resolveMessageActionDetails, type MessageReplyTarget, } from "./chat-message-markdown.ts"; +import type { ArtifactDownloadResolver } from "./chat-message-media.ts"; import { renderStreamGroupParts, type StreamGroupOptions, @@ -76,6 +77,7 @@ type RenderMessageGroupOptions = { basePath?: string; localMediaPreviewRoots?: readonly string[]; assistantAttachmentAuthToken?: string | null; + resolveArtifactDownload?: ArtifactDownloadResolver; canvasPluginSurfaceUrl?: string | null; embedSandboxMode?: EmbedSandboxMode; allowExternalEmbedUrls?: boolean; @@ -126,6 +128,7 @@ function buildGroupedMessageRenderOptions( basePath: opts.basePath, localMediaPreviewRoots: opts.localMediaPreviewRoots, assistantAttachmentAuthToken: opts.assistantAttachmentAuthToken, + resolveArtifactDownload: opts.resolveArtifactDownload, embedSandboxMode: opts.embedSandboxMode, allowExternalEmbedUrls: opts.allowExternalEmbedUrls, }; diff --git a/ui/src/pages/chat/components/chat-message-images.ts b/ui/src/pages/chat/components/chat-message-images.ts index 5ab0d439d0e5..83078bca51b2 100644 --- a/ui/src/pages/chat/components/chat-message-images.ts +++ b/ui/src/pages/chat/components/chat-message-images.ts @@ -67,10 +67,11 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image const requestVersion = opts?.onRequestOpenImage?.(); const managedSource = isManagedOutgoingImageSource(img.displayUrl); const cacheKey = managedSource - ? resolveManagedOutgoingImageBlobUrlCacheKey(img.displayUrl, opts) + ? resolveManagedOutgoingImageBlobUrlCacheKey(img.displayUrl, opts, img.artifactId) : undefined; const previewIsCurrent = - !managedSource || readManagedOutgoingImageBlobUrl(img.displayUrl, opts) === previewUrl; + !managedSource || + readManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId) === previewUrl; if (previewIsCurrent) { const release = opts?.onOpenImage && cacheKey ? retainManagedImageBlobUrl(cacheKey) : undefined; @@ -82,7 +83,7 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image // Re-resolve before opening so the modal never receives a revoked URL. if (!opts?.onOpenImage) { const pendingWindow = reserveExternalWindowForDeferredNavigation(); - void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts) + void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId) .then((freshUrl) => { const safeUrl = freshUrl ? resolveSafeExternalUrl(freshUrl, window.location.href, { allowDataImage: true }) @@ -98,7 +99,7 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image .catch(() => pendingWindow?.close()); return; } - void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts) + void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId) .then((freshUrl) => { if (!freshUrl) { return; @@ -133,12 +134,14 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image if (!isManagedOutgoingImageSource(img.displayUrl)) { return renderImageElement(img, img.displayUrl); } - const preview = resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts).then((previewUrl) => { - if (!previewUrl) { - return nothing; - } - return renderImageElement(img, previewUrl); - }); + const preview = resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId).then( + (previewUrl) => { + if (!previewUrl) { + return nothing; + } + return renderImageElement(img, previewUrl); + }, + ); return until(preview, nothing); }; @@ -175,24 +178,29 @@ function resolveManagedOutgoingImageRequesterSessionKey(source: string): string function resolveManagedOutgoingImageBlobUrlCacheKey( source: string, opts?: ImageRenderOptions, + artifactId?: string, ): string { const authToken = opts?.authToken?.trim() ?? ""; - return `${source}::${authToken}`; + return `${source}::${authToken}::${artifactId?.trim() ?? ""}`; } function readManagedOutgoingImageBlobUrl( source: string, opts?: ImageRenderOptions, + artifactId?: string, ): string | undefined { - return readManagedImageBlobUrl(resolveManagedOutgoingImageBlobUrlCacheKey(source, opts)); + return readManagedImageBlobUrl( + resolveManagedOutgoingImageBlobUrlCacheKey(source, opts, artifactId), + ); } async function resolveManagedOutgoingImageBlobUrl( source: string, opts?: ImageRenderOptions, + artifactId?: string, ): Promise { const authToken = opts?.authToken?.trim() ?? ""; - const cacheKey = resolveManagedOutgoingImageBlobUrlCacheKey(source, opts); + const cacheKey = resolveManagedOutgoingImageBlobUrlCacheKey(source, opts, artifactId); const cached = readManagedImageBlobUrl(cacheKey); if (cached) { return cached; @@ -204,11 +212,18 @@ async function resolveManagedOutgoingImageBlobUrl( if (!pending) { pending = (async () => { const requesterSessionKey = resolveManagedOutgoingImageRequesterSessionKey(source); + const artifactDownload = + requesterSessionKey && artifactId && opts?.resolveArtifactDownload + ? await opts + .resolveArtifactDownload({ sessionKey: requesterSessionKey, artifactId }) + .catch(() => null) + : null; + const requestUrl = artifactDownload?.url ?? source; const headers = new Headers({ Accept: "image/*" }); - if (authToken) { + if (!artifactDownload && authToken) { headers.set("Authorization", `Bearer ${authToken}`); } - if (requesterSessionKey) { + if (!artifactDownload && requesterSessionKey) { headers.set("x-openclaw-requester-session-key", requesterSessionKey); } const controller = new AbortController(); @@ -220,7 +235,7 @@ async function resolveManagedOutgoingImageBlobUrl( try { // Managed media is a Gateway API at the origin root. Rebasing it under // the Control UI mount path serves the HTML shell instead of image bytes. - const res = await fetch(source, { + const res = await fetch(requestUrl, { method: "GET", headers, credentials: "same-origin", diff --git a/ui/src/pages/chat/components/chat-message-media.ts b/ui/src/pages/chat/components/chat-message-media.ts index ab07bf2e514a..fe9ce90e21c6 100644 --- a/ui/src/pages/chat/components/chat-message-media.ts +++ b/ui/src/pages/chat/components/chat-message-media.ts @@ -17,12 +17,18 @@ const pairingQrExpiryRefreshTimers = new Map Promise<{ url: string; expiresAt?: string } | null>; + export type ImageRenderOptions = { localMediaPreviewRoots?: readonly string[]; basePath?: string; @@ -30,6 +36,7 @@ export type ImageRenderOptions = { onRequestUpdate?: () => void; onRequestOpenImage?: () => number; onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void; + resolveArtifactDownload?: ArtifactDownloadResolver; }; export type RenderableImageBlock = ImageBlock & { @@ -209,6 +216,7 @@ export function extractImages(message: unknown): ImageBlock[] { // Handle source object format from optimistic user sends. const source = b.source as Record | undefined; const imageMeta = { + artifactId: typeof b.artifactId === "string" ? b.artifactId : undefined, alt: typeof b.alt === "string" ? b.alt : undefined, openUrl: typeof b.openUrl === "string" ? b.openUrl : undefined, width: typeof b.width === "number" ? b.width : undefined, diff --git a/ui/src/pages/chat/components/chat-message.test.ts b/ui/src/pages/chat/components/chat-message.test.ts index 8c1a58f742da..be9afdee3cbb 100644 --- a/ui/src/pages/chat/components/chat-message.test.ts +++ b/ui/src/pages/chat/components/chat-message.test.ts @@ -3638,6 +3638,52 @@ describe("grouped chat rendering", () => { activeItem?.release?.(); }); + it("prefers an artifact ticket without forwarding the gateway bearer", async () => { + const artifactId = `artifact_managed_image_${crypto.randomUUID()}`; + const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const ticketedUrl = `${managedChatImageUrl}?mediaTicket=ticket`; + const resolveArtifactDownload = vi.fn(async () => ({ + url: ticketedUrl, + expiresAt: "2026-07-28T05:00:00.000Z", + })); + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + expect(url).toBe(ticketedUrl); + const headers = new Headers(init?.headers); + expect(headers.get("Authorization")).toBeNull(); + expect(headers.get("x-openclaw-requester-session-key")).toBeNull(); + return { ok: true, blob: async () => new Blob(["png"], { type: "image/png" }) }; + }); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + const container = document.createElement("div"); + renderAssistantMessage( + container, + { + role: "assistant", + content: [ + { + type: "image", + artifactId, + url: managedChatImageUrl, + alt: "Ticketed image", + }, + ], + timestamp: Date.now(), + }, + { + showToolCalls: false, + assistantAttachmentAuthToken: "must-not-be-forwarded", + resolveArtifactDownload, + }, + ); + + await vi.waitFor(() => expect(container.querySelector(".chat-message-image")).not.toBeNull()); + expect(resolveArtifactDownload).toHaveBeenCalledWith({ + sessionKey: "agent:main:main", + artifactId, + }); + }); + it("aborts a stalled managed outgoing image fetch after the deadline", async () => { vi.useFakeTimers(); const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; diff --git a/ui/src/pages/chat/components/chat-thread.ts b/ui/src/pages/chat/components/chat-thread.ts index d3fc09400b35..2390bde322b0 100644 --- a/ui/src/pages/chat/components/chat-thread.ts +++ b/ui/src/pages/chat/components/chat-thread.ts @@ -75,6 +75,7 @@ import { getToolTitlesVersion } from "../tool-titles.ts"; import { renderBackgroundTasksStatusRow } from "./chat-background-tasks-status.ts"; import type { BackgroundTasksProps } from "./chat-background-tasks.ts"; import { renderChatDivider, renderChatNotice } from "./chat-divider.ts"; +import type { ArtifactDownloadResolver } from "./chat-message-media.ts"; import { dismissConfirmedActionPopovers, getAssistantAttachmentAvailabilityRenderVersion, @@ -149,6 +150,7 @@ type ChatThreadProps = { fullMessageAgentId?: string; localMediaPreviewRoots?: string[]; assistantAttachmentAuthToken?: string | null; + resolveArtifactDownload?: ArtifactDownloadResolver; canvasPluginSurfaceUrl?: string | null; embedSandboxMode?: EmbedSandboxMode; allowExternalEmbedUrls?: boolean; @@ -1545,6 +1547,7 @@ function renderChatThreadContents( basePath: props.basePath, localMediaPreviewRoots: props.localMediaPreviewRoots ?? [], assistantAttachmentAuthToken: props.assistantAttachmentAuthToken ?? null, + resolveArtifactDownload: props.resolveArtifactDownload, canvasPluginSurfaceUrl: props.canvasPluginSurfaceUrl, embedSandboxMode: props.embedSandboxMode ?? "scripts", allowExternalEmbedUrls: props.allowExternalEmbedUrls ?? false,