fix: generated images do not appear in native chat (#115042)

* fix(chat): render managed images in native clients

Co-authored-by: Francesco Giannicola <francescogiannicola1@gmail.com>

* chore: keep release changelog owned

* refactor(macos): split managed image loading

* test(apple): prove managed image header policy

* test(native): stabilize managed image proof

* test(apple): satisfy strict concurrency checks

---------

Co-authored-by: Francesco Giannicola <francescogiannicola1@gmail.com>
This commit is contained in:
Peter Steinberger
2026-07-28 04:39:53 -04:00
committed by GitHub
parent c141496217
commit 4b05d83035
61 changed files with 2638 additions and 440 deletions
File diff suppressed because it is too large Load Diff
@@ -1305,6 +1305,8 @@ class MainViewModel private constructor(
failedResource: ChatWidgetResource?,
) = ensureRuntime().resolveInlineWidgetResource(path, failedResource)
internal suspend fun loadChatImageArtifact(artifactId: String) = ensureRuntime().loadChatImageArtifact(artifactId)
fun requestCanvasRehydrate(source: String = "screen_tab") {
ensureRuntime().requestCanvasRehydrate(source = source, force = true)
}
@@ -4910,6 +4910,8 @@ class NodeRuntime private constructor(
}
}
internal suspend fun loadChatImageArtifact(artifactId: String) = chat.loadImageArtifact(artifactId)
fun loadChat(
sessionKey: String,
ownerAgentId: String? = null,
@@ -1,6 +1,7 @@
package ai.openclaw.app.chat
import ai.openclaw.app.GatewayModelSummary
import ai.openclaw.app.gateway.GatewayLoadedImage
import ai.openclaw.app.gateway.GatewayRequestDefinitiveFailure
import ai.openclaw.app.gateway.GatewayRequestNotEnqueued
import ai.openclaw.app.gateway.GatewayRequestOutcomeUnknown
@@ -56,6 +57,8 @@ internal const val SESSION_LIST_FETCH_LIMIT = 200
private val QUESTION_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
private val SWARM_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
private const val SESSION_EDITOR_MAX_BASE64_CHARS = ((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 2) / 3) * 4
private val MANAGED_IMAGE_PATH_REGEX =
Regex("^/api/chat/media/outgoing/[^/]+/([0-9a-fA-F-]{36})/full(?:\\?.*)?$")
internal fun chatOutboxQueueFailureText(): NativeText = ChatController.queueFailureText()
@@ -111,6 +114,12 @@ class ChatController internal constructor(
private val cacheScope: () -> ChatCacheScope? = { null },
private val currentDefaultAgentId: () -> String? = { "main" },
private val currentDefaultAgentRevision: () -> Long = { 0L },
private val loadGatewayImageArtifact: suspend (
gatewayId: String?,
sessionKey: String,
agentId: String?,
artifactId: String,
) -> GatewayLoadedImage? = { _, _, _, _ -> null },
private val commandOutbox: ChatCommandOutbox? = null,
private val recordModelRecent: (String) -> Unit = {},
private val onSessionDeleted: (ChatSessionDeletion) -> Unit = {},
@@ -142,12 +151,26 @@ class ChatController internal constructor(
cacheScope = cacheScope,
currentDefaultAgentId = currentDefaultAgentId,
currentDefaultAgentRevision = currentDefaultAgentRevision,
loadGatewayImageArtifact = { gatewayId, sessionKey, agentId, artifactId ->
session.loadImageArtifact(gatewayId, sessionKey, agentId, artifactId)
},
commandOutbox = commandOutbox,
recordModelRecent = recordModelRecent,
onSessionDeleted = onSessionDeleted,
onOfflineDefaultAgentRestored = onOfflineDefaultAgentRestored,
)
suspend fun loadImageArtifact(artifactId: String): GatewayLoadedImage? {
val normalizedArtifactId = artifactId.trim().takeIf(String::isNotEmpty) ?: return null
val sessionKey = normalizeRequestedSessionKey(_sessionKey.value)
return loadGatewayImageArtifact(
currentCacheScope()?.gatewayId,
sessionKey,
resolveAgentIdForSessionKey(sessionKey),
normalizedArtifactId,
)
}
private var appliedMainSessionKey = "main"
private val cacheMutationMutex = Mutex()
private val defaultAgentPersistenceMutex = Mutex()
@@ -6368,10 +6391,18 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
"image", "audio" -> {
val type = obj["type"].asStringOrNull() ?: "image"
val inlineContent = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() }
val url = obj["url"].asStringOrNull()
ChatMessageContent(
type = type,
mimeType = obj["mimeType"].asStringOrNull(),
fileName = obj["fileName"].asStringOrNull(),
artifactId = obj["artifactId"].asStringOrNull() ?: managedImageArtifactId(url),
url = url,
openUrl = obj["openUrl"].asStringOrNull(),
alt = obj["alt"].asStringOrNull(),
width = obj["width"].asLongOrNull()?.toInt(),
height = obj["height"].asLongOrNull()?.toInt(),
sizeBytes = obj["sizeBytes"].asLongOrNull(),
base64 = inlineContent?.takeIf { type != "image" || it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
)
}
@@ -6415,6 +6446,16 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
}
}
internal fun managedImageArtifactId(rawUrl: String?): String? {
val match =
rawUrl
?.trim()
?.let(MANAGED_IMAGE_PATH_REGEX::matchEntire)
?: return null
val attachmentId = runCatching { UUID.fromString(match.groupValues[1]).toString() }.getOrNull() ?: return null
return "artifact_managed_image_$attachmentId"
}
internal fun parseChatMessageContents(obj: JsonObject): List<ChatMessageContent> {
val content =
obj["content"].asArrayOrNull()?.mapNotNull(::parseChatMessageContent)
@@ -6655,6 +6696,9 @@ private fun messageContentIdentityKey(message: ChatMessage): String? {
?.lowercase()
.orEmpty(),
part.fileName?.trim().orEmpty(),
part.artifactId?.trim().orEmpty(),
part.url?.trim().orEmpty(),
part.openUrl?.trim().orEmpty(),
part.base64
?.hashCode()
?.toString()
@@ -63,13 +63,20 @@ data class ChatTranscriptAnchorState(
)
/**
* One content part in a chat message; binary parts carry base64 plus their MIME metadata.
* One content part in a chat message; images carry either bounded base64 or a managed artifact reference.
*/
data class ChatMessageContent(
val type: String = "text",
val text: String? = null,
val mimeType: String? = null,
val fileName: String? = null,
val artifactId: String? = null,
val url: String? = null,
val openUrl: String? = null,
val alt: String? = null,
val width: Int? = null,
val height: Int? = null,
val sizeBytes: Long? = null,
val base64: String? = null,
val durationMs: Long? = null,
val widget: ChatWidgetPreview? = null,
@@ -6,6 +6,7 @@ import androidx.room.Insert
import androidx.room.OnConflictStrategy
import androidx.room.Query
import androidx.room.withTransaction
import kotlinx.serialization.Serializable
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.json.Json
@@ -17,6 +18,21 @@ internal const val MAX_CACHED_SESSIONS = 50
/** Upper bound of cached transcript rows per session; only the newest messages are kept. */
internal const val MAX_CACHED_MESSAGES_PER_SESSION = 200
@Serializable
private data class CachedMessageContent(
val type: String,
val text: String? = null,
val mimeType: String? = null,
val fileName: String? = null,
val artifactId: String? = null,
val url: String? = null,
val openUrl: String? = null,
val alt: String? = null,
val width: Int? = null,
val height: Int? = null,
val sizeBytes: Long? = null,
)
/**
* Read-only offline cache of chat sessions and transcripts.
*
@@ -92,7 +108,7 @@ internal data class CachedMessageEntity(
val sessionKey: String,
val rowOrder: Int,
val role: String,
// JSON array of text part strings; attachments/binary parts are never persisted.
// JSON array of text and managed-image references; attachment bytes are never persisted.
val textPartsJson: String,
val timestampMs: Long?,
// Kept so live history reconciliation can match cached rows by identity key.
@@ -232,7 +248,8 @@ class RoomChatTranscriptCache internal constructor(
private val database: GatewayCacheDatabase,
) : ChatTranscriptCache {
private val json = Json
private val textPartsSerializer = ListSerializer(String.serializer())
private val cachedContentSerializer = ListSerializer(CachedMessageContent.serializer())
private val legacyTextPartsSerializer = ListSerializer(String.serializer())
override suspend fun loadLastDefaultAgentId(gatewayId: String): String? {
val gateway = scopedGatewayId(gatewayId) ?: return null
@@ -287,7 +304,22 @@ class RoomChatTranscriptCache internal constructor(
ChatMessage(
id = UUID.randomUUID().toString(),
role = role,
content = decodeTextParts(row.textPartsJson).map { ChatMessageContent(type = "text", text = it) },
content =
decodeCachedContent(row.textPartsJson).map { part ->
ChatMessageContent(
type = part.type,
text = part.text,
mimeType = part.mimeType,
fileName = part.fileName,
artifactId = part.artifactId,
url = part.url,
openUrl = part.openUrl,
alt = part.alt,
width = part.width,
height = part.height,
sizeBytes = part.sizeBytes,
)
},
timestampMs = row.timestampMs,
idempotencyKey = row.idempotencyKey,
// Canonical tree ids stay live-only; cached rows regain actions after history refresh.
@@ -367,23 +399,44 @@ class RoomChatTranscriptCache internal constructor(
val gateway = scopedGatewayId(gatewayId) ?: return
val agent = scopedAgentId(agentId) ?: return
val key = sessionKey.trim().takeIf { it.isNotEmpty() } ?: return
// Text rows only: attachment/binary parts are dropped, and messages without any text are skipped.
// Persist small managed-image references, never attachment bytes. This keeps generated images
// visible offline without turning the disposable transcript cache into a binary store.
val rows =
messages
.mapNotNull { message ->
val role = normalizeVisibleChatMessageRole(message.role) ?: return@mapNotNull null
val textParts = message.content.filter { it.type == "text" }.mapNotNull { it.text }
if (textParts.isEmpty()) return@mapNotNull null
Triple(message, role, textParts)
val content =
message.content.mapNotNull { part ->
when {
part.type == "text" && !part.text.isNullOrBlank() ->
CachedMessageContent(type = "text", text = part.text)
part.type == "image" && !part.artifactId.isNullOrBlank() && !part.url.isNullOrBlank() ->
CachedMessageContent(
type = "image",
mimeType = part.mimeType,
fileName = part.fileName,
artifactId = part.artifactId,
url = part.url,
openUrl = part.openUrl,
alt = part.alt,
width = part.width,
height = part.height,
sizeBytes = part.sizeBytes,
)
else -> null
}
}
if (content.isEmpty()) return@mapNotNull null
Triple(message, role, content)
}.takeLast(MAX_CACHED_MESSAGES_PER_SESSION)
.mapIndexed { index, (message, role, textParts) ->
.mapIndexed { index, (message, role, content) ->
CachedMessageEntity(
gatewayId = gateway,
agentId = agent,
sessionKey = key,
rowOrder = index,
role = role,
textPartsJson = json.encodeToString(textPartsSerializer, textParts),
textPartsJson = json.encodeToString(cachedContentSerializer, content),
timestampMs = message.timestampMs,
idempotencyKey = message.idempotencyKey,
)
@@ -452,5 +505,12 @@ class RoomChatTranscriptCache internal constructor(
private fun scopedAgentId(agentId: String): String? = agentId.trim().takeIf { it.isNotEmpty() }
private fun decodeTextParts(encoded: String): List<String> = runCatching { json.decodeFromString(textPartsSerializer, encoded) }.getOrDefault(emptyList())
private fun decodeCachedContent(encoded: String): List<CachedMessageContent> =
runCatching { json.decodeFromString(cachedContentSerializer, encoded) }.getOrElse {
// Offline transcript browsing is shipped behavior. Keep the previous string-array rows
// readable until a live history refresh naturally rewrites this disposable cache entry.
runCatching { json.decodeFromString(legacyTextPartsSerializer, encoded) }
.getOrDefault(emptyList())
.map { CachedMessageContent(type = "text", text = it) }
}
}
@@ -33,6 +33,8 @@ import okhttp3.Request
import okhttp3.Response
import okhttp3.WebSocket
import okhttp3.WebSocketListener
import okio.Buffer
import java.net.URI
import java.util.Locale
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
@@ -53,6 +55,11 @@ data class GatewayClientInfo(
val modelIdentifier: String?,
)
data class GatewayLoadedImage(
val bytes: ByteArray,
val mimeType: String,
)
/**
* Role, scopes, commands, and permission snapshot sent with the connect frame.
*/
@@ -616,6 +623,38 @@ class GatewaySession(
throw GatewayRequestRejected(res.error ?: ErrorShape("UNAVAILABLE", "request failed"))
}
suspend fun loadImageArtifact(
expectedEndpointStableId: String?,
sessionKey: String,
agentId: String?,
artifactId: String,
): GatewayLoadedImage? {
val conn = readyConnection(expectedEndpointStableId) ?: return null
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(sessionKey))
agentId?.trim()?.takeIf(String::isNotEmpty)?.let { put("agentId", JsonPrimitive(it)) }
put("artifactId", JsonPrimitive(artifactId))
}
val response = conn.request(GatewayMethod.ArtifactsDownload.rawValue, params, timeoutMs = 15_000)
if (!response.ok) {
throw GatewayRequestRejected(response.error ?: ErrorShape("UNAVAILABLE", "artifact download failed"))
}
val ticketedPath =
response.payloadJson
?.let(::parseJsonOrNull)
.asObjectOrNull()
?.get("url")
.asStringOrNull()
?.trim()
?.takeIf(String::isNotEmpty)
?: return null
val loaded = conn.loadTicketedImage(ticketedPath) ?: return null
return synchronized(lifecycleLock) {
loaded.takeIf { currentConnection === conn && conn.isReady() }
}
}
internal suspend fun requestForEndpoint(
expectedEndpointStableId: String,
method: String,
@@ -830,6 +869,46 @@ class GatewaySession(
}
}
suspend fun loadTicketedImage(ticketedPath: String): GatewayLoadedImage? =
withContext(Dispatchers.IO) {
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return@withContext null
val rawPath = uri.rawPath ?: return@withContext null
val rawQuery = uri.rawQuery ?: return@withContext null
if (uri.isAbsolute || uri.rawAuthority != null || uri.rawFragment != null) return@withContext null
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !rawQuery.contains("mediaTicket=")) {
return@withContext null
}
val scheme = if (tlsConfig != null) "https" else "http"
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
val request = Request.Builder().url(url).header("Accept", "image/*")
if (tlsConfig != null) {
for ((name, value) in GatewayCustomHeaders.sanitized(customHeadersProvider?.invoke(endpoint.stableId).orEmpty())) {
request.header(name, value)
}
}
val call = client.newCall(request.build())
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
call.execute().use { response ->
if (!response.isSuccessful) return@withContext null
val body = response.body
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
if (!mimeType.startsWith("image/")) return@withContext null
val maximumBytes = 12L * 1024L * 1024L
val declaredLength = body.contentLength()
if (declaredLength > maximumBytes) return@withContext null
val buffer = Buffer()
val source = body.source()
var total = 0L
while (true) {
val read = source.read(buffer, minOf(8192L, maximumBytes + 1L - total))
if (read == -1L) break
total += read
if (total > maximumBytes) return@withContext null
}
GatewayLoadedImage(bytes = buffer.readByteArray(), mimeType = mimeType)
}
}
@OptIn(DelicateCoroutinesApi::class)
suspend fun sendRequestFrame(
method: String,
@@ -171,11 +171,18 @@ internal fun decodeBase64Bitmap(
maxDimension: Int = CHAT_DECODE_MAX_DIMENSION,
): Bitmap? {
if (base64.length > CHAT_IMAGE_MAX_BASE64_CHARS) return null
val cacheKey = "$maxDimension:${base64.length}:${base64.hashCode()}"
decodedBitmapCache.get(cacheKey)?.let { return it }
val bytes = Base64.decode(base64, Base64.DEFAULT)
if (bytes.isEmpty()) return null
return decodeImageBytes(bytes, maxDimension)
}
/** Decodes already-authorized image bytes without base64 expansion. */
internal fun decodeImageBytes(
bytes: ByteArray,
maxDimension: Int = CHAT_DECODE_MAX_DIMENSION,
): Bitmap? {
if (bytes.isEmpty() || bytes.size > 12 * 1024 * 1024) return null
val cacheKey = "$maxDimension:${bytes.size}:${bytes.contentHashCode()}"
decodedBitmapCache.get(cacheKey)?.let { return it }
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds)
@@ -10,6 +10,7 @@ import ai.openclaw.app.chat.MessageSpeechState
import ai.openclaw.app.chat.OUTBOX_BRANCH_CHANGED_ERROR
import ai.openclaw.app.chat.chatOutboxDisplayError
import ai.openclaw.app.chat.normalizeVisibleChatMessageRole
import ai.openclaw.app.gateway.GatewayLoadedImage
import ai.openclaw.app.i18n.nativeString
import ai.openclaw.app.i18n.nativeStringResource
import ai.openclaw.app.tools.ToolDisplayRegistry
@@ -83,6 +84,8 @@ import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.compose.ui.window.Dialog
import androidx.compose.ui.window.DialogProperties
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import java.util.Locale
private data class ChatBubbleStyle(
@@ -102,6 +105,8 @@ internal fun ChatMessageBubble(
onForkMessage: (String) -> Unit = {},
speechState: MessageSpeechState? = null,
onToggleListen: ((String, String) -> Unit)? = null,
imageResolverReady: Boolean = false,
loadImageArtifact: suspend (String) -> GatewayLoadedImage? = { null },
) {
val role = normalizeVisibleChatMessageRole(message.role) ?: return
val style = bubbleStyle(role)
@@ -111,7 +116,7 @@ internal fun ChatMessageBubble(
message.content.filter { part ->
when (part.type) {
"text" -> !part.text.isNullOrBlank()
"image" -> !part.base64.isNullOrBlank()
"image" -> !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank()
else -> part.isAudioAttachment()
}
}
@@ -138,7 +143,12 @@ internal fun ChatMessageBubble(
modifier = Modifier.fillMaxWidth(),
) {
ChatBubbleContainer(style = style, roleLabel = roleLabel(role)) {
ChatMessageBody(content = displayableContent, textColor = mobileText)
ChatMessageBody(
content = displayableContent,
textColor = mobileText,
imageResolverReady = imageResolverReady,
loadImageArtifact = loadImageArtifact,
)
ChatMessageLinkPreview(messageId = message.id, role = role, content = displayableContent)
messageSpeech?.let { speech ->
MessageSpeechIndicator(
@@ -223,6 +233,8 @@ private fun ChatBubbleContainer(
private fun ChatMessageBody(
content: List<ChatMessageContent>,
textColor: Color,
imageResolverReady: Boolean,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
) {
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
for (part in content) {
@@ -232,9 +244,17 @@ private fun ChatMessageBody(
ChatMarkdown(text = text, textColor = textColor)
}
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
part.type == "image" && !part.base64.isNullOrBlank() ->
ChatBase64Image(base64 = part.base64, mimeType = part.mimeType)
part.type == "image" && !part.artifactId.isNullOrBlank() ->
ChatManagedImage(
artifactId = part.artifactId,
label = part.alt?.takeIf(String::isNotBlank) ?: part.fileName ?: nativeString("Image"),
resolverReady = imageResolverReady,
loadImage = loadImageArtifact,
)
else -> {
val b64 = part.base64 ?: continue
ChatBase64Image(base64 = b64, mimeType = part.mimeType)
Text(part.fileName ?: nativeString("Attachment"), style = mobileCaption1, color = mobileTextSecondary)
}
}
}
@@ -583,75 +603,138 @@ internal fun ChatBase64Image(
mimeType: String?,
) {
val imageState = rememberBase64ImageState(base64)
var previewVisible by rememberSaveable(base64) { mutableStateOf(false) }
val image = imageState.image
if (image != null) {
Surface(
onClick = { previewVisible = true },
shape = RoundedCornerShape(10.dp),
border = BorderStroke(1.dp, mobileBorder),
color = mobileCardSurface,
modifier = Modifier.fillMaxWidth(),
ChatImagePreview(image = image, description = mimeType ?: nativeString("Attachment"), stateKey = base64)
} else if (imageState.failed) {
Text(nativeString("Unsupported attachment"), style = mobileCaption1, color = mobileTextSecondary)
}
}
@Composable
internal fun ChatManagedImage(
artifactId: String,
label: String,
resolverReady: Boolean,
loadImage: suspend (String) -> GatewayLoadedImage?,
) {
var image by remember(artifactId) { mutableStateOf<ImageBitmap?>(null) }
var failed by remember(artifactId) { mutableStateOf(false) }
var retryGeneration by rememberSaveable(artifactId) { mutableStateOf(0) }
LaunchedEffect(artifactId, resolverReady, retryGeneration) {
if (!resolverReady) {
failed = true
image = null
return@LaunchedEffect
}
failed = false
image = null
val loaded = runCatching { loadImage(artifactId) }.getOrNull()
image =
loaded?.let { value ->
withContext(Dispatchers.Default) { decodeImageBytes(value.bytes)?.asImageBitmap() }
}
failed = image == null
}
when {
image != null -> ChatImagePreview(image = checkNotNull(image), description = label, stateKey = artifactId)
failed ->
Surface(
onClick = { retryGeneration += 1 },
shape = RoundedCornerShape(10.dp),
border = BorderStroke(1.dp, mobileBorder),
color = mobileCardSurface,
modifier = Modifier.fillMaxWidth(),
) {
Text(
nativeString("Image unavailable · Tap to retry"),
modifier = Modifier.padding(12.dp),
style = mobileCaption1,
color = mobileTextSecondary,
)
}
else ->
Text(
nativeString("Loading image…"),
modifier = Modifier.padding(12.dp),
style = mobileCaption1,
color = mobileTextSecondary,
)
}
}
@Composable
private fun ChatImagePreview(
image: ImageBitmap,
description: String,
stateKey: String,
) {
var previewVisible by rememberSaveable(stateKey) { mutableStateOf(false) }
Surface(
onClick = { previewVisible = true },
shape = RoundedCornerShape(10.dp),
border = BorderStroke(1.dp, mobileBorder),
color = mobileCardSurface,
modifier = Modifier.fillMaxWidth(),
) {
Box {
Image(
bitmap = image,
contentDescription = description,
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
)
Surface(
modifier = Modifier.align(Alignment.BottomEnd).padding(8.dp).size(32.dp),
shape = CircleShape,
color = Color.Black.copy(alpha = 0.62f),
contentColor = Color.White,
) {
Box(contentAlignment = Alignment.Center) {
Icon(
imageVector = Icons.Default.OpenInFull,
contentDescription = nativeString("Open image preview"),
modifier = Modifier.size(17.dp),
)
}
}
}
}
if (previewVisible) {
Dialog(
onDismissRequest = { previewVisible = false },
properties = DialogProperties(usePlatformDefaultWidth = false),
) {
Box {
Box(
modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.96f)).clickable { previewVisible = false },
contentAlignment = Alignment.Center,
) {
Image(
bitmap = image,
contentDescription = mimeType ?: nativeString("Attachment"),
contentDescription = nativeString("Image preview"),
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
modifier = Modifier.fillMaxSize().padding(20.dp),
)
Surface(
modifier = Modifier.align(Alignment.BottomEnd).padding(8.dp).size(32.dp),
onClick = { previewVisible = false },
modifier = Modifier.align(Alignment.TopEnd).padding(16.dp).size(44.dp),
shape = CircleShape,
color = Color.Black.copy(alpha = 0.62f),
contentColor = Color.White,
) {
Box(contentAlignment = Alignment.Center) {
Icon(
imageVector = Icons.Default.OpenInFull,
contentDescription = nativeString("Open image preview"),
modifier = Modifier.size(17.dp),
imageVector = Icons.Default.Close,
contentDescription = nativeString("Close image preview"),
modifier = Modifier.size(22.dp),
)
}
}
}
}
if (previewVisible) {
Dialog(
onDismissRequest = { previewVisible = false },
properties = DialogProperties(usePlatformDefaultWidth = false),
) {
Box(
modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.96f)).clickable { previewVisible = false },
contentAlignment = Alignment.Center,
) {
Image(
bitmap = image,
contentDescription = nativeString("Image preview"),
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxSize().padding(20.dp),
)
Surface(
onClick = { previewVisible = false },
modifier = Modifier.align(Alignment.TopEnd).padding(16.dp).size(44.dp),
shape = CircleShape,
color = Color.Black.copy(alpha = 0.62f),
contentColor = Color.White,
) {
Box(contentAlignment = Alignment.Center) {
Icon(
imageVector = Icons.Default.Close,
contentDescription = nativeString("Close image preview"),
modifier = Modifier.size(22.dp),
)
}
}
}
}
}
} else if (imageState.failed) {
Text(nativeString("Unsupported attachment"), style = mobileCaption1, color = mobileTextSecondary)
}
}
@@ -32,6 +32,7 @@ import ai.openclaw.app.chat.questionsForSession
import ai.openclaw.app.chat.resolveChatComposerOwner
import ai.openclaw.app.chat.resolveGatewayDefaultAgentId
import ai.openclaw.app.currentAppLanguage
import ai.openclaw.app.gateway.GatewayLoadedImage
import ai.openclaw.app.i18n.NativeText
import ai.openclaw.app.i18n.joinedNativeText
import ai.openclaw.app.i18n.nativeString
@@ -739,6 +740,7 @@ fun ChatScreen(
speechState = messageSpeechState,
onToggleListen = viewModel::toggleChatMessageSpeech,
resolveInlineWidgetResource = viewModel::resolveInlineWidgetResource,
loadImageArtifact = viewModel::loadChatImageArtifact,
modifier = Modifier.weight(1f),
)
@@ -1255,6 +1257,7 @@ private fun ChatMessageList(
speechState: MessageSpeechState?,
onToggleListen: (String, String) -> Unit,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
modifier: Modifier = Modifier,
) {
val baseTimeline =
@@ -1323,6 +1326,7 @@ private fun ChatMessageList(
onToggleListen = onToggleListen,
inlineWidgetResolverReady = healthOk,
resolveInlineWidgetResource = resolveInlineWidgetResource,
loadImageArtifact = loadImageArtifact,
)
is ChatTimelineItem.OutboxCommand ->
ChatOutboxBubble(
@@ -1366,6 +1370,7 @@ private fun ChatMessageList(
onToggleListen = onToggleListen,
inlineWidgetResolverReady = healthOk,
resolveInlineWidgetResource = resolveInlineWidgetResource,
loadImageArtifact = loadImageArtifact,
)
ChatTimelineItem.Thinking -> {
val run = workingRun
@@ -1616,18 +1621,26 @@ private fun ChatBubble(
onToggleListen: (String, String) -> Unit,
inlineWidgetResolverReady: Boolean,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
) {
val normalizedRole = role.trim().lowercase(Locale.US)
val isUser = normalizedRole == "user"
var visibleImageCount = 0
val displayableContent =
content.filter { part ->
when (part.type) {
"text" -> !part.text.isNullOrBlank()
"image" -> !part.base64.isNullOrBlank()
"image" -> {
val displayable = !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank()
val visible = displayable && visibleImageCount < 4
if (displayable) visibleImageCount += 1
visible
}
"canvas" -> normalizedRole == "assistant" && part.widget != null
else -> part.isAudioAttachment()
}
}
val omittedImageCount = (visibleImageCount - 4).coerceAtLeast(0)
if (displayableContent.isEmpty()) return
val messageText = chatMessagePlainText(displayableContent)
@@ -1692,10 +1705,19 @@ private fun ChatBubble(
part.type == "text" -> Unit
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
part.type == "image" ->
ChatBase64Image(
base64 = checkNotNull(part.base64),
mimeType = part.mimeType,
)
if (!part.base64.isNullOrBlank()) {
ChatBase64Image(
base64 = part.base64,
mimeType = part.mimeType,
)
} else {
ChatManagedImage(
artifactId = checkNotNull(part.artifactId),
label = part.alt?.takeIf(String::isNotBlank) ?: part.fileName ?: nativeString("Image"),
resolverReady = inlineWidgetResolverReady,
loadImage = loadImageArtifact,
)
}
part.type == "canvas" && normalizedRole == "assistant" ->
ChatInlineWidget(
preview = checkNotNull(part.widget),
@@ -1705,6 +1727,13 @@ private fun ChatBubble(
else -> Text(text = part.fileName ?: nativeString("Attachment"), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
}
}
if (omittedImageCount > 0) {
Text(
text = nativeString("Additional images hidden: \${omittedImageCount}", omittedImageCount),
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
)
}
if (messageId != null) {
ChatMessageLinkPreview(messageId = messageId, role = normalizedRole, content = displayableContent)
}
@@ -60,6 +60,30 @@ class ChatControllerMessageIdentityTest {
assertEquals(listOf(ChatMessageContent(type = "text", text = "Hi there")), content)
}
@Test
fun managedImagesParticipateInMessageIdentity() {
fun message(artifactId: String) =
ChatMessage(
id = artifactId,
role = "assistant",
content =
listOf(
ChatMessageContent(
type = "image",
artifactId = artifactId,
url = "/api/chat/media/outgoing/main/$artifactId/full",
mimeType = "image/png",
),
),
timestampMs = 1,
)
assertNotEquals(
messageIdentityKey(message("artifact_managed_image_11111111-1111-4111-8111-111111111111")),
messageIdentityKey(message("artifact_managed_image_22222222-2222-4222-8222-222222222222")),
)
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun liveHistoryDropsInternalRoleRows() =
@@ -276,14 +276,14 @@ class ChatMessageContentParsingTest {
}
@Test
fun parsesImageBlocksOnlyWhenInlineContentExists() {
fun parsesInlineAndManagedImageBlocks() {
val image =
Json.parseToJsonElement(
"""{"type":"image","mimeType":"image/png","fileName":"chart.png","content":"abc123"}""",
)
val managedImage =
Json.parseToJsonElement(
"""{"type":"image","mimeType":"image/png","fileName":"chart.png","url":"/api/chat/media/outgoing/main/id"}""",
"""{"type":"image","artifactId":"artifact_managed_image_11111111-1111-4111-8111-111111111111","mimeType":"image/png","fileName":"chart.png","url":"/api/chat/media/outgoing/main/id","openUrl":"/api/chat/media/outgoing/main/id","alt":"Chart","width":1200,"height":800,"sizeBytes":2048}""",
)
assertEquals(
@@ -291,11 +291,35 @@ class ChatMessageContentParsingTest {
parseChatMessageContent(image),
)
assertEquals(
ChatMessageContent(type = "image", mimeType = "image/png", fileName = "chart.png", base64 = null),
ChatMessageContent(
type = "image",
mimeType = "image/png",
fileName = "chart.png",
artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111",
url = "/api/chat/media/outgoing/main/id",
openUrl = "/api/chat/media/outgoing/main/id",
alt = "Chart",
width = 1200,
height = 800,
sizeBytes = 2048,
),
parseChatMessageContent(managedImage),
)
}
@Test
fun derivesArtifactIdentityForShippedManagedImageBlocks() {
val image =
Json.parseToJsonElement(
"""{"type":"image","mimeType":"image/png","url":"/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full"}""",
)
assertEquals(
"artifact_managed_image_11111111-1111-4111-8111-111111111111",
parseChatMessageContent(image)?.artifactId,
)
}
@Test
fun dropsOversizedInlineImageContentBeforeRendering() {
val oversized = "A".repeat(CHAT_IMAGE_MAX_BASE64_CHARS + 1)
@@ -40,10 +40,18 @@ class RoomChatTranscriptCacheTest {
)
@Test
fun transcriptRoundTripKeepsTextRowsOnly() =
fun transcriptRoundTripKeepsTextAndManagedReferencesWithoutBinaryParts() =
runTest {
val store = cache()
val imagePart = ChatMessageContent(type = "image", mimeType = "image/png", fileName = "a.png", base64 = "AAAA")
val managedImage =
ChatMessageContent(
type = "image",
mimeType = "image/png",
artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111",
url = "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
alt = "Managed image",
)
store.saveTranscript(
gatewayId = "gateway-a",
agentId = "main",
@@ -51,19 +59,44 @@ class RoomChatTranscriptCacheTest {
messages =
listOf(
message("hello", role = "user", timestampMs = 10, idempotencyKey = "run-1:user", extraParts = listOf(imagePart)),
// Attachment-only messages have no cacheable text and are skipped entirely.
// Inline binary-only messages remain disposable and are skipped entirely.
ChatMessage(id = "img", role = "user", content = listOf(imagePart), timestampMs = 11),
ChatMessage(id = "managed", role = "assistant", content = listOf(managedImage), timestampMs = 11),
message("world", role = "assistant", timestampMs = 12),
),
)
val loaded = store.loadTranscript("gateway-a", "main", "main")
assertEquals(listOf("hello", "world"), loaded.map { it.content.single().text })
assertTrue(loaded.all { message -> message.content.all { part -> part.type == "text" && part.base64 == null } })
assertEquals(listOf("user", "assistant"), loaded.map { it.role })
assertEquals(listOf(10L, 12L), loaded.map { it.timestampMs })
assertEquals(listOf("run-1:user", null), loaded.map { it.idempotencyKey })
assertEquals(listOf("hello", null, "world"), loaded.map { it.content.single().text })
assertTrue(loaded.all { message -> message.content.all { part -> part.base64 == null } })
assertEquals(managedImage.artifactId, loaded[1].content.single().artifactId)
assertEquals(listOf("user", "assistant", "assistant"), loaded.map { it.role })
assertEquals(listOf(10L, 11L, 12L), loaded.map { it.timestampMs })
assertEquals(listOf("run-1:user", null, null), loaded.map { it.idempotencyKey })
}
@Test
fun legacyStringArrayTranscriptRowsRemainReadable() =
runTest {
database.dao().insertMessages(
listOf(
CachedMessageEntity(
gatewayId = "gateway-a",
agentId = "main",
sessionKey = "main",
rowOrder = 0,
role = "assistant",
textPartsJson = """["legacy one","legacy two"]""",
timestampMs = 10,
idempotencyKey = null,
),
),
)
val loaded = cache().loadTranscript("gateway-a", "main", "main").single()
assertEquals(listOf("legacy one", "legacy two"), loaded.content.map { it.text })
}
@Test
@@ -19,6 +19,8 @@ import okhttp3.mockwebserver.Dispatcher
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import okhttp3.mockwebserver.RecordedRequest
import okio.Buffer
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
@@ -60,6 +62,115 @@ private class NoopDeviceAuthStore : DeviceAuthTokenStore {
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
class GatewaySessionCustomHeadersTest {
@Test
fun managedImageDownload_usesArtifactTicketWithoutGatewayBearer() =
runBlocking {
val app = RuntimeEnvironment.getApplication()
val json = Json { ignoreUnknownKeys = true }
val connected = CompletableDeferred<Unit>()
val imageRequest = CompletableDeferred<RecordedRequest>()
val imageBytes = byteArrayOf(1, 2, 3, 4)
val attachmentId = "11111111-1111-4111-8111-111111111111"
val artifactId = "artifact_managed_image_$attachmentId"
val imagePath = "/api/chat/media/outgoing/main/$attachmentId/full?mediaTicket=ticket"
val server =
MockWebServer().apply {
dispatcher =
object : Dispatcher() {
override fun dispatch(request: RecordedRequest): MockResponse {
if (request.path == imagePath) {
imageRequest.complete(request)
return MockResponse()
.setHeader("Content-Type", "image/png")
.setBody(Buffer().write(imageBytes))
}
return MockResponse().withWebSocketUpgrade(
object : WebSocketListener() {
override fun onOpen(
webSocket: WebSocket,
response: Response,
) {
webSocket.send(CONNECT_CHALLENGE_FRAME)
}
override fun onMessage(
webSocket: WebSocket,
text: String,
) {
val frame = json.parseToJsonElement(text).jsonObject
if (frame["type"]?.jsonPrimitive?.content != "req") return
val id = frame["id"]?.jsonPrimitive?.content ?: return
when (frame["method"]?.jsonPrimitive?.content) {
"connect" ->
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"snapshot":{"sessionDefaults":{"mainSessionKey":"main"}}}}""",
)
"artifacts.download" ->
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
)
}
}
},
)
}
}
start()
}
val stableId = "manual|127.0.0.1|${server.port}"
val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
val session =
GatewaySession(
scope = scope,
identityStore = testDeviceIdentityStore(app),
deviceAuthStore = NoopDeviceAuthStore(),
onConnected = { if (!connected.isCompleted) connected.complete(Unit) },
onDisconnected = {},
onEvent = { _, _ -> },
)
try {
session.connect(
endpoint = GatewayEndpoint(stableId, "test", "127.0.0.1", server.port, tlsEnabled = false),
token = "bootstrap-token",
bootstrapToken = null,
password = null,
options =
GatewayConnectOptions(
role = "operator",
scopes = listOf("operator.read"),
caps = emptyList(),
commands = emptyList(),
permissions = emptyMap(),
client =
GatewayClientInfo(
id = "openclaw-android-test",
displayName = "Android Test",
version = "1.0.0-test",
platform = "android",
mode = "ui",
instanceId = "android-test-instance",
deviceFamily = "android",
modelIdentifier = "test",
),
),
tls = null,
)
withTimeout(TEST_TIMEOUT_MS) { connected.await() }
val loaded = session.loadImageArtifact(stableId, "main", "main", artifactId)
assertArrayEquals(imageBytes, loaded?.bytes)
assertEquals("image/png", loaded?.mimeType)
val request = withTimeout(TEST_TIMEOUT_MS) { imageRequest.await() }
assertNull(request.getHeader("Authorization"))
assertEquals("image/*", request.getHeader("Accept"))
} finally {
session.disconnectAndJoin()
scope.cancel()
server.shutdown()
}
}
@Test
fun tlsUpgradeRequest_carriesLatestSanitizedHeadersForOnlyThisGateway() {
val app = RuntimeEnvironment.getApplication()
@@ -0,0 +1,56 @@
package ai.openclaw.app.ui.chat
import ai.openclaw.app.chat.ChatMessage
import ai.openclaw.app.chat.ChatMessageContent
import android.os.Looper
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import org.junit.Assert.assertEquals
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.Robolectric
import org.robolectric.RobolectricTestRunner
import org.robolectric.Shadows.shadowOf
@RunWith(RobolectricTestRunner::class)
class ChatMessageViewsTest {
@Test
fun managedImageCompositionRequestsItsArtifact() {
val artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111"
val requested = mutableListOf<String>()
val controller = Robolectric.buildActivity(ComponentActivity::class.java).setup()
try {
controller.get().setContent {
ChatMessageBubble(
message =
ChatMessage(
id = "managed-image",
role = "assistant",
content =
listOf(
ChatMessageContent(
type = "image",
mimeType = "image/png",
artifactId = artifactId,
alt = "Managed image",
),
),
timestampMs = 1,
),
imageResolverReady = true,
loadImageArtifact = { requestedArtifactId ->
requested += requestedArtifactId
null
},
)
}
shadowOf(Looper.getMainLooper()).idle()
assertEquals(listOf(artifactId), requested)
} finally {
controller.pause().stop().destroy()
shadowOf(Looper.getMainLooper()).idle()
}
}
}
@@ -11,6 +11,7 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
private let globalAgentId: String?
private let outboxGatewayID: String?
private let sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)?
private let imageArtifactLoader: IOSImageArtifactLoader?
var outboxRequiresSessionRoutingContract: Bool {
true
@@ -21,7 +22,8 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
widgetGateway: GatewayNodeSession? = nil,
globalAgentId: String? = nil,
outboxGatewayID: String? = nil,
sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? = nil)
sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? = nil,
imageArtifactLoader: IOSImageArtifactLoader? = nil)
{
self.gateway = gateway
self.widgetGateway = widgetGateway
@@ -30,6 +32,7 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
let normalizedGatewayID = outboxGatewayID?.trimmingCharacters(in: .whitespacesAndNewlines)
self.outboxGatewayID = normalizedGatewayID?.isEmpty == false ? normalizedGatewayID : nil
self.sessionMutationRequest = sessionMutationRequest
self.imageArtifactLoader = imageArtifactLoader
}
func acquireOutboxRouteLease() async -> OpenClawChatTransportRouteLeaseResult {
@@ -562,6 +565,31 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
})
}
func loadImageArtifact(
sessionKey: String,
artifactId: String) async throws -> OpenClawChatLoadedImage?
{
guard let imageArtifactLoader,
let route = await gateway.currentRoute(),
let gatewayID = await gateway.currentGatewayID(ifCurrentRoute: route)
else { return nil }
let target = self.sessionTarget(for: sessionKey)
let request = OpenClawChatGatewayRequests.artifactDownload(
sessionKey: target.sessionKey,
agentID: target.agentID,
artifactId: artifactId)
let data = try await gateway.request(request, ifCurrentRoute: route)
let response = try JSONDecoder().decode(ArtifactsDownloadResult.self, from: data)
guard let url = response.url?.trimmingCharacters(in: .whitespacesAndNewlines), !url.isEmpty
else { return nil }
guard await self.gateway.currentRoute() == route else { throw CancellationError() }
let loaded = try await imageArtifactLoader.load(
ticketedPath: url,
expectedGatewayID: gatewayID)
guard await self.gateway.currentRoute() == route else { throw CancellationError() }
return loaded
}
func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL? {
await self.resolveInlineWidgetResource(
path: path,
@@ -0,0 +1,111 @@
import Foundation
import OpenClawChatUI
import OpenClawKit
struct IOSImageArtifactLoader: Sendable {
struct Connection: Sendable {
let config: GatewayConnectConfig
let gatewayID: String
let customHeaders: [String: String]
}
enum LoadError: Error, Equatable {
case invalidSource
case invalidResponse
case requestFailed(statusCode: Int)
case unsupportedMediaType
case payloadTooLarge
}
typealias Request = @Sendable (URLRequest) async throws -> (Data, URLResponse)
typealias RequestFactory = @Sendable (GatewayTLSParams, Int) -> Request
typealias ConnectionProvider = @MainActor @Sendable () -> Connection?
static let maximumImageBytes = 12 * 1024 * 1024
private static let managedImagePathPrefix = "/api/chat/media/outgoing/"
private let connectionProvider: ConnectionProvider
private let requestFactory: RequestFactory
init(connectionProvider: @escaping ConnectionProvider) {
self.init(connectionProvider: connectionProvider) { tls, maximumBytes in
let session = GatewayTLSPinningSession(params: tls)
return { request in
defer { session.finishTasksAndInvalidate() }
return try await session.data(for: request, maximumBytes: maximumBytes)
}
}
}
init(
connectionProvider: @escaping ConnectionProvider,
requestFactory: @escaping RequestFactory)
{
self.connectionProvider = connectionProvider
self.requestFactory = requestFactory
}
func load(
ticketedPath rawPath: String,
expectedGatewayID: String) async throws -> OpenClawChatLoadedImage
{
let path = rawPath.trimmingCharacters(in: .whitespacesAndNewlines)
guard let connection = await self.connectionProvider(),
connection.gatewayID == expectedGatewayID,
let url = Self.managedImageURL(config: connection.config, path: path)
else { throw LoadError.invalidSource }
var request = URLRequest(url: url)
request.timeoutInterval = 20
request.setValue("image/*", forHTTPHeaderField: "Accept")
if url.scheme?.lowercased() == "https" {
for (name, value) in GatewayCustomHeaders.sanitized(connection.customHeaders) {
request.setValue(value, forHTTPHeaderField: name)
}
}
let tls = connection.config.tls ?? GatewayTLSParams(
required: false,
expectedFingerprint: nil,
allowTOFU: false,
storeKey: nil)
let data: Data
let response: URLResponse
do {
(data, response) = try await self.requestFactory(tls, Self.maximumImageBytes)(request)
} catch is GatewayBoundedDataError {
throw LoadError.payloadTooLarge
}
guard let http = response as? HTTPURLResponse else { throw LoadError.invalidResponse }
guard (200..<300).contains(http.statusCode) else {
throw LoadError.requestFailed(statusCode: http.statusCode)
}
guard let mimeType = http.mimeType?.lowercased(), mimeType.hasPrefix("image/") else {
throw LoadError.unsupportedMediaType
}
guard data.count <= Self.maximumImageBytes else { throw LoadError.payloadTooLarge }
return OpenClawChatLoadedImage(data: data, mimeType: mimeType)
}
private static func managedImageURL(config: GatewayConnectConfig, path: String) -> URL? {
guard path.hasPrefix(self.managedImagePathPrefix),
let relative = URLComponents(string: path),
relative.scheme == nil,
relative.host == nil,
relative.fragment == nil,
relative.percentEncodedPath.hasPrefix(Self.managedImagePathPrefix),
relative.queryItems?.contains(where: {
$0.name == "mediaTicket" && $0.value?.isEmpty == false
}) == true,
var base = URLComponents(url: config.url, resolvingAgainstBaseURL: false),
base.host != nil
else { return nil }
switch base.scheme?.lowercased() {
case "wss", "https": base.scheme = "https"
case "ws", "http": base.scheme = "http"
default: return nil
}
base.percentEncodedPath = relative.percentEncodedPath
base.percentEncodedQuery = relative.percentEncodedQuery
base.fragment = nil
return base.url
}
}
+10 -1
View File
@@ -619,11 +619,20 @@ final class NodeAppModel {
if self.isAppleReviewDemoModeEnabled {
return AppleReviewDemoChatTransport()
}
let imageArtifactLoader = IOSImageArtifactLoader { [weak self] in
guard let config = self?.activeGatewayConnectConfig else { return nil }
return IOSImageArtifactLoader.Connection(
config: config,
gatewayID: config.nodeOptions.deviceAuthGatewayID ?? config.effectiveStableID,
customHeaders: GatewaySettingsStore.loadGatewayCustomHeaders(
gatewayStableID: config.effectiveStableID))
}
return IOSGatewayChatTransport(
gateway: self.operatorSession,
widgetGateway: self.nodeGateway,
globalAgentId: self.chatDeliveryAgentId,
outboxGatewayID: outboxGatewayID)
outboxGatewayID: outboxGatewayID,
imageArtifactLoader: imageArtifactLoader)
}
/// Gateway identity the transcript cache is scoped to: the active
@@ -0,0 +1,91 @@
import Foundation
import OpenClawKit
import Testing
@testable import OpenClaw
@Suite("iOS managed image artifact loader")
struct IOSImageArtifactLoaderTests {
@Test @MainActor func `loads ticketed image with proxy headers and without a gateway bearer`() async throws {
let gatewayURL = try #require(URL(string: "wss://gateway.example"))
let config = Self.config(url: gatewayURL)
let loader = IOSImageArtifactLoader(
connectionProvider: {
IOSImageArtifactLoader.Connection(
config: config,
gatewayID: config.effectiveStableID,
customHeaders: ["X-Proxy-Token": "proxy"])
},
requestFactory: { _, maximumBytes in
#expect(maximumBytes == 12 * 1024 * 1024)
return { request in
#expect(request.url?.absoluteString ==
"https://gateway.example/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full?mediaTicket=ticket")
#expect(request.value(forHTTPHeaderField: "Authorization") == nil)
#expect(request.value(forHTTPHeaderField: "X-Proxy-Token") == "proxy")
let responseURL = try #require(request.url)
let response = try #require(HTTPURLResponse(
url: responseURL,
statusCode: 200,
httpVersion: nil,
headerFields: ["Content-Type": "image/png"]))
return (Data([1, 2, 3]), response)
}
})
let loaded = try await loader.load(
ticketedPath:
"/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full?mediaTicket=ticket",
expectedGatewayID: config.effectiveStableID)
#expect(loaded.data == Data([1, 2, 3]))
#expect(loaded.mimeType == "image/png")
}
@Test @MainActor func `rejects absolute and unticketed paths before fetching`() async {
let config = Self.config()
let loader = IOSImageArtifactLoader(
connectionProvider: {
IOSImageArtifactLoader.Connection(
config: config,
gatewayID: config.effectiveStableID,
customHeaders: [:])
},
requestFactory: { _, _ in
Issue.record("invalid paths must not reach the network")
return { _ in throw CancellationError() }
})
await #expect(throws: IOSImageArtifactLoader.LoadError.invalidSource) {
try await loader.load(
ticketedPath: "https://example.com/image.png?mediaTicket=ticket",
expectedGatewayID: config.effectiveStableID)
}
await #expect(throws: IOSImageArtifactLoader.LoadError.invalidSource) {
try await loader.load(
ticketedPath:
"/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
expectedGatewayID: config.effectiveStableID)
}
}
private static func config(
url: URL = URL(string: "ws://127.0.0.1:18789")!) -> GatewayConnectConfig
{
GatewayConnectConfig(
url: url,
stableID: "manual|127.0.0.1|18789",
tls: nil,
token: nil,
bootstrapToken: nil,
password: nil,
nodeOptions: GatewayConnectOptions(
role: "node",
scopes: [],
caps: [],
commands: [],
permissions: [:],
clientId: "ios",
clientMode: "node",
clientDisplayName: "Phone"))
}
}
+59 -3
View File
@@ -269,13 +269,65 @@ struct SwiftUIRenderSmokeTests {
userMessageExpanded: false,
onToggleUserMessageExpanded: {},
inlineWidgetResolverReady: true,
inlineWidgetResourceResolver: { _, _ in nil })
inlineWidgetResourceResolver: { _, _ in nil },
imageArtifactResolverReady: false,
loadImageArtifact: { _ in nil })
.environment(\.dynamicTypeSize, typeSize)
_ = Self.host(root, size: CGSize(width: 320, height: 420))
}
}
@Test @MainActor func `managed assistant image starts its artifact load`() async throws {
let artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111"
let message = OpenClawChatMessage(
role: "assistant",
content: [OpenClawChatMessageContent(
type: "image",
text: nil,
mimeType: "image/png",
fileName: nil,
artifactId: artifactId,
url: "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
alt: "Managed preview",
content: nil)],
timestamp: 1)
var requestedArtifactId: String?
let root = ChatMessageBubble(
message: message,
style: .standard,
markdownVariant: .standard,
userAccent: nil,
displayOptions: [],
assistantName: "OpenClaw",
assistantAvatarText: "OC",
assistantAvatarTint: nil,
showsAssistantAvatar: true,
isClean: false,
contextWindowTokens: nil,
userMessageExpanded: false,
onToggleUserMessageExpanded: {},
inlineWidgetResolverReady: true,
inlineWidgetResourceResolver: { _, _ in nil },
imageArtifactResolverReady: true,
loadImageArtifact: { requested in
requestedArtifactId = requested
return OpenClawChatLoadedImage(
data: Data(base64Encoded:
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=")!,
mimeType: "image/png")
})
let window = Self.host(root, size: CGSize(width: 393, height: 420))
defer { window.isHidden = true }
let deadline = ContinuousClock().now.advanced(by: .seconds(2))
while requestedArtifactId == nil, ContinuousClock().now < deadline {
try await Task.sleep(for: .milliseconds(10))
}
#expect(requestedArtifactId == artifactId)
}
@Test @MainActor func `streaming assistant bubble builds mixed prose and code`() {
let text = """
Earlier prose stays visible.
@@ -336,7 +388,9 @@ struct SwiftUIRenderSmokeTests {
userMessageExpanded: false,
onToggleUserMessageExpanded: {},
inlineWidgetResolverReady: true,
inlineWidgetResourceResolver: { _, _ in nil })
inlineWidgetResourceResolver: { _, _ in nil },
imageArtifactResolverReady: false,
loadImageArtifact: { _ in nil })
ChatStreamingAssistantBubble(
text: text,
markdownVariant: .standard,
@@ -390,7 +444,9 @@ struct SwiftUIRenderSmokeTests {
userMessageExpanded: false,
onToggleUserMessageExpanded: {},
inlineWidgetResolverReady: true,
inlineWidgetResourceResolver: { _, _ in nil })
inlineWidgetResourceResolver: { _, _ in nil },
imageArtifactResolverReady: false,
loadImageArtifact: { _ in nil })
.environment(\.dynamicTypeSize, typeSize)
_ = Self.host(root, size: CGSize(width: 320, height: 280))
@@ -0,0 +1,77 @@
import Foundation
import OpenClawChatUI
import OpenClawKit
import OpenClawProtocol
private let gatewayManagedImagePathPrefix = "/api/chat/media/outgoing/"
extension GatewayConnection {
func loadImageArtifact(
sessionKey: String,
agentID: String?,
artifactId: String,
ifCurrentServerLease lease: ServerLease) async throws -> OpenClawChatLoadedImage?
{
let request = OpenClawChatGatewayRequests.artifactDownload(
sessionKey: sessionKey,
agentID: agentID,
artifactId: artifactId)
let responseData = try await self.request(
method: request.method,
params: request.params,
timeoutMs: request.timeoutMs,
ifCurrentServerLease: lease)
let response = try JSONDecoder().decode(ArtifactsDownloadResult.self, from: responseData)
guard let ticketedPath = response.url?.trimmingCharacters(in: .whitespacesAndNewlines),
let url = Self.managedImageURL(gatewayURL: lease.route.url, ticketedPath: ticketedPath)
else { return nil }
var urlRequest = URLRequest(url: url)
urlRequest.timeoutInterval = 20
urlRequest.setValue("image/*", forHTTPHeaderField: "Accept")
// Native macOS has no per-Gateway proxy-header configuration surface today. If one is
// added, carry its immutable snapshot on Route so the socket and ticket GET cannot diverge.
let tls = lease.route.tls?.params ?? GatewayTLSParams(
required: false,
expectedFingerprint: nil,
allowTOFU: false,
storeKey: nil)
let session = GatewayTLSPinningSession(params: tls)
defer { session.finishTasksAndInvalidate() }
let (data, urlResponse) = try await session.data(
for: urlRequest,
maximumBytes: 12 * 1024 * 1024)
guard await self.isCurrentServerLease(lease) else {
throw OpenClawChatTransportSendError.notDispatched
}
guard let http = urlResponse as? HTTPURLResponse,
(200..<300).contains(http.statusCode),
let mimeType = http.mimeType?.lowercased(),
mimeType.hasPrefix("image/")
else { return nil }
return OpenClawChatLoadedImage(data: data, mimeType: mimeType)
}
private static func managedImageURL(gatewayURL: URL, ticketedPath: String) -> URL? {
guard ticketedPath.hasPrefix(gatewayManagedImagePathPrefix),
let relative = URLComponents(string: ticketedPath),
relative.scheme == nil,
relative.host == nil,
relative.fragment == nil,
relative.queryItems?.contains(where: {
$0.name == "mediaTicket" && $0.value?.isEmpty == false
}) == true,
var base = URLComponents(url: gatewayURL, resolvingAgainstBaseURL: false),
base.host != nil
else { return nil }
switch base.scheme?.lowercased() {
case "wss", "https": base.scheme = "https"
case "ws", "http": base.scheme = "http"
default: return nil
}
base.percentEncodedPath = relative.percentEncodedPath
base.percentEncodedQuery = relative.percentEncodedQuery
base.fragment = nil
return base.url
}
}
@@ -47,10 +47,10 @@ actor GatewayConnection {
struct Route: Equatable, Sendable {
fileprivate let generation: UInt64
fileprivate let authority: UInt64?
fileprivate let url: URL
let url: URL
fileprivate let token: String?
fileprivate let password: String?
fileprivate let tls: GatewayTLSRoute?
let tls: GatewayTLSRoute?
fileprivate let deviceAuthGatewayID: String?
let activationOwnershipFingerprint: String?
@@ -67,7 +67,9 @@ actor GatewayConnection {
/// One connected Gateway server, not merely an endpoint configuration.
/// A reconnect at the same URL creates a different lease.
struct ServerLease: Sendable {
fileprivate let route: Route
// Managed-image HTTP reuses this captured route from its focused extension file.
// Carrying the snapshot forward prevents endpoint or TLS rediscovery after suspension.
let route: Route
fileprivate let socketGeneration: UInt64
fileprivate let client: GatewayChannelActor
}
@@ -566,6 +566,21 @@ struct MacGatewayChatTransport: OpenClawChatTransport {
connection: self.connection)
}
func loadImageArtifact(
sessionKey: String,
artifactId: String) async throws -> OpenClawChatLoadedImage?
{
guard let serverLease = await connection.captureServerLease() else {
throw OpenClawChatTransportSendError.notDispatched
}
let target = self.sessionTarget(for: sessionKey)
return try await self.connection.loadImageArtifact(
sessionKey: target.sessionKey,
agentID: target.agentID,
artifactId: artifactId,
ifCurrentServerLease: serverLease)
}
var supportsSlashCommandCatalog: Bool {
true
}
@@ -80,6 +80,22 @@ public enum OpenClawChatGatewayRequests {
OpenClawChatGatewayRequest(method: "models.list", timeoutMs: self.defaultTimeoutMs)
}
public static func artifactDownload(
sessionKey: String,
agentID: String?,
artifactId: String) -> OpenClawChatGatewayRequest
{
var params: [String: AnyCodable] = [
"sessionKey": AnyCodable(sessionKey),
"artifactId": AnyCodable(artifactId),
]
self.add(agentID, to: &params, key: "agentId")
return OpenClawChatGatewayRequest(
method: "artifacts.download",
params: params,
timeoutMs: self.defaultTimeoutMs)
}
public static func chatMetadata(
sessionKey: String,
fallbackAgentID: String?) -> OpenClawChatGatewayRequest
@@ -0,0 +1,149 @@
import Foundation
import ImageIO
import SwiftUI
#if canImport(AppKit)
import AppKit
#elseif canImport(UIKit)
import UIKit
#endif
enum ChatMediaImageDecoder {
static let maximumThumbnailPixels = 2048
static func decode(_ data: Data) -> OpenClawPlatformImage? {
guard let source = CGImageSourceCreateWithData(data as CFData, nil),
CGImageSourceGetCount(source) > 0
else { return nil }
let options: [CFString: Any] = [
kCGImageSourceCreateThumbnailFromImageAlways: true,
kCGImageSourceCreateThumbnailWithTransform: true,
kCGImageSourceThumbnailMaxPixelSize: self.maximumThumbnailPixels,
kCGImageSourceShouldCacheImmediately: true,
]
guard let cgImage = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary)
else { return nil }
#if canImport(AppKit)
return NSImage(cgImage: cgImage, size: .zero)
#elseif canImport(UIKit)
return UIImage(cgImage: cgImage)
#endif
}
}
@MainActor
struct ChatMediaImageAttachment: View {
private enum LoadState {
case loading
case loaded(OpenClawPlatformImage)
case unavailable
}
let artifactId: String
let label: String
let resolverReady: Bool
let load: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
@State private var state: LoadState = .loading
@State private var retryGeneration = 0
@State private var showsFullImage = false
var body: some View {
Group {
switch self.state {
case .loading:
HStack(spacing: 8) {
ProgressView()
Text(String(localized: "Loading image…"))
.font(OpenClawChatTypography.footnote)
.foregroundStyle(.secondary)
}
.frame(minHeight: 88)
.frame(maxWidth: .infinity)
case let .loaded(image):
Button {
self.showsFullImage = true
} label: {
OpenClawPlatformImageFactory.image(image)
.resizable()
.scaledToFit()
.frame(maxHeight: 320)
.clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: 12, style: .continuous)
.strokeBorder(Color.white.opacity(0.12), lineWidth: 1))
}
.buttonStyle(.plain)
.accessibilityLabel(self.label)
.accessibilityHint(String(localized: "Opens image preview"))
.sheet(isPresented: self.$showsFullImage) {
ZStack(alignment: .topTrailing) {
Color.black.ignoresSafeArea()
ScrollView([.horizontal, .vertical]) {
OpenClawPlatformImageFactory.image(image)
.resizable()
.scaledToFit()
.padding(20)
}
Button {
self.showsFullImage = false
} label: {
Image(systemName: "xmark.circle.fill")
.font(.title2)
.symbolRenderingMode(.hierarchical)
}
.buttonStyle(.plain)
.foregroundStyle(.white)
.padding(16)
.accessibilityLabel(String(localized: "Close image preview"))
}
}
case .unavailable:
HStack(spacing: 8) {
Image(systemName: "photo.badge.exclamationmark")
Text(String(localized: "Image unavailable"))
.font(OpenClawChatTypography.footnote)
Spacer()
Button {
self.retryGeneration &+= 1
} label: {
Text(String(localized: "Retry"))
.font(OpenClawChatTypography.footnote)
}
.buttonStyle(.plain)
}
.foregroundStyle(.secondary)
.padding(10)
.background(Color.black.opacity(0.04))
.clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous))
}
}
.task(id: "\(self.artifactId):\(self.resolverReady):\(self.retryGeneration)") {
await self.loadImage()
}
}
private func loadImage() async {
guard self.resolverReady else {
self.state = .unavailable
return
}
self.state = .loading
do {
guard let loaded = try await self.load(self.artifactId), !Task.isCancelled else {
if !Task.isCancelled { self.state = .unavailable }
return
}
let image = await Task.detached(priority: .userInitiated) {
ChatMediaImageDecoder.decode(loaded.data)
}.value
guard !Task.isCancelled else { return }
self.state = image.map(LoadState.loaded) ?? .unavailable
} catch is CancellationError {
return
} catch {
guard !Task.isCancelled else { return }
self.state = .unavailable
}
}
}
@@ -219,6 +219,8 @@ struct ChatMessageBubble: View {
let inlineWidgetResourceResolver: @MainActor @Sendable (
String,
OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
let imageArtifactResolverReady: Bool
let loadImageArtifact: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
var body: some View {
if self.isUser {
@@ -261,7 +263,9 @@ struct ChatMessageBubble: View {
userMessageExpanded: self.userMessageExpanded,
onToggleUserMessageExpanded: self.onToggleUserMessageExpanded,
inlineWidgetResolverReady: self.inlineWidgetResolverReady,
inlineWidgetResourceResolver: self.inlineWidgetResourceResolver)
inlineWidgetResourceResolver: self.inlineWidgetResourceResolver,
imageArtifactResolverReady: self.imageArtifactResolverReady,
loadImageArtifact: self.loadImageArtifact)
}
}
@@ -314,6 +318,8 @@ private struct ChatMessageBody: View {
let inlineWidgetResourceResolver: @MainActor @Sendable (
String,
OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
let imageArtifactResolverReady: Bool
let loadImageArtifact: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
var body: some View {
let text = self.primaryText
@@ -370,12 +376,24 @@ private struct ChatMessageBody: View {
ChatLinkPreview(url: previewURL)
}
if !self.inlineAttachments.isEmpty {
ForEach(self.inlineAttachments.indices, id: \.self) { idx in
AttachmentRow(att: self.inlineAttachments[idx], isUser: self.isUser)
if !self.visibleInlineAttachments.isEmpty {
ForEach(self.visibleInlineAttachments.indices, id: \.self) { idx in
AttachmentRow(
att: self.visibleInlineAttachments[idx],
isUser: self.isUser,
resolverReady: self.imageArtifactResolverReady,
loadImage: self.loadImageArtifact)
}
}
if self.omittedImageAttachmentCount > 0 {
Text(String(
format: String(localized: "Additional images hidden: %lld"),
Int64(self.omittedImageAttachmentCount)))
.font(OpenClawChatTypography.footnote)
.foregroundStyle(.secondary)
}
ForEach(self.inlineWidgets.indices, id: \.self) { idx in
ChatInlineWidgetView(
preview: self.inlineWidgets[idx],
@@ -503,6 +521,19 @@ private struct ChatMessageBody: View {
self.message.content.filter(\.isInlineAttachment)
}
private var visibleInlineAttachments: [OpenClawChatMessageContent] {
var imageCount = 0
return self.inlineAttachments.filter { attachment in
guard attachment.isImageAttachment else { return true }
defer { imageCount += 1 }
return imageCount < 4
}
}
private var omittedImageAttachmentCount: Int {
max(0, self.inlineAttachments.filter(\.isImageAttachment).count - 4)
}
private var inlineWidgets: [OpenClawChatCanvasPreview] {
guard self.message.role.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "assistant"
else { return [] }
@@ -620,11 +651,25 @@ private struct ChatMessageBody: View {
private struct AttachmentRow: View {
let att: OpenClawChatMessageContent
let isUser: Bool
let resolverReady: Bool
let loadImage: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
var body: some View {
if self.att.isImageAttachment, let artifactId = self.normalizedArtifactId {
ChatMediaImageAttachment(
artifactId: artifactId,
label: self.attachmentLabel,
resolverReady: self.resolverReady,
load: self.loadImage)
} else {
self.fallbackRow
}
}
private var fallbackRow: some View {
HStack(spacing: 8) {
Image(systemName: self.isAudio ? "waveform" : "paperclip")
Text(self.isAudio ? "Voice note" : (self.att.fileName ?? "Attachment"))
Text(self.isAudio ? "Voice note" : self.attachmentLabel)
.font(OpenClawChatTypography.footnote)
.lineLimit(1)
.foregroundStyle(self.isUser ? OpenClawChatTheme.userText : OpenClawChatTheme.assistantText)
@@ -646,6 +691,26 @@ private struct AttachmentRow: View {
private var isAudio: Bool {
self.att.mimeType?.hasPrefix("audio/") == true
}
private var normalizedArtifactId: String? {
let value = self.att.artifactId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return value.isEmpty ? nil : value
}
private var attachmentLabel: String {
let values = [self.att.alt, self.att.fileName]
return values.lazy
.compactMap { $0?.trimmingCharacters(in: .whitespacesAndNewlines) }
.first { !$0.isEmpty } ?? String(localized: "Attachment")
}
}
extension OpenClawChatMessageContent {
fileprivate var isImageAttachment: Bool {
self.type?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "image" ||
self.mimeType?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
.hasPrefix("image/") == true
}
}
@MainActor
@@ -116,6 +116,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
public let thinkingSignature: String?
public let mimeType: String?
public let fileName: String?
public let artifactId: String?
public let url: String?
public let openUrl: String?
public let alt: String?
public let width: Int?
public let height: Int?
public let sizeBytes: Int?
public let durationSeconds: Double?
public let content: AnyCodable?
public let preview: OpenClawChatCanvasPreview?
@@ -144,6 +151,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
thinkingSignature: String? = nil,
mimeType: String?,
fileName: String?,
artifactId: String? = nil,
url: String? = nil,
openUrl: String? = nil,
alt: String? = nil,
width: Int? = nil,
height: Int? = nil,
sizeBytes: Int? = nil,
durationSeconds: Double? = nil,
content: AnyCodable?,
preview: OpenClawChatCanvasPreview? = nil,
@@ -159,6 +173,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
self.thinkingSignature = thinkingSignature
self.mimeType = mimeType
self.fileName = fileName
self.artifactId = artifactId
self.url = url
self.openUrl = openUrl
self.alt = alt
self.width = width
self.height = height
self.sizeBytes = sizeBytes
self.durationSeconds = durationSeconds
self.content = content
self.preview = preview
@@ -176,6 +197,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
case thinkingSignature
case mimeType
case fileName
case artifactId
case url
case openUrl
case alt
case width
case height
case sizeBytes
case durationSeconds
case content
case preview
@@ -195,6 +223,15 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
self.thinkingSignature = try container.decodeIfPresent(String.self, forKey: .thinkingSignature)
self.mimeType = try container.decodeIfPresent(String.self, forKey: .mimeType)
self.fileName = try container.decodeIfPresent(String.self, forKey: .fileName)
let decodedURL = try container.decodeIfPresent(String.self, forKey: .url)
self.url = decodedURL
self.openUrl = try container.decodeIfPresent(String.self, forKey: .openUrl)
self.artifactId = try container.decodeIfPresent(String.self, forKey: .artifactId)
?? Self.managedImageArtifactId(from: decodedURL)
self.alt = try container.decodeIfPresent(String.self, forKey: .alt)
self.width = try container.decodeIfPresent(Int.self, forKey: .width)
self.height = try container.decodeIfPresent(Int.self, forKey: .height)
self.sizeBytes = try container.decodeIfPresent(Int.self, forKey: .sizeBytes)
self.durationSeconds = try container.decodeIfPresent(Double.self, forKey: .durationSeconds)
self.id = try container.decodeIfPresent(String.self, forKey: .id)
self.name = try container.decodeIfPresent(String.self, forKey: .name)
@@ -221,6 +258,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
try container.encodeIfPresent(self.thinkingSignature, forKey: .thinkingSignature)
try container.encodeIfPresent(self.mimeType, forKey: .mimeType)
try container.encodeIfPresent(self.fileName, forKey: .fileName)
try container.encodeIfPresent(self.artifactId, forKey: .artifactId)
try container.encodeIfPresent(self.url, forKey: .url)
try container.encodeIfPresent(self.openUrl, forKey: .openUrl)
try container.encodeIfPresent(self.alt, forKey: .alt)
try container.encodeIfPresent(self.width, forKey: .width)
try container.encodeIfPresent(self.height, forKey: .height)
try container.encodeIfPresent(self.sizeBytes, forKey: .sizeBytes)
try container.encodeIfPresent(self.durationSeconds, forKey: .durationSeconds)
try container.encodeIfPresent(self.content, forKey: .content)
try container.encodeIfPresent(self.preview, forKey: .preview)
@@ -230,6 +274,21 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
try container.encodeIfPresent(self.details, forKey: .details)
try container.encodeIfPresent(self.isError, forKey: .isError)
}
private static func managedImageArtifactId(from rawURL: String?) -> String? {
guard let rawURL,
let components = URLComponents(string: rawURL),
components.scheme == nil,
components.host == nil
else { return nil }
let segments = components.percentEncodedPath.split(separator: "/", omittingEmptySubsequences: true)
guard segments.count == 7,
segments[0...3] == ["api", "chat", "media", "outgoing"],
segments[6] == "full",
let attachmentId = UUID(uuidString: String(segments[5]))?.uuidString.lowercased()
else { return nil }
return "artifact_managed_image_\(attachmentId)"
}
}
public struct OpenClawChatCanvasPreview: Codable, Hashable, Sendable {
@@ -1405,6 +1405,13 @@ extension OpenClawChatSQLiteTranscriptCache {
thinkingSignature: nil,
mimeType: item.mimeType,
fileName: item.fileName,
artifactId: item.artifactId,
url: item.url,
openUrl: item.openUrl,
alt: item.alt,
width: item.width,
height: item.height,
sizeBytes: item.sizeBytes,
durationSeconds: item.durationSeconds,
content: nil,
id: item.id,
@@ -548,6 +548,16 @@ public struct OpenClawChatMetadataCapabilities: Codable, Sendable, Equatable {
}
}
public struct OpenClawChatLoadedImage: Sendable {
public let data: Data
public let mimeType: String
public init(data: Data, mimeType: String) {
self.data = data
self.mimeType = mimeType
}
}
/// One physical Gateway route for Swarm capability discovery and child paging.
/// All pages use the captured route so a reconnect cannot combine two servers.
public struct OpenClawChatSwarmRouteLease: Sendable {
@@ -671,6 +681,7 @@ public protocol OpenClawChatTransport: Sendable {
path: String,
replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL?
func loadImageArtifact(sessionKey: String, artifactId: String) async throws -> OpenClawChatLoadedImage?
func setActiveSessionKey(_ sessionKey: String) async throws
func resetSession(sessionKey: String) async throws
@@ -678,6 +689,13 @@ public protocol OpenClawChatTransport: Sendable {
}
extension OpenClawChatTransport {
public func loadImageArtifact(
sessionKey _: String,
artifactId _: String) async throws -> OpenClawChatLoadedImage?
{
nil
}
public func isSwarmEnabled(sessionKey _: String) async throws -> Bool {
false
}
@@ -544,6 +544,13 @@ public struct OpenClawChatView: View {
inlineWidgetResolverReady: self.viewModel.healthOK,
inlineWidgetResourceResolver: { [weak viewModel] path, failedResource in
await viewModel?.resolveInlineWidgetResource(path: path, replacing: failedResource)
},
imageArtifactResolverReady: self.viewModel.healthOK,
loadImageArtifact: { [weak viewModel] artifactId in
guard let viewModel else { return nil }
return try await viewModel.transport.loadImageArtifact(
sessionKey: viewModel.sessionKey,
artifactId: artifactId)
})
.frame(
maxWidth: .infinity,
@@ -25,6 +25,13 @@ extension OpenClawChatViewModel {
thinkingSignature: content.thinkingSignature,
mimeType: content.mimeType,
fileName: content.fileName,
artifactId: content.artifactId,
url: content.url,
openUrl: content.openUrl,
alt: content.alt,
width: content.width,
height: content.height,
sizeBytes: content.sizeBytes,
durationSeconds: content.durationSeconds,
content: content.content,
id: content.id,
@@ -56,7 +63,12 @@ extension OpenClawChatViewModel {
let id = (item.id ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let name = (item.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let fileName = (item.fileName ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return [type, text, id, name, fileName].joined(separator: "\\u{001F}")
let artifactId = (item.artifactId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let url = (item.url ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let openUrl = (item.openUrl ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let mimeType = (item.mimeType ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return [type, text, id, name, fileName, artifactId, url, openUrl, mimeType]
.joined(separator: "\\u{001F}")
}.joined(separator: "\\u{001E}")
}
@@ -64,7 +76,10 @@ extension OpenClawChatViewModel {
message.content.map { item in
let type = (item.type ?? "text").trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
let text = (item.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return [type, text].joined(separator: "\\u{001F}")
let artifactId = (item.artifactId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let url = (item.url ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let openUrl = (item.openUrl ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return [type, text, artifactId, url, openUrl].joined(separator: "\\u{001F}")
}.joined(separator: "\\u{001E}")
}
@@ -176,6 +191,13 @@ extension OpenClawChatViewModel {
thinkingSignature: content.thinkingSignature,
mimeType: content.mimeType,
fileName: content.fileName,
artifactId: content.artifactId,
url: content.url,
openUrl: content.openUrl,
alt: content.alt,
width: content.width,
height: content.height,
sizeBytes: content.sizeBytes,
durationSeconds: localDuration,
content: content.content,
id: content.id,
@@ -710,6 +710,13 @@ public actor GatewayNodeSession {
socketGeneration: socketGeneration)
}
public func currentGatewayID(ifCurrentRoute route: GatewayNodeSessionRoute) -> String? {
guard self.isCurrentRoute(route), self.channel != nil else { return nil }
// iOS operator routes normalize this to the effective stable ID before connect.
// Keep nil for unscoped clients rather than letting artifact HTTP bind to a guessed owner.
return self.connectOptions?.deviceAuthGatewayID
}
public func supportsServerCapability(
_ capability: GatewayServerCapability,
ifCurrentRoute expectedRoute: GatewayNodeSessionRoute) -> Bool?
@@ -81,6 +81,10 @@ public struct GatewayTLSValidationError: LocalizedError, Sendable {
}
}
public enum GatewayBoundedDataError: Error, Equatable, Sendable {
case responseTooLarge(maximumBytes: Int)
}
protocol GatewayTLSFailureProviding: AnyObject {
func consumeLastTLSFailure() -> GatewayTLSValidationFailure?
}
@@ -764,6 +768,42 @@ public final class GatewayTLSPinningSession: NSObject, WebSocketSessioning, URLS
return WebSocketTaskBox(task: task)
}
public func data(for request: URLRequest, maximumBytes: Int) async throws -> (Data, URLResponse) {
self.registerExpectedAuthority(url: request.url)
guard maximumBytes >= 0 else {
throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes)
}
let (bytes, response) = try await self.session.bytes(for: request)
let expectedLength = response.expectedContentLength
guard expectedLength < 0 || expectedLength <= Int64(maximumBytes) else {
bytes.task.cancel()
throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes)
}
var data = Data()
if expectedLength > 0 {
data.reserveCapacity(Int(expectedLength))
}
do {
for try await byte in bytes {
guard data.count < maximumBytes else {
bytes.task.cancel()
throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes)
}
data.append(byte)
}
} catch {
bytes.task.cancel()
throw error
}
return (data, response)
}
public func finishTasksAndInvalidate() {
self.session.finishTasksAndInvalidate()
}
public func urlSession(
_ session: URLSession,
didReceive challenge: URLAuthenticationChallenge,
@@ -11813,17 +11813,20 @@ public struct ArtifactsDownloadResult: Codable, Sendable {
public let encoding: String?
public let data: String?
public let url: String?
public let expiresat: String?
public init(
artifact: ArtifactSummary,
encoding: String? = nil,
data: String? = nil,
url: String? = nil)
url: String? = nil,
expiresat: String? = nil)
{
self.artifact = artifact
self.encoding = encoding
self.data = data
self.url = url
self.expiresat = expiresat
}
private enum CodingKeys: String, CodingKey {
@@ -11831,6 +11834,7 @@ public struct ArtifactsDownloadResult: Codable, Sendable {
case encoding
case data
case url
case expiresat = "expiresAt"
}
}
@@ -0,0 +1,96 @@
import Foundation
import Testing
@testable import OpenClawChatUI
@Suite("Managed chat image attachments")
struct ChatMessageMediaAttachmentTests {
@Test func `decodes canonical managed image fields`() throws {
let message = try JSONDecoder().decode(
OpenClawChatMessage.self,
from: Data(
"""
{
"role": "assistant",
"content": [{
"type": "image",
"artifactId": "artifact_managed_image_11111111-1111-4111-8111-111111111111",
"url": "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full",
"openUrl": "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full",
"alt": "Chart",
"mimeType": "image/png",
"width": 1200,
"height": 800,
"sizeBytes": 2048
}]
}
""".utf8))
let image = try #require(message.content.first)
#expect(image.artifactId == "artifact_managed_image_11111111-1111-4111-8111-111111111111")
#expect(image.alt == "Chart")
#expect(image.mimeType == "image/png")
#expect(image.width == 1200)
#expect(image.height == 800)
#expect(image.sizeBytes == 2048)
#expect(image.isInlineAttachment)
}
@Test @MainActor func `distinct images never reconcile as the same final message`() {
let first = Self.message(artifactId: "artifact_managed_image_11111111-1111-4111-8111-111111111111")
let second = Self.message(artifactId: "artifact_managed_image_22222222-2222-4222-8222-222222222222")
#expect(
OpenClawChatViewModel.finalMessageContentFingerprint(for: first) !=
OpenClawChatViewModel.finalMessageContentFingerprint(for: second))
#expect(
OpenClawChatViewModel.messageContentFingerprint(for: first) !=
OpenClawChatViewModel.messageContentFingerprint(for: second))
}
@Test func `derives stable identity for shipped managed image blocks`() throws {
let message = try JSONDecoder().decode(
OpenClawChatMessage.self,
from: Data(
"""
{
"role": "assistant",
"content": [{
"type": "image",
"url": "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
"mimeType": "image/png"
}]
}
""".utf8))
#expect(
message.content.first?.artifactId ==
"artifact_managed_image_11111111-1111-4111-8111-111111111111")
}
@Test func `transcript cache preserves references without image bytes`() throws {
let message = Self.message(
artifactId: "artifact_managed_image_11111111-1111-4111-8111-111111111111")
let cached = try #require(OpenClawChatSQLiteTranscriptCache.cacheableMessages([message]).first)
let image = try #require(cached.content.first)
#expect(image.artifactId == message.content.first?.artifactId)
#expect(image.url == message.content.first?.url)
#expect(image.content == nil)
}
private static func message(artifactId: String) -> OpenClawChatMessage {
OpenClawChatMessage(
role: "assistant",
content: [
OpenClawChatMessageContent(
type: "image",
text: nil,
mimeType: "image/png",
fileName: nil,
artifactId: artifactId,
url: "/api/chat/media/outgoing/main/\(artifactId)/full",
content: nil),
],
timestamp: nil)
}
}
+1
View File
@@ -3310,6 +3310,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
- H2: Choose scopes and pair the device
- H2: Advertise client capabilities
- H2: Recover state after reconnect
- H2: Render generated image artifacts
- H2: Use history metadata and stable anchors
- H2: Subscribe instead of polling usage
- H2: Backfill exec approvals
+24
View File
@@ -133,6 +133,30 @@ current WebSocket connection. It resets with a new connection. The `seq` inside
an `agent` event payload is assigned per run and orders that run's lifecycle,
assistant, plan, tool, and other stream events.
## Render generated image artifacts
Assistant-generated images arrive as canonical `type: "image"` content blocks.
Managed blocks include a stable `artifactId`, a Gateway-relative `url`, MIME
type, dimensions, size, and accessible alt text. Keep that reference in the
transcript cache; do not persist downloaded bytes or temporary download URLs.
Resolve the image through the authenticated WebSocket connection:
1. Call `artifacts.download` with the current `sessionKey`, optional `agentId`,
and the block's `artifactId`.
2. Use the returned short-lived `url` before `expiresAt`. The URL is scoped to
that exact transcript-backed artifact and does not contain a reusable Gateway
or device credential.
3. Fetch it from the Gateway origin using the same TLS pin and reverse-proxy
headers as the active connection. Validate the response as an image and
enforce a 12 MiB source limit plus a bounded decoded thumbnail.
4. If the URL expires, repeat `artifacts.download` once. Reconnect or route
changes cancel the old load rather than retargeting it to another Gateway.
Older image blocks without `artifactId` remain displayable by existing Control
UI clients, but native clients should show a readable attachment fallback rather
than forward a shared owner credential.
## Use history metadata and stable anchors
Rows returned by `chat.history` can carry an `__openclaw` metadata envelope:
+1
View File
@@ -301,6 +301,7 @@ The Android Chat tab supports session selection (default `main`, plus other exis
- History: `chat.history` (display-normalized — inline directive tags, plain-text tool-call XML payloads (`<tool_call>`, `<function_call>`, `<tool_calls>`, `<function_calls>`, and truncated variants), and leaked ASCII/full-width model control tokens are stripped; silent-token assistant rows such as exact `NO_REPLY` / `no_reply` are omitted; oversized rows can be replaced with placeholders)
- Send: `chat.send`
- Durable sending: every send (text, picked images, and voice notes) is journaled to a per-gateway on-device outbox before any network attempt, so app termination cannot lose submitted input. Sends queued while offline deliver in order on reconnect with stable idempotency keys, and a send is retired only after the turn is visible in canonical `chat.history` — an acknowledgement alone is not treated as proof of delivery. Ambiguous outcomes (lost acknowledgement, app killed mid-send, gateway restart before the transcript write) surface as visible rows with explicit **Retry**/**Delete** instead of auto-resending. Slash commands never auto-replay across a reconnect; they park for explicit retry. The queue is bounded (50 messages and 48 MB of attachment bytes per gateway) and unsent rows expire after 48 hours. Composer drafts that were never submitted are not process-durable.
- Image input works through the picker and Android Sharesheet. Assistant-generated images resolve through the paired Gateway connection, render inline with a full-screen preview, and retain only their small artifact references in the offline transcript cache. Downloads are capped at 12 MiB and decoded to bounded display bitmaps.
- Push updates (best-effort): `chat.subscribe` -> `event:"chat"`
- Listen: long-press an assistant message and choose **Listen** to hear it; audio renders via gateway `tts.speak` with the configured TTS provider chain, and on-device system TTS is used when the gateway cannot render audio. Playback stops on session switch, new chat, app backgrounding, or chat close.
+1
View File
@@ -19,6 +19,7 @@ Availability: iPhone app builds are distributed through Apple channels when enab
- Keeps a small read-only offline cache of recent chat sessions and transcripts per paired gateway: cold opens paint the last known transcript immediately and refresh once the gateway responds, recent chats stay browsable while disconnected, and reset/forget purges the protected local cache.
- Queues text messages sent while disconnected in a durable per-gateway outbox (up to 50): queued bubbles show in the transcript, flush in order on reconnect with idempotent retries, remain durable until canonical history confirms the send, retry with backoff before surfacing a retry/delete action, and expire instead of sending after 48 hours offline; reset/forget clears the queue with the cache.
- Chat is the single text-and-voice surface. Chat actions can open the full Sessions screen without leaving Chat and can show or hide assistant reasoning and tool activity. Tap the microphone for draft dictation, open its menu to record a voice note, or use the inline Talk control for realtime voice; the Talk control animates from live microphone or playback level while listening or speaking.
- Chat accepts images from the photo picker, camera, Files, paste, and the iOS share sheet. Assistant-generated images render inline from short-lived Gateway artifact URLs, open in a full-screen preview, and remain available after reconnect or history reload without storing image bytes in the transcript cache.
- **Settings -> OpenClaw** opens a dedicated Gateway settings assistant when the operator connection has `operator.admin` and the Gateway supports `openclaw.chat`. Its setup conversation stays separate from ordinary Chat, redacts secret replies locally, and moves to Chat only after you tap **Open Chat**.
- Speaks assistant messages on demand: long-press a message in Chat and choose **Listen**. The app plays supported gateway `tts.speak` clips with the configured TTS provider and falls back to on-device speech when gateway audio is unavailable or unplayable. Playback stops on session switch or backgrounding.
+5
View File
@@ -13,6 +13,11 @@ Mac-hosted node tools such as `system.run`.
Use **Quick Chat** for a Spotlight-style main-session composer without opening a full window. Press Option-Space (⌥Space) by default, choose it from the menu bar menu, or record another shortcut in **Settings → General**.
The full native chat accepts image attachments through its picker, paste, and
drag and drop. Assistant-generated images render inline through short-lived
Gateway artifact URLs and open in a larger preview; iOS and macOS share the same
bounded image model and renderer.
Only need the CLI and Gateway? Start with [Getting started](/start/getting-started).
## Download
+8 -1
View File
@@ -417,7 +417,7 @@ The macOS app keeps its native link-browser sidebar for links clicked in the das
- Re-sending with the same `idempotencyKey` returns `{ status: "in_flight" }` while running, and `{ status: "ok" }` after completion.
- `chat.history` responses are size-bounded for UI safety. When transcript entries are too large, Gateway may truncate long text fields, omit heavy metadata blocks, and replace oversized messages with a placeholder (`[chat.history omitted: message too large]`).
- When a visible assistant message was truncated in `chat.history`, the side reader can fetch the full display-normalized transcript entry on demand through `chat.message.get` by `sessionKey`, active `agentId` when needed, and transcript `messageId`. If the Gateway still cannot return more, the reader shows an explicit unavailable state instead of silently repeating the truncated preview.
- Assistant/generated images are persisted as managed media references and served back through authenticated Gateway media URLs, so reloads do not depend on raw base64 image payloads staying in the chat history response.
- Assistant/generated images are persisted as managed media references. New clients resolve their stable artifact ids through authenticated `artifacts.download` and receive short-lived, exact-resource media URLs, so reloads do not depend on raw base64 payloads or reusable credentials in image URLs.
- When rendering `chat.history`, the Control UI strips display-only inline directive tags from visible assistant text (for example `[[reply_to_*]]` and `[[audio_as_voice]]`), plain-text tool-call XML payloads (including `<tool_call>...</tool_call>`, `<function_call>...</function_call>`, `<tool_calls>...</tool_calls>`, `<function_calls>...</function_calls>`, and truncated tool-call blocks), and leaked ASCII/full-width model control tokens. It omits assistant entries whose whole visible text is only the exact silent token `NO_REPLY` / `no_reply` or the heartbeat acknowledgement token `HEARTBEAT_OK`.
- During an active send and the final history refresh, the chat view keeps local optimistic user/assistant messages visible if `chat.history` briefly returns an older snapshot; the canonical transcript replaces those local messages once the Gateway history catches up.
- Live `chat` events are delivery state, while `chat.history` is rebuilt from the durable session transcript. After tool-final events the Control UI reloads history and merges only a small optimistic tail; the transcript boundary is documented in [WebChat](/web/webchat).
@@ -663,6 +663,13 @@ When gateway auth is configured, assistant local-media previews use a two-step r
This keeps media rendering compatible with browser-native media elements without putting reusable gateway credentials in visible media URLs.
Generated images under `/api/chat/media/outgoing/...` use the same capability
principle through `artifacts.download`. The authenticated WebSocket request
authorizes the transcript artifact and returns a short-lived URL. The HTTP media
route rechecks that the artifact still belongs to the transcript before serving
bytes. The previous shared-owner bearer path remains available for older Control
UI clients during the compatibility window.
## Approval links
Operator approval notifications can deep-link to a [standalone approval document](/web/urls#special-documents-and-startup-modes). The URL is stable for the lifetime of the approval and safe to forward between your own devices: it identifies the approval, never authorizes it.
@@ -68,6 +68,7 @@ export const ArtifactsDownloadResultSchema = closedObject({
encoding: Type.Optional(Type.Literal("base64")),
data: Type.Optional(Type.String()),
url: Type.Optional(NonEmptyString),
expiresAt: Type.Optional(NonEmptyString),
});
// Wire types derive directly from local schema consts so public d.ts graphs never
@@ -55,10 +55,12 @@ vi.mock("./session-transcript-readers.js", () => ({
const {
DEFAULT_MANAGED_IMAGE_ATTACHMENT_LIMITS,
MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX,
attachManagedOutgoingImagesToMessage,
cleanupManagedOutgoingImageRecords,
createManagedOutgoingImageBlocks,
handleManagedOutgoingImageHttpRequest,
resolveManagedOutgoingImageArtifactDownload,
resolveManagedImageAttachmentLimits,
} = await import("./managed-image-attachments.js");
@@ -101,8 +103,10 @@ async function expectPathMissing(targetPath: string): Promise<void> {
type ManagedImageBlock = {
type?: string;
artifactId?: string;
alt?: string;
mimeType?: string;
sizeBytes?: number;
url?: string;
openUrl?: string;
};
@@ -319,6 +323,50 @@ describe("handleManagedOutgoingImageHttpRequest", () => {
);
});
it("serves an exact transcript image through a short-lived artifact ticket", async () => {
const { attachmentId, sessionKey } = await createFixture(stateDir);
const canonicalPath = `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/${attachmentId}/full`;
loadSessionEntryMock.mockReturnValue({
storePath: path.join(stateDir, "gateway-sessions.json"),
entry: { sessionId: "sess-1", sessionFile: "session.jsonl" },
});
resolveSessionHistoryTranscriptPathMock.mockResolvedValue("session.jsonl");
readSessionMessagesMock.mockResolvedValue([
{
role: "assistant",
content: [{ type: "image", url: canonicalPath, openUrl: canonicalPath }],
__openclaw: { id: "msg-1" },
},
]);
const download = await resolveManagedOutgoingImageArtifactDownload({
sessionKey,
artifactId: `${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`,
stateDir,
});
expect(download?.url).toContain("mediaTicket=");
vi.clearAllMocks();
const { result } = await requestManagedImage({
stateDir,
pathName: download?.url ?? "",
denyAuth: true,
});
expect(result.statusCode).toBe(200);
expect(result.body.toString("utf-8")).toBe("original-image");
expect(authorizeGatewayHttpRequestOrReplyMock).not.toHaveBeenCalled();
const wrongAttachmentId = "22222222-2222-4222-8222-222222222222";
const wrong = await requestManagedImage({
stateDir,
pathName: (download?.url ?? "").replace(attachmentId, wrongAttachmentId),
denyAuth: true,
});
expect(wrong.result.statusCode).toBe(401);
expect(authorizeGatewayHttpRequestOrReplyMock).toHaveBeenCalledTimes(1);
});
it("keeps serving and deleting an original after the configured media root changes", async () => {
const fixture = await createFixture(stateDir);
const externalConfigDir = tempDirs.make("managed-image-moved-config-");
@@ -653,6 +701,8 @@ describe("createManagedOutgoingImageBlocks", () => {
expect(String(block.url)).toMatch(/\/full$/);
const attachmentId = requireAttachmentIdFromUrl(block.url);
expect(block.artifactId).toBe(`${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`);
expect(block.sizeBytes).toBe(Buffer.from(TINY_PNG_BASE64, "base64").byteLength);
const record = readManagedImageRecord(attachmentId, stateDir);
expect(record?.original.mediaSubdir).toBe(MANAGED_OUTGOING_ORIGINALS_SUBDIR);
expect(record?.original.mediaId).toMatch(/\.png$/);
+230 -34
View File
@@ -1,10 +1,14 @@
// Gateway managed image attachment store.
// Validates, stores, serves, and cleans up outgoing image attachments.
import { randomUUID } from "node:crypto";
import { createHmac, randomBytes, randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import type { IncomingMessage, ServerResponse } from "node:http";
import path from "node:path";
import { expectDefined } from "@openclaw/normalization-core";
import {
asDateTimestampMs,
resolveTimestampMsToIsoString,
} from "@openclaw/normalization-core/number-coercion";
import { resolveDefaultAgentId } from "../agents/agent-scope-config.js";
import { getRuntimeConfig } from "../config/config.js";
import { resolveStateDir } from "../config/paths.js";
@@ -17,6 +21,7 @@ import {
readImageProbeFromHeader,
} from "../media/media-services.js";
import { getMediaDir, MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "../media/store.js";
import { safeEqualSecret } from "../security/secret-equal.js";
import type { AuthRateLimiter } from "./auth-rate-limit.js";
import type { ResolvedGatewayAuth } from "./auth.js";
import { sendJson, sendMethodNotAllowed, sendMissingScopeForbidden } from "./http-common.js";
@@ -44,8 +49,12 @@ import {
const OUTGOING_IMAGE_ROUTE_PREFIX = "/api/chat/media/outgoing";
const DEFAULT_TRANSIENT_OUTGOING_IMAGE_TTL_MS = 15 * 60 * 1000;
const MANAGED_OUTGOING_IMAGE_TICKET_SCOPE = "managed-outgoing-image";
export const MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS = 5 * 60 * 1000;
export const MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX = "artifact_managed_image_";
const MANAGED_OUTGOING_ATTACHMENT_ID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const managedOutgoingImageTicketSecret = randomBytes(32);
export const DEFAULT_MANAGED_IMAGE_ATTACHMENT_LIMITS = {
maxBytes: 12 * 1024 * 1024,
@@ -86,6 +95,24 @@ type SessionManagedOutgoingAttachmentIndexCacheEntry = {
size: number;
index: SessionManagedOutgoingAttachmentIndex;
};
type ManagedOutgoingImageTicketPayload = {
scope: typeof MANAGED_OUTGOING_IMAGE_TICKET_SCOPE;
sessionKey: string;
attachmentId: string;
variant: "full";
exp: number;
};
export type ManagedOutgoingImageArtifactDownload = {
artifactId: string;
sessionKey: string;
title: string;
mimeType?: string;
sizeBytes?: number;
url: string;
expiresAt: string;
};
type SessionManagedOutgoingAttachmentTranscriptStat = Omit<
SessionManagedOutgoingAttachmentIndexCacheEntry,
"index"
@@ -310,6 +337,91 @@ function buildOutgoingVariantUrl(sessionKey: string, attachmentId: string, varia
return `${OUTGOING_IMAGE_ROUTE_PREFIX}/${encodeURIComponent(sessionKey)}/${attachmentId}/${variant}`;
}
function buildManagedOutgoingImageArtifactId(attachmentId: string): string {
return `${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`;
}
export function parseManagedOutgoingImageArtifactId(value: string): string | null {
if (!value.startsWith(MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX)) {
return null;
}
const attachmentId = value.slice(MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX.length);
return MANAGED_OUTGOING_ATTACHMENT_ID_RE.test(attachmentId) ? attachmentId : null;
}
function signManagedOutgoingImageTicketPayload(encodedPayload: string): string {
return createHmac("sha256", managedOutgoingImageTicketSecret)
.update(encodedPayload)
.digest("base64url");
}
function createManagedOutgoingImageTicket(params: {
sessionKey: string;
attachmentId: string;
nowMs?: number;
}): { ticket: string; expiresAt: string } | null {
const now = asDateTimestampMs(params.nowMs ?? Date.now());
if (now === undefined) {
return null;
}
const exp = asDateTimestampMs(now + MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS);
if (exp === undefined) {
return null;
}
const payload: ManagedOutgoingImageTicketPayload = {
scope: MANAGED_OUTGOING_IMAGE_TICKET_SCOPE,
sessionKey: params.sessionKey,
attachmentId: params.attachmentId,
variant: "full",
exp,
};
const encodedPayload = Buffer.from(JSON.stringify(payload), "utf8").toString("base64url");
const signature = signManagedOutgoingImageTicketPayload(encodedPayload);
return {
ticket: `v1.${encodedPayload}.${signature}`,
expiresAt: resolveTimestampMsToIsoString(exp),
};
}
function verifyManagedOutgoingImageTicket(params: {
ticket: string | null;
sessionKey: string;
attachmentId: string;
nowMs?: number;
}): boolean {
const now = asDateTimestampMs(params.nowMs ?? Date.now());
if (now === undefined) {
return false;
}
const parts = params.ticket?.split(".");
if (!parts || parts.length !== 3 || parts[0] !== "v1") {
return false;
}
const [, encodedPayload, signature] = parts;
if (!encodedPayload || !signature) {
return false;
}
if (!safeEqualSecret(signature, signManagedOutgoingImageTicketPayload(encodedPayload))) {
return false;
}
try {
const payload = JSON.parse(
Buffer.from(encodedPayload, "base64url").toString("utf8"),
) as Partial<ManagedOutgoingImageTicketPayload>;
return (
payload.scope === MANAGED_OUTGOING_IMAGE_TICKET_SCOPE &&
payload.sessionKey === params.sessionKey &&
payload.attachmentId === params.attachmentId &&
payload.variant === "full" &&
typeof payload.exp === "number" &&
Number.isFinite(payload.exp) &&
payload.exp >= now
);
} catch {
return false;
}
}
function deriveAltText(source: string, index: number) {
const fallback = `Generated image ${index + 1}`;
try {
@@ -508,12 +620,14 @@ function buildManagedImageBlock(record: ManagedImageRecord): ManagedImageBlock {
const fullUrl = buildOutgoingVariantUrl(record.sessionKey, record.attachmentId, "full");
return {
type: "image",
artifactId: buildManagedOutgoingImageArtifactId(record.attachmentId),
url: fullUrl,
openUrl: fullUrl,
alt: record.alt,
mimeType: record.original.contentType,
width: record.original.width,
height: record.original.height,
sizeBytes: record.original.sizeBytes,
};
}
@@ -565,7 +679,7 @@ function parseManagedOutgoingRoute(value: string) {
sessionKey: decodeURIComponent(
expectDefined(match[1], "managed image attachments regex capture 1"),
),
attachmentId: match[2],
attachmentId: expectDefined(match[2], "managed image attachments regex capture 2"),
};
} catch {
return null;
@@ -792,6 +906,74 @@ async function recordMatchesTranscriptMessage(
);
}
async function resolveManagedOutgoingImageArtifactDownloadForRecord(
record: ManagedImageRecord,
): Promise<ManagedOutgoingImageArtifactDownload | null> {
if (!(await recordMatchesTranscriptMessage(record))) {
return null;
}
const ticket = createManagedOutgoingImageTicket({
sessionKey: record.sessionKey,
attachmentId: record.attachmentId,
});
if (!ticket) {
return null;
}
try {
const stat = await fs.stat(resolveManagedImageOriginalPath(record));
if (!stat.isFile()) {
return null;
}
} catch {
return null;
}
const canonicalUrl = buildOutgoingVariantUrl(record.sessionKey, record.attachmentId, "full");
const params = new URLSearchParams({ mediaTicket: ticket.ticket });
return {
artifactId: buildManagedOutgoingImageArtifactId(record.attachmentId),
sessionKey: record.sessionKey,
title: record.alt,
...(record.original.contentType ? { mimeType: record.original.contentType } : {}),
...(record.original.sizeBytes != null ? { sizeBytes: record.original.sizeBytes } : {}),
url: `${canonicalUrl}?${params.toString()}`,
expiresAt: ticket.expiresAt,
};
}
/** Resolve one transcript-backed image to a short-lived HTTP capability. */
export async function resolveManagedOutgoingImageArtifactDownload(params: {
sessionKey: string;
artifactId: string;
stateDir?: string;
}): Promise<ManagedOutgoingImageArtifactDownload | null> {
const attachmentId = parseManagedOutgoingImageArtifactId(params.artifactId);
if (!attachmentId) {
return null;
}
const record = readManagedImageRecord(attachmentId, params.stateDir);
if (!record || record.sessionKey !== params.sessionKey) {
return null;
}
return await resolveManagedOutgoingImageArtifactDownloadForRecord(record);
}
/** Upgrade legacy managed-image URLs that predate stable artifact ids. */
export async function resolveManagedOutgoingImageUrlDownload(params: {
sessionKey: string;
url: string;
stateDir?: string;
}): Promise<ManagedOutgoingImageArtifactDownload | null> {
const parsed = parseManagedOutgoingRoute(params.url);
if (!parsed || parsed.sessionKey !== params.sessionKey) {
return null;
}
const record = readManagedImageRecord(parsed.attachmentId, params.stateDir);
if (!record || record.sessionKey !== params.sessionKey) {
return null;
}
return await resolveManagedOutgoingImageArtifactDownloadForRecord(record);
}
export async function attachManagedOutgoingImagesToMessage(params: {
messageId: string;
blocks?: readonly Record<string, unknown>[];
@@ -1051,25 +1233,6 @@ export async function handleManagedOutgoingImageHttpRequest(
return true;
}
const requestAuth = await authorizeGatewayHttpRequestOrReply({
req,
res,
auth: opts.auth,
trustedProxies: opts.trustedProxies,
allowRealIpFallback: opts.allowRealIpFallback,
rateLimiter: opts.rateLimiter,
});
if (!requestAuth) {
return true;
}
const requestedScopes = resolveOpenAiCompatibleHttpOperatorScopes(req, requestAuth);
const scopeAuth = authorizeOperatorScopesForMethod("chat.history", requestedScopes);
if (!scopeAuth.allowed) {
sendMissingScopeForbidden(res, scopeAuth.missingScope);
return true;
}
const encodedSessionKey = match[1];
const attachmentId = match[2];
if (!encodedSessionKey || !attachmentId) {
@@ -1086,23 +1249,49 @@ export async function handleManagedOutgoingImageHttpRequest(
sendStatus(res, 404, "not found");
return true;
}
const hasValidMediaTicket = verifyManagedOutgoingImageTicket({
ticket: requestUrl.searchParams.get("mediaTicket"),
sessionKey,
attachmentId,
});
if (!hasValidMediaTicket) {
const requestAuth = await authorizeGatewayHttpRequestOrReply({
req,
res,
auth: opts.auth,
trustedProxies: opts.trustedProxies,
allowRealIpFallback: opts.allowRealIpFallback,
rateLimiter: opts.rateLimiter,
});
if (!requestAuth) {
return true;
}
const requestedScopes = resolveOpenAiCompatibleHttpOperatorScopes(req, requestAuth);
const scopeAuth = authorizeOperatorScopesForMethod("chat.history", requestedScopes);
if (!scopeAuth.allowed) {
sendMissingScopeForbidden(res, scopeAuth.missingScope);
return true;
}
// The reusable shared-secret route remains for older Control UI clients.
// Ticketed clients prove the exact transcript attachment instead of
// forwarding an owner credential through another HTTP stack.
if (!resolveOpenAiCompatibleHttpSenderIsOwner(req, requestAuth)) {
sendJson(res, 403, {
ok: false,
error: {
type: "forbidden",
message: "owner access required",
},
});
return true;
}
}
const record = readManagedImageRecord(attachmentId, opts.stateDir);
if (!record || record.sessionKey !== sessionKey) {
sendStatus(res, 404, "not found");
return true;
}
// Requester-session headers are client-declared, so media bytes require
// authenticated owner/admin context rather than trusting a URL-scoped header.
if (!resolveOpenAiCompatibleHttpSenderIsOwner(req, requestAuth)) {
sendJson(res, 403, {
ok: false,
error: {
type: "forbidden",
message: "owner access required",
},
});
return true;
}
if (!(await recordMatchesTranscriptMessage(record))) {
sendStatus(res, 404, "not found");
return true;
@@ -1123,7 +1312,14 @@ export async function handleManagedOutgoingImageHttpRequest(
res.statusCode = 200;
res.setHeader("content-type", record.original.contentType || "application/octet-stream");
res.setHeader("content-length", String(body.byteLength));
res.setHeader("cache-control", "private, max-age=31536000, immutable");
res.setHeader("x-content-type-options", "nosniff");
res.setHeader("referrer-policy", "no-referrer");
res.setHeader(
"cache-control",
hasValidMediaTicket
? `private, max-age=${MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS / 1000}, immutable`
: "private, max-age=31536000, immutable",
);
res.setHeader(
"content-disposition",
`inline; filename="${safeAttachmentFilename(record.original.filename)}"`,
@@ -7,6 +7,8 @@ import { artifactsHandlers } from "./artifacts.js";
const hoisted = vi.hoisted(() => ({
getTaskSessionLookupByIdForStatus: vi.fn(),
loadSessionEntry: vi.fn(),
resolveManagedArtifactDownload: vi.fn(),
resolveManagedUrlDownload: vi.fn(),
visitSessionMessagesAsync: vi.fn(),
resolveSessionKeyForRun: vi.fn(),
}));
@@ -44,6 +46,17 @@ vi.mock("../server-session-key.js", async () => {
};
});
vi.mock("../managed-image-attachments.js", async () => {
const actual = await vi.importActual<typeof import("../managed-image-attachments.js")>(
"../managed-image-attachments.js",
);
return {
...actual,
resolveManagedOutgoingImageArtifactDownload: hoisted.resolveManagedArtifactDownload,
resolveManagedOutgoingImageUrlDownload: hoisted.resolveManagedUrlDownload,
};
});
function createResponder() {
const calls: Array<{ ok: boolean; payload?: unknown; error?: unknown }> = [];
return {
@@ -211,6 +224,8 @@ describe("artifacts RPC handlers", () => {
beforeEach(() => {
vi.clearAllMocks();
hoisted.resolveSessionKeyForRun.mockReset();
hoisted.resolveManagedArtifactDownload.mockResolvedValue(null);
hoisted.resolveManagedUrlDownload.mockResolvedValue(null);
hoisted.getTaskSessionLookupByIdForStatus.mockReturnValue(undefined);
hoisted.loadSessionEntry.mockReturnValue({
storePath: "/tmp/sessions.json",
@@ -360,6 +375,48 @@ describe("artifacts RPC handlers", () => {
expectFields(downloadPayload.artifact, { id: artifactId });
});
it("preserves managed artifact identity and returns a ticketed download URL", async () => {
const artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111";
mockedMessages([
{
role: "assistant",
content: [
{
type: "image",
artifactId,
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full",
alt: "chart.png",
mimeType: "image/png",
sizeBytes: 14,
},
],
__openclaw: { seq: 2 },
},
]);
hoisted.resolveManagedArtifactDownload.mockResolvedValue({
artifactId,
sessionKey: "agent:main:main",
title: "chart.png",
mimeType: "image/png",
sizeBytes: 14,
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/id/full?mediaTicket=ticket",
expiresAt: "2026-07-28T05:00:00.000Z",
});
const listed = await listArtifacts({ sessionKey: "agent:main:main" });
expectFields(expectFirstArtifact(listed.calls), { id: artifactId, sizeBytes: 14 });
const downloaded = await downloadArtifact({ sessionKey: "agent:main:main", artifactId });
expectFields(expectOkPayload(downloaded.calls), {
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/id/full?mediaTicket=ticket",
expiresAt: "2026-07-28T05:00:00.000Z",
});
expect(hoisted.resolveManagedArtifactDownload).toHaveBeenCalledWith({
sessionKey: "agent:main:main",
artifactId,
});
});
it("can scan artifact summaries without retaining inline data", async () => {
mockedMessages([
{
+60 -8
View File
@@ -22,6 +22,11 @@ import {
toAgentStoreSessionKey,
} from "../../routing/session-key.js";
import { getTaskSessionLookupByIdForStatus } from "../../tasks/task-status-access.js";
import {
parseManagedOutgoingImageArtifactId,
resolveManagedOutgoingImageArtifactDownload,
resolveManagedOutgoingImageUrlDownload,
} from "../managed-image-attachments.js";
import { resolveSessionKeyForRun } from "../server-session-key.js";
import {
resolveSessionStoreAgentId,
@@ -411,13 +416,17 @@ function collectArtifactsFromMessage(params: {
asNonEmptyString(block.filename) ??
asNonEmptyString(block.alt) ??
`${type} ${params.artifacts.length + 1}`;
const id = artifactId({
sessionKey: params.sessionKey,
messageSeq,
contentIndex,
title,
type,
});
const declaredArtifactId = asNonEmptyString(block.artifactId);
const id =
declaredArtifactId && parseManagedOutgoingImageArtifactId(declaredArtifactId)
? declaredArtifactId
: artifactId({
sessionKey: params.sessionKey,
messageSeq,
contentIndex,
title,
type,
});
const includeData = params.downloadArtifactId
? params.downloadArtifactId === id
: params.includeDownloadData !== false;
@@ -639,6 +648,37 @@ export const artifactsHandlers: GatewayRequestHandlers = {
if (!requireQueryable(params, respond)) {
return;
}
if (
params.sessionKey &&
!params.runId &&
!params.taskId &&
parseManagedOutgoingImageArtifactId(params.artifactId)
) {
const resolved = resolveQuerySession(params, context.getRuntimeConfig?.());
const managed = resolved
? await resolveManagedOutgoingImageArtifactDownload({
sessionKey: resolved.sessionKey,
artifactId: params.artifactId,
})
: null;
if (managed) {
respond(true, {
artifact: {
id: managed.artifactId,
type: "image",
title: managed.title,
...(managed.mimeType ? { mimeType: managed.mimeType } : {}),
...(managed.sizeBytes !== undefined ? { sizeBytes: managed.sizeBytes } : {}),
sessionKey: managed.sessionKey,
source: "session-transcript",
download: { mode: "url" as const },
},
url: managed.url,
expiresAt: managed.expiresAt,
});
return;
}
}
const { artifact } = await findArtifact(params, context.getRuntimeConfig?.(), {
downloadArtifactId: params.artifactId,
});
@@ -662,12 +702,24 @@ export const artifactsHandlers: GatewayRequestHandlers = {
);
return;
}
const managedUrl =
artifact.download.mode === "url" && artifact.url && artifact.sessionKey
? await resolveManagedOutgoingImageUrlDownload({
sessionKey: artifact.sessionKey,
url: artifact.url,
})
: null;
respond(true, {
artifact: toSummary(artifact),
...(artifact.download.mode === "bytes"
? { encoding: "base64" as const, data: artifact.data }
: {}),
...(artifact.download.mode === "url" ? { url: artifact.url } : {}),
...(artifact.download.mode === "url"
? {
url: managedUrl?.url ?? artifact.url,
...(managedUrl ? { expiresAt: managedUrl.expiresAt } : {}),
}
: {}),
});
},
};
+1
View File
@@ -464,6 +464,7 @@ export type ArtifactDownloadResult = {
encoding?: "base64";
data?: string;
url?: string;
expiresAt?: string;
};
export type SessionRunStatus = "running" | "done" | "failed" | "killed" | "timeout";
@@ -109,6 +109,83 @@ suite.define(() => {
}
});
it("renders a managed image through an artifact-scoped ticket", async () => {
const context = await suite.newBrowserContext({
locale: "en-US",
serviceWorkers: "block",
viewport: { height: 900, width: 1280 },
});
const page = await context.newPage();
const attachmentId = crypto.randomUUID();
const artifactId = `artifact_managed_image_${attachmentId}`;
const imageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${attachmentId}/full`;
const ticketedUrl = `${imageUrl}?mediaTicket=ticket-e2e`;
await page.route("**/api/chat/media/outgoing/**", async (route) => {
const request = route.request();
expect(new URL(request.url()).searchParams.get("mediaTicket")).toBe("ticket-e2e");
expect(request.headers().authorization).toBeUndefined();
await route.fulfill({
contentType: "image/png",
body: Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=",
"base64",
),
});
});
const gateway = await installMockGateway(page, {
historyMessages: [
{
role: "assistant",
content: [
{
type: "image",
artifactId,
url: imageUrl,
alt: "Ticketed generated image",
mimeType: "image/png",
width: 1,
height: 1,
},
],
timestamp: Date.now(),
},
],
methodResponses: {
"artifacts.download": {
artifact: {
id: artifactId,
type: "image",
title: "Ticketed generated image",
mimeType: "image/png",
download: { mode: "url" },
},
url: ticketedUrl,
expiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
},
},
});
try {
await page.goto(`${suite.server.baseUrl}chat`);
const image = page.getByAltText("Ticketed generated image");
await image.waitFor({ state: "visible", timeout: 10_000 });
await expect
.poll(() =>
image.evaluate((element) =>
element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0,
),
)
.toBe(1);
const request = await gateway.waitForRequest("artifacts.download");
expect(request.params).toMatchObject({
sessionKey: "agent:main:main",
artifactId,
});
} finally {
await suite.closeBrowserContext(context);
}
});
it("renders a canonical inbound image through the ticketed media route", async () => {
const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim();
const context = await suite.newBrowserContext({
+26 -1
View File
@@ -1,6 +1,31 @@
import { describe, expect, it } from "vitest";
import type { SessionObserverDigest } from "../../../packages/gateway-protocol/src/schema/sessions.js";
import { isCriticalObserverHealth, ObserverDigestHistory } from "./observer-digest.ts";
import {
isCriticalObserverHealth,
ObserverDigestHistory,
projectSessionObserverDigest,
} from "./observer-digest.ts";
describe("projectSessionObserverDigest", () => {
it("binds a session-row projection to its owning session", () => {
expect(
projectSessionObserverDigest("agent:main:projected", {
runId: "run-1",
revision: 2,
updatedAt: 3,
headline: "Projected",
health: "on-track",
}),
).toEqual({
sessionKey: "agent:main:projected",
runId: "run-1",
revision: 2,
updatedAt: 3,
headline: "Projected",
health: "on-track",
});
});
});
describe("isCriticalObserverHealth", () => {
it("recognizes only health states that require operator attention", () => {
+18
View File
@@ -13,6 +13,24 @@ type ProjectedObserverDigest = Pick<
"agentId" | "runId" | "headline" | "health" | "updatedAt" | "revision"
>;
export function projectSessionObserverDigest(
sessionKey: string,
digest: ProjectedObserverDigest | null | undefined,
): SessionObserverDigest | null {
if (!digest) {
return null;
}
return {
sessionKey,
...(digest.agentId ? { agentId: digest.agentId } : {}),
runId: digest.runId,
revision: digest.revision,
updatedAt: digest.updatedAt,
headline: digest.headline,
health: digest.health,
};
}
export function isCriticalObserverHealth(health: unknown): health is "stuck" | "waiting-on-user" {
return health === "stuck" || health === "waiting-on-user";
}
+2
View File
@@ -115,6 +115,7 @@ export {
export {
ObserverDigestHistory,
pickFreshestObserverDigest,
projectSessionObserverDigest,
resolveChatPaneObserverRunId,
} from "../../lib/observer-digest.ts";
export { isWorkboardEnabledInConfigSnapshot } from "../../lib/plugin-activation.ts";
@@ -175,6 +176,7 @@ export { sendSessionObserverVisibility } from "./chat-observer.ts";
export {
applySelectedSessionProjection,
dismissChatError,
resolveChatArtifactDownload,
resolveAssistantAttachmentAuthToken,
SessionParticipationTracker,
} from "./chat-pane-state.ts";
+7 -14
View File
@@ -21,6 +21,7 @@ import {
openSessionWorkspaceFile,
parseCatalogSessionKey,
pickFreshestObserverDigest,
projectSessionObserverDigest,
readPresenceEntries,
refreshChatCommands,
refreshPageChat,
@@ -30,6 +31,7 @@ import {
resolveActiveRunOutputTokens,
resolveChatProjectionRunId,
resolveAssistantAttachmentAuthToken,
resolveChatArtifactDownload,
resolveChatAgentId,
resolveChatAvatarUrl,
resolveControlUiFollowUpMode,
@@ -54,7 +56,6 @@ import {
workspaceResultConflictFromPlacement,
type BoardViewCallbacks,
type ChatProps,
type SessionObserverDigest,
type SidebarSide,
type SidebarSlotId,
} from "./chat-pane-deps.ts";
@@ -79,19 +80,10 @@ export class ChatPaneRender extends ChatPaneHeaderRender {
return html`<main class="app-shell app-shell--booting" aria-busy="true"></main>`;
}
const selectedSession = selectedChatSessionRow(state);
const projectedObserverDigest: SessionObserverDigest | null = selectedSession?.observerDigest
? {
sessionKey: selectedSession.key,
...(selectedSession.observerDigest.agentId
? { agentId: selectedSession.observerDigest.agentId }
: {}),
runId: selectedSession.observerDigest.runId,
revision: selectedSession.observerDigest.revision,
updatedAt: selectedSession.observerDigest.updatedAt,
headline: selectedSession.observerDigest.headline,
health: selectedSession.observerDigest.health,
}
: null;
const projectedObserverDigest = projectSessionObserverDigest(
selectedSession?.key ?? state.sessionKey,
selectedSession?.observerDigest,
);
const observerDigest = pickFreshestObserverDigest(
state.observerDigest,
projectedObserverDigest,
@@ -559,6 +551,7 @@ export class ChatPaneRender extends ChatPaneHeaderRender {
allowExternalEmbedUrls: state.allowExternalEmbedUrls,
chatMessageMaxWidth: state.settings.chatMessageMaxWidth,
assistantAttachmentAuthToken: resolveAssistantAttachmentAuthToken(state as never),
resolveArtifactDownload: (params) => resolveChatArtifactDownload(state, params),
basePath: state.basePath,
gatewayUrl: state.settings.gatewayUrl,
};
+43 -1
View File
@@ -2,7 +2,11 @@ import { describe, expect, it } from "vitest";
import type { SessionsListResult } from "../../api/types.ts";
import { reconcileSessionHistory } from "../../lib/sessions/reconcile.ts";
import { areUiSessionKeysEquivalent } from "../../lib/sessions/session-key.ts";
import { applySelectedSessionProjection, SessionParticipationTracker } from "./chat-pane-state.ts";
import {
applySelectedSessionProjection,
resolveChatArtifactDownload,
SessionParticipationTracker,
} from "./chat-pane-state.ts";
function projectionState(): Parameters<typeof applySelectedSessionProjection>[0] {
return {
@@ -80,6 +84,44 @@ describe("applySelectedSessionProjection", () => {
});
});
describe("resolveChatArtifactDownload", () => {
it("returns a trimmed ticket without exposing a gateway bearer credential", async () => {
const requests: Array<{ method: string; params: unknown }> = [];
const result = await resolveChatArtifactDownload(
{
connected: true,
client: {
request: async (method: string, params: unknown) => {
requests.push({ method, params });
return {
artifact: {
id: "artifact-1",
type: "image",
title: "image",
download: { mode: "url" },
},
url: " /api/chat/media/outgoing/main/image/full?mediaTicket=ticket ",
expiresAt: " 2026-07-28T00:00:00.000Z ",
};
},
} as never,
},
{ sessionKey: "agent:main:main", artifactId: "artifact-1" },
);
expect(requests).toEqual([
{
method: "artifacts.download",
params: { sessionKey: "agent:main:main", artifactId: "artifact-1" },
},
]);
expect(result).toEqual({
url: "/api/chat/media/outgoing/main/image/full?mediaTicket=ticket",
expiresAt: "2026-07-28T00:00:00.000Z",
});
});
});
describe("SessionParticipationTracker", () => {
const resolve = (
tracker: SessionParticipationTracker,
+21 -1
View File
@@ -1,4 +1,5 @@
import type { GatewaySessionRow } from "../../api/types.ts";
import type { GatewayBrowserClient } from "../../api/gateway.ts";
import type { ArtifactDownloadResult, GatewaySessionRow } from "../../api/types.ts";
import { resolveControlUiAuthToken } from "../../app/control-ui-auth.ts";
type SelectedSessionProjectionState = {
@@ -83,6 +84,25 @@ export function resolveAssistantAttachmentAuthToken(state: {
return resolveControlUiAuthToken(state);
}
export async function resolveChatArtifactDownload(
state: { connected: boolean; client?: GatewayBrowserClient | null },
params: { sessionKey: string; artifactId: string },
): Promise<{ url: string; expiresAt?: string } | null> {
if (!state.connected || !state.client) {
return null;
}
const result = await state.client.request<ArtifactDownloadResult | null>(
"artifacts.download",
params,
);
const url = typeof result?.url === "string" ? result.url.trim() : "";
if (!url) {
return null;
}
const expiresAt = typeof result?.expiresAt === "string" ? result.expiresAt.trim() : undefined;
return { url, ...(expiresAt ? { expiresAt } : {}) };
}
export function dismissChatError(state: {
chatError?: string | null;
lastError: string | null;
+3
View File
@@ -43,6 +43,7 @@ import {
import type { ChatComposerDisabledBanner } from "./components/chat-composer-types.ts";
import { isChatRunWorking, renderChatComposer } from "./components/chat-composer.ts";
import { inlineChatImageFromEvent, openInlineChatImage } from "./components/chat-image-lightbox.ts";
import type { ArtifactDownloadResolver } from "./components/chat-message-media.ts";
import { renderChatPullRequests } from "./components/chat-pull-requests.ts";
import type { SessionRailMode } from "./components/chat-session-rail.ts";
import { renderChatSessionSuggestions } from "./components/chat-session-suggestions.ts";
@@ -176,6 +177,7 @@ export type ChatProps = {
userAvatar?: string | null;
localMediaPreviewRoots?: string[];
assistantAttachmentAuthToken?: string | null;
resolveArtifactDownload?: ArtifactDownloadResolver;
autoExpandToolCalls?: boolean;
attachments?: ChatAttachment[];
getAttachments?: () => ChatAttachment[];
@@ -331,6 +333,7 @@ export function renderChat(props: ChatProps) {
fullMessageAgentId: props.fullMessageAgentId,
localMediaPreviewRoots: props.localMediaPreviewRoots,
assistantAttachmentAuthToken: props.assistantAttachmentAuthToken,
resolveArtifactDownload: props.resolveArtifactDownload,
canvasPluginSurfaceUrl: props.canvasPluginSurfaceUrl,
embedSandboxMode: props.embedSandboxMode,
allowExternalEmbedUrls: props.allowExternalEmbedUrls,
@@ -51,6 +51,7 @@ import {
extractTranscriptAttachments,
schedulePairingQrExpiryRefresh,
type AttachmentItem,
type ArtifactDownloadResolver,
type PairingQrExpiryNotice,
} from "./chat-message-media.ts";
import type { SidebarContent } from "./chat-sidebar.ts";
@@ -176,6 +177,7 @@ export function renderGroupedMessage(
basePath?: string;
localMediaPreviewRoots?: readonly string[];
assistantAttachmentAuthToken?: string | null;
resolveArtifactDownload?: ArtifactDownloadResolver;
onAssistantAttachmentLoaded?: () => void;
onRequestOpenImage?: () => number;
onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void;
@@ -209,6 +211,7 @@ export function renderGroupedMessage(
onRequestUpdate: opts.onRequestUpdate,
onRequestOpenImage: opts.onRequestOpenImage,
onOpenImage: opts.onOpenImage,
resolveArtifactDownload: opts.resolveArtifactDownload,
};
schedulePairingQrExpiryRefresh(messageKey, message, opts.onRequestUpdate);
const images = resolveRenderableMessageImages(extractImages(message), imageRenderOptions);
@@ -24,6 +24,7 @@ import {
resolveMessageActionDetails,
type MessageReplyTarget,
} from "./chat-message-markdown.ts";
import type { ArtifactDownloadResolver } from "./chat-message-media.ts";
import {
renderStreamGroupParts,
type StreamGroupOptions,
@@ -76,6 +77,7 @@ type RenderMessageGroupOptions = {
basePath?: string;
localMediaPreviewRoots?: readonly string[];
assistantAttachmentAuthToken?: string | null;
resolveArtifactDownload?: ArtifactDownloadResolver;
canvasPluginSurfaceUrl?: string | null;
embedSandboxMode?: EmbedSandboxMode;
allowExternalEmbedUrls?: boolean;
@@ -126,6 +128,7 @@ function buildGroupedMessageRenderOptions(
basePath: opts.basePath,
localMediaPreviewRoots: opts.localMediaPreviewRoots,
assistantAttachmentAuthToken: opts.assistantAttachmentAuthToken,
resolveArtifactDownload: opts.resolveArtifactDownload,
embedSandboxMode: opts.embedSandboxMode,
allowExternalEmbedUrls: opts.allowExternalEmbedUrls,
};
@@ -67,10 +67,11 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
const requestVersion = opts?.onRequestOpenImage?.();
const managedSource = isManagedOutgoingImageSource(img.displayUrl);
const cacheKey = managedSource
? resolveManagedOutgoingImageBlobUrlCacheKey(img.displayUrl, opts)
? resolveManagedOutgoingImageBlobUrlCacheKey(img.displayUrl, opts, img.artifactId)
: undefined;
const previewIsCurrent =
!managedSource || readManagedOutgoingImageBlobUrl(img.displayUrl, opts) === previewUrl;
!managedSource ||
readManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId) === previewUrl;
if (previewIsCurrent) {
const release =
opts?.onOpenImage && cacheKey ? retainManagedImageBlobUrl(cacheKey) : undefined;
@@ -82,7 +83,7 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
// Re-resolve before opening so the modal never receives a revoked URL.
if (!opts?.onOpenImage) {
const pendingWindow = reserveExternalWindowForDeferredNavigation();
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts)
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId)
.then((freshUrl) => {
const safeUrl = freshUrl
? resolveSafeExternalUrl(freshUrl, window.location.href, { allowDataImage: true })
@@ -98,7 +99,7 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
.catch(() => pendingWindow?.close());
return;
}
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts)
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId)
.then((freshUrl) => {
if (!freshUrl) {
return;
@@ -133,12 +134,14 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
if (!isManagedOutgoingImageSource(img.displayUrl)) {
return renderImageElement(img, img.displayUrl);
}
const preview = resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts).then((previewUrl) => {
if (!previewUrl) {
return nothing;
}
return renderImageElement(img, previewUrl);
});
const preview = resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId).then(
(previewUrl) => {
if (!previewUrl) {
return nothing;
}
return renderImageElement(img, previewUrl);
},
);
return until(preview, nothing);
};
@@ -175,24 +178,29 @@ function resolveManagedOutgoingImageRequesterSessionKey(source: string): string
function resolveManagedOutgoingImageBlobUrlCacheKey(
source: string,
opts?: ImageRenderOptions,
artifactId?: string,
): string {
const authToken = opts?.authToken?.trim() ?? "";
return `${source}::${authToken}`;
return `${source}::${authToken}::${artifactId?.trim() ?? ""}`;
}
function readManagedOutgoingImageBlobUrl(
source: string,
opts?: ImageRenderOptions,
artifactId?: string,
): string | undefined {
return readManagedImageBlobUrl(resolveManagedOutgoingImageBlobUrlCacheKey(source, opts));
return readManagedImageBlobUrl(
resolveManagedOutgoingImageBlobUrlCacheKey(source, opts, artifactId),
);
}
async function resolveManagedOutgoingImageBlobUrl(
source: string,
opts?: ImageRenderOptions,
artifactId?: string,
): Promise<string | null> {
const authToken = opts?.authToken?.trim() ?? "";
const cacheKey = resolveManagedOutgoingImageBlobUrlCacheKey(source, opts);
const cacheKey = resolveManagedOutgoingImageBlobUrlCacheKey(source, opts, artifactId);
const cached = readManagedImageBlobUrl(cacheKey);
if (cached) {
return cached;
@@ -204,11 +212,18 @@ async function resolveManagedOutgoingImageBlobUrl(
if (!pending) {
pending = (async () => {
const requesterSessionKey = resolveManagedOutgoingImageRequesterSessionKey(source);
const artifactDownload =
requesterSessionKey && artifactId && opts?.resolveArtifactDownload
? await opts
.resolveArtifactDownload({ sessionKey: requesterSessionKey, artifactId })
.catch(() => null)
: null;
const requestUrl = artifactDownload?.url ?? source;
const headers = new Headers({ Accept: "image/*" });
if (authToken) {
if (!artifactDownload && authToken) {
headers.set("Authorization", `Bearer ${authToken}`);
}
if (requesterSessionKey) {
if (!artifactDownload && requesterSessionKey) {
headers.set("x-openclaw-requester-session-key", requesterSessionKey);
}
const controller = new AbortController();
@@ -220,7 +235,7 @@ async function resolveManagedOutgoingImageBlobUrl(
try {
// Managed media is a Gateway API at the origin root. Rebasing it under
// the Control UI mount path serves the HTML shell instead of image bytes.
const res = await fetch(source, {
const res = await fetch(requestUrl, {
method: "GET",
headers,
credentials: "same-origin",
@@ -17,12 +17,18 @@ const pairingQrExpiryRefreshTimers = new Map<string, PairingQrExpiryRefreshTimer
export type ImageBlock = {
url: string;
artifactId?: string;
openUrl?: string;
alt?: string;
width?: number;
height?: number;
};
export type ArtifactDownloadResolver = (params: {
sessionKey: string;
artifactId: string;
}) => Promise<{ url: string; expiresAt?: string } | null>;
export type ImageRenderOptions = {
localMediaPreviewRoots?: readonly string[];
basePath?: string;
@@ -30,6 +36,7 @@ export type ImageRenderOptions = {
onRequestUpdate?: () => void;
onRequestOpenImage?: () => number;
onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void;
resolveArtifactDownload?: ArtifactDownloadResolver;
};
export type RenderableImageBlock = ImageBlock & {
@@ -209,6 +216,7 @@ export function extractImages(message: unknown): ImageBlock[] {
// Handle source object format from optimistic user sends.
const source = b.source as Record<string, unknown> | undefined;
const imageMeta = {
artifactId: typeof b.artifactId === "string" ? b.artifactId : undefined,
alt: typeof b.alt === "string" ? b.alt : undefined,
openUrl: typeof b.openUrl === "string" ? b.openUrl : undefined,
width: typeof b.width === "number" ? b.width : undefined,
@@ -3638,6 +3638,52 @@ describe("grouped chat rendering", () => {
activeItem?.release?.();
});
it("prefers an artifact ticket without forwarding the gateway bearer", async () => {
const artifactId = `artifact_managed_image_${crypto.randomUUID()}`;
const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
const ticketedUrl = `${managedChatImageUrl}?mediaTicket=ticket`;
const resolveArtifactDownload = vi.fn(async () => ({
url: ticketedUrl,
expiresAt: "2026-07-28T05:00:00.000Z",
}));
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
expect(url).toBe(ticketedUrl);
const headers = new Headers(init?.headers);
expect(headers.get("Authorization")).toBeNull();
expect(headers.get("x-openclaw-requester-session-key")).toBeNull();
return { ok: true, blob: async () => new Blob(["png"], { type: "image/png" }) };
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
const container = document.createElement("div");
renderAssistantMessage(
container,
{
role: "assistant",
content: [
{
type: "image",
artifactId,
url: managedChatImageUrl,
alt: "Ticketed image",
},
],
timestamp: Date.now(),
},
{
showToolCalls: false,
assistantAttachmentAuthToken: "must-not-be-forwarded",
resolveArtifactDownload,
},
);
await vi.waitFor(() => expect(container.querySelector(".chat-message-image")).not.toBeNull());
expect(resolveArtifactDownload).toHaveBeenCalledWith({
sessionKey: "agent:main:main",
artifactId,
});
});
it("aborts a stalled managed outgoing image fetch after the deadline", async () => {
vi.useFakeTimers();
const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
@@ -75,6 +75,7 @@ import { getToolTitlesVersion } from "../tool-titles.ts";
import { renderBackgroundTasksStatusRow } from "./chat-background-tasks-status.ts";
import type { BackgroundTasksProps } from "./chat-background-tasks.ts";
import { renderChatDivider, renderChatNotice } from "./chat-divider.ts";
import type { ArtifactDownloadResolver } from "./chat-message-media.ts";
import {
dismissConfirmedActionPopovers,
getAssistantAttachmentAvailabilityRenderVersion,
@@ -149,6 +150,7 @@ type ChatThreadProps = {
fullMessageAgentId?: string;
localMediaPreviewRoots?: string[];
assistantAttachmentAuthToken?: string | null;
resolveArtifactDownload?: ArtifactDownloadResolver;
canvasPluginSurfaceUrl?: string | null;
embedSandboxMode?: EmbedSandboxMode;
allowExternalEmbedUrls?: boolean;
@@ -1545,6 +1547,7 @@ function renderChatThreadContents(
basePath: props.basePath,
localMediaPreviewRoots: props.localMediaPreviewRoots ?? [],
assistantAttachmentAuthToken: props.assistantAttachmentAuthToken ?? null,
resolveArtifactDownload: props.resolveArtifactDownload,
canvasPluginSurfaceUrl: props.canvasPluginSurfaceUrl,
embedSandboxMode: props.embedSandboxMode ?? "scripts",
allowExternalEmbedUrls: props.allowExternalEmbedUrls ?? false,