mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix: generated images do not appear in native chat (#115042)
* fix(chat): render managed images in native clients Co-authored-by: Francesco Giannicola <francescogiannicola1@gmail.com> * chore: keep release changelog owned * refactor(macos): split managed image loading * test(apple): prove managed image header policy * test(native): stabilize managed image proof * test(apple): satisfy strict concurrency checks --------- Co-authored-by: Francesco Giannicola <francescogiannicola1@gmail.com>
This commit is contained in:
committed by
GitHub
parent
c141496217
commit
4b05d83035
+369
-265
File diff suppressed because it is too large
Load Diff
@@ -1305,6 +1305,8 @@ class MainViewModel private constructor(
|
||||
failedResource: ChatWidgetResource?,
|
||||
) = ensureRuntime().resolveInlineWidgetResource(path, failedResource)
|
||||
|
||||
internal suspend fun loadChatImageArtifact(artifactId: String) = ensureRuntime().loadChatImageArtifact(artifactId)
|
||||
|
||||
fun requestCanvasRehydrate(source: String = "screen_tab") {
|
||||
ensureRuntime().requestCanvasRehydrate(source = source, force = true)
|
||||
}
|
||||
|
||||
@@ -4910,6 +4910,8 @@ class NodeRuntime private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
internal suspend fun loadChatImageArtifact(artifactId: String) = chat.loadImageArtifact(artifactId)
|
||||
|
||||
fun loadChat(
|
||||
sessionKey: String,
|
||||
ownerAgentId: String? = null,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package ai.openclaw.app.chat
|
||||
|
||||
import ai.openclaw.app.GatewayModelSummary
|
||||
import ai.openclaw.app.gateway.GatewayLoadedImage
|
||||
import ai.openclaw.app.gateway.GatewayRequestDefinitiveFailure
|
||||
import ai.openclaw.app.gateway.GatewayRequestNotEnqueued
|
||||
import ai.openclaw.app.gateway.GatewayRequestOutcomeUnknown
|
||||
@@ -56,6 +57,8 @@ internal const val SESSION_LIST_FETCH_LIMIT = 200
|
||||
private val QUESTION_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
|
||||
private val SWARM_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
|
||||
private const val SESSION_EDITOR_MAX_BASE64_CHARS = ((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 2) / 3) * 4
|
||||
private val MANAGED_IMAGE_PATH_REGEX =
|
||||
Regex("^/api/chat/media/outgoing/[^/]+/([0-9a-fA-F-]{36})/full(?:\\?.*)?$")
|
||||
|
||||
internal fun chatOutboxQueueFailureText(): NativeText = ChatController.queueFailureText()
|
||||
|
||||
@@ -111,6 +114,12 @@ class ChatController internal constructor(
|
||||
private val cacheScope: () -> ChatCacheScope? = { null },
|
||||
private val currentDefaultAgentId: () -> String? = { "main" },
|
||||
private val currentDefaultAgentRevision: () -> Long = { 0L },
|
||||
private val loadGatewayImageArtifact: suspend (
|
||||
gatewayId: String?,
|
||||
sessionKey: String,
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
) -> GatewayLoadedImage? = { _, _, _, _ -> null },
|
||||
private val commandOutbox: ChatCommandOutbox? = null,
|
||||
private val recordModelRecent: (String) -> Unit = {},
|
||||
private val onSessionDeleted: (ChatSessionDeletion) -> Unit = {},
|
||||
@@ -142,12 +151,26 @@ class ChatController internal constructor(
|
||||
cacheScope = cacheScope,
|
||||
currentDefaultAgentId = currentDefaultAgentId,
|
||||
currentDefaultAgentRevision = currentDefaultAgentRevision,
|
||||
loadGatewayImageArtifact = { gatewayId, sessionKey, agentId, artifactId ->
|
||||
session.loadImageArtifact(gatewayId, sessionKey, agentId, artifactId)
|
||||
},
|
||||
commandOutbox = commandOutbox,
|
||||
recordModelRecent = recordModelRecent,
|
||||
onSessionDeleted = onSessionDeleted,
|
||||
onOfflineDefaultAgentRestored = onOfflineDefaultAgentRestored,
|
||||
)
|
||||
|
||||
suspend fun loadImageArtifact(artifactId: String): GatewayLoadedImage? {
|
||||
val normalizedArtifactId = artifactId.trim().takeIf(String::isNotEmpty) ?: return null
|
||||
val sessionKey = normalizeRequestedSessionKey(_sessionKey.value)
|
||||
return loadGatewayImageArtifact(
|
||||
currentCacheScope()?.gatewayId,
|
||||
sessionKey,
|
||||
resolveAgentIdForSessionKey(sessionKey),
|
||||
normalizedArtifactId,
|
||||
)
|
||||
}
|
||||
|
||||
private var appliedMainSessionKey = "main"
|
||||
private val cacheMutationMutex = Mutex()
|
||||
private val defaultAgentPersistenceMutex = Mutex()
|
||||
@@ -6368,10 +6391,18 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
"image", "audio" -> {
|
||||
val type = obj["type"].asStringOrNull() ?: "image"
|
||||
val inlineContent = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() }
|
||||
val url = obj["url"].asStringOrNull()
|
||||
ChatMessageContent(
|
||||
type = type,
|
||||
mimeType = obj["mimeType"].asStringOrNull(),
|
||||
fileName = obj["fileName"].asStringOrNull(),
|
||||
artifactId = obj["artifactId"].asStringOrNull() ?: managedImageArtifactId(url),
|
||||
url = url,
|
||||
openUrl = obj["openUrl"].asStringOrNull(),
|
||||
alt = obj["alt"].asStringOrNull(),
|
||||
width = obj["width"].asLongOrNull()?.toInt(),
|
||||
height = obj["height"].asLongOrNull()?.toInt(),
|
||||
sizeBytes = obj["sizeBytes"].asLongOrNull(),
|
||||
base64 = inlineContent?.takeIf { type != "image" || it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
|
||||
)
|
||||
}
|
||||
@@ -6415,6 +6446,16 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
}
|
||||
}
|
||||
|
||||
internal fun managedImageArtifactId(rawUrl: String?): String? {
|
||||
val match =
|
||||
rawUrl
|
||||
?.trim()
|
||||
?.let(MANAGED_IMAGE_PATH_REGEX::matchEntire)
|
||||
?: return null
|
||||
val attachmentId = runCatching { UUID.fromString(match.groupValues[1]).toString() }.getOrNull() ?: return null
|
||||
return "artifact_managed_image_$attachmentId"
|
||||
}
|
||||
|
||||
internal fun parseChatMessageContents(obj: JsonObject): List<ChatMessageContent> {
|
||||
val content =
|
||||
obj["content"].asArrayOrNull()?.mapNotNull(::parseChatMessageContent)
|
||||
@@ -6655,6 +6696,9 @@ private fun messageContentIdentityKey(message: ChatMessage): String? {
|
||||
?.lowercase()
|
||||
.orEmpty(),
|
||||
part.fileName?.trim().orEmpty(),
|
||||
part.artifactId?.trim().orEmpty(),
|
||||
part.url?.trim().orEmpty(),
|
||||
part.openUrl?.trim().orEmpty(),
|
||||
part.base64
|
||||
?.hashCode()
|
||||
?.toString()
|
||||
|
||||
@@ -63,13 +63,20 @@ data class ChatTranscriptAnchorState(
|
||||
)
|
||||
|
||||
/**
|
||||
* One content part in a chat message; binary parts carry base64 plus their MIME metadata.
|
||||
* One content part in a chat message; images carry either bounded base64 or a managed artifact reference.
|
||||
*/
|
||||
data class ChatMessageContent(
|
||||
val type: String = "text",
|
||||
val text: String? = null,
|
||||
val mimeType: String? = null,
|
||||
val fileName: String? = null,
|
||||
val artifactId: String? = null,
|
||||
val url: String? = null,
|
||||
val openUrl: String? = null,
|
||||
val alt: String? = null,
|
||||
val width: Int? = null,
|
||||
val height: Int? = null,
|
||||
val sizeBytes: Long? = null,
|
||||
val base64: String? = null,
|
||||
val durationMs: Long? = null,
|
||||
val widget: ChatWidgetPreview? = null,
|
||||
|
||||
@@ -6,6 +6,7 @@ import androidx.room.Insert
|
||||
import androidx.room.OnConflictStrategy
|
||||
import androidx.room.Query
|
||||
import androidx.room.withTransaction
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.builtins.ListSerializer
|
||||
import kotlinx.serialization.builtins.serializer
|
||||
import kotlinx.serialization.json.Json
|
||||
@@ -17,6 +18,21 @@ internal const val MAX_CACHED_SESSIONS = 50
|
||||
/** Upper bound of cached transcript rows per session; only the newest messages are kept. */
|
||||
internal const val MAX_CACHED_MESSAGES_PER_SESSION = 200
|
||||
|
||||
@Serializable
|
||||
private data class CachedMessageContent(
|
||||
val type: String,
|
||||
val text: String? = null,
|
||||
val mimeType: String? = null,
|
||||
val fileName: String? = null,
|
||||
val artifactId: String? = null,
|
||||
val url: String? = null,
|
||||
val openUrl: String? = null,
|
||||
val alt: String? = null,
|
||||
val width: Int? = null,
|
||||
val height: Int? = null,
|
||||
val sizeBytes: Long? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
* Read-only offline cache of chat sessions and transcripts.
|
||||
*
|
||||
@@ -92,7 +108,7 @@ internal data class CachedMessageEntity(
|
||||
val sessionKey: String,
|
||||
val rowOrder: Int,
|
||||
val role: String,
|
||||
// JSON array of text part strings; attachments/binary parts are never persisted.
|
||||
// JSON array of text and managed-image references; attachment bytes are never persisted.
|
||||
val textPartsJson: String,
|
||||
val timestampMs: Long?,
|
||||
// Kept so live history reconciliation can match cached rows by identity key.
|
||||
@@ -232,7 +248,8 @@ class RoomChatTranscriptCache internal constructor(
|
||||
private val database: GatewayCacheDatabase,
|
||||
) : ChatTranscriptCache {
|
||||
private val json = Json
|
||||
private val textPartsSerializer = ListSerializer(String.serializer())
|
||||
private val cachedContentSerializer = ListSerializer(CachedMessageContent.serializer())
|
||||
private val legacyTextPartsSerializer = ListSerializer(String.serializer())
|
||||
|
||||
override suspend fun loadLastDefaultAgentId(gatewayId: String): String? {
|
||||
val gateway = scopedGatewayId(gatewayId) ?: return null
|
||||
@@ -287,7 +304,22 @@ class RoomChatTranscriptCache internal constructor(
|
||||
ChatMessage(
|
||||
id = UUID.randomUUID().toString(),
|
||||
role = role,
|
||||
content = decodeTextParts(row.textPartsJson).map { ChatMessageContent(type = "text", text = it) },
|
||||
content =
|
||||
decodeCachedContent(row.textPartsJson).map { part ->
|
||||
ChatMessageContent(
|
||||
type = part.type,
|
||||
text = part.text,
|
||||
mimeType = part.mimeType,
|
||||
fileName = part.fileName,
|
||||
artifactId = part.artifactId,
|
||||
url = part.url,
|
||||
openUrl = part.openUrl,
|
||||
alt = part.alt,
|
||||
width = part.width,
|
||||
height = part.height,
|
||||
sizeBytes = part.sizeBytes,
|
||||
)
|
||||
},
|
||||
timestampMs = row.timestampMs,
|
||||
idempotencyKey = row.idempotencyKey,
|
||||
// Canonical tree ids stay live-only; cached rows regain actions after history refresh.
|
||||
@@ -367,23 +399,44 @@ class RoomChatTranscriptCache internal constructor(
|
||||
val gateway = scopedGatewayId(gatewayId) ?: return
|
||||
val agent = scopedAgentId(agentId) ?: return
|
||||
val key = sessionKey.trim().takeIf { it.isNotEmpty() } ?: return
|
||||
// Text rows only: attachment/binary parts are dropped, and messages without any text are skipped.
|
||||
// Persist small managed-image references, never attachment bytes. This keeps generated images
|
||||
// visible offline without turning the disposable transcript cache into a binary store.
|
||||
val rows =
|
||||
messages
|
||||
.mapNotNull { message ->
|
||||
val role = normalizeVisibleChatMessageRole(message.role) ?: return@mapNotNull null
|
||||
val textParts = message.content.filter { it.type == "text" }.mapNotNull { it.text }
|
||||
if (textParts.isEmpty()) return@mapNotNull null
|
||||
Triple(message, role, textParts)
|
||||
val content =
|
||||
message.content.mapNotNull { part ->
|
||||
when {
|
||||
part.type == "text" && !part.text.isNullOrBlank() ->
|
||||
CachedMessageContent(type = "text", text = part.text)
|
||||
part.type == "image" && !part.artifactId.isNullOrBlank() && !part.url.isNullOrBlank() ->
|
||||
CachedMessageContent(
|
||||
type = "image",
|
||||
mimeType = part.mimeType,
|
||||
fileName = part.fileName,
|
||||
artifactId = part.artifactId,
|
||||
url = part.url,
|
||||
openUrl = part.openUrl,
|
||||
alt = part.alt,
|
||||
width = part.width,
|
||||
height = part.height,
|
||||
sizeBytes = part.sizeBytes,
|
||||
)
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
if (content.isEmpty()) return@mapNotNull null
|
||||
Triple(message, role, content)
|
||||
}.takeLast(MAX_CACHED_MESSAGES_PER_SESSION)
|
||||
.mapIndexed { index, (message, role, textParts) ->
|
||||
.mapIndexed { index, (message, role, content) ->
|
||||
CachedMessageEntity(
|
||||
gatewayId = gateway,
|
||||
agentId = agent,
|
||||
sessionKey = key,
|
||||
rowOrder = index,
|
||||
role = role,
|
||||
textPartsJson = json.encodeToString(textPartsSerializer, textParts),
|
||||
textPartsJson = json.encodeToString(cachedContentSerializer, content),
|
||||
timestampMs = message.timestampMs,
|
||||
idempotencyKey = message.idempotencyKey,
|
||||
)
|
||||
@@ -452,5 +505,12 @@ class RoomChatTranscriptCache internal constructor(
|
||||
|
||||
private fun scopedAgentId(agentId: String): String? = agentId.trim().takeIf { it.isNotEmpty() }
|
||||
|
||||
private fun decodeTextParts(encoded: String): List<String> = runCatching { json.decodeFromString(textPartsSerializer, encoded) }.getOrDefault(emptyList())
|
||||
private fun decodeCachedContent(encoded: String): List<CachedMessageContent> =
|
||||
runCatching { json.decodeFromString(cachedContentSerializer, encoded) }.getOrElse {
|
||||
// Offline transcript browsing is shipped behavior. Keep the previous string-array rows
|
||||
// readable until a live history refresh naturally rewrites this disposable cache entry.
|
||||
runCatching { json.decodeFromString(legacyTextPartsSerializer, encoded) }
|
||||
.getOrDefault(emptyList())
|
||||
.map { CachedMessageContent(type = "text", text = it) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,8 @@ import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import okhttp3.WebSocket
|
||||
import okhttp3.WebSocketListener
|
||||
import okio.Buffer
|
||||
import java.net.URI
|
||||
import java.util.Locale
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -53,6 +55,11 @@ data class GatewayClientInfo(
|
||||
val modelIdentifier: String?,
|
||||
)
|
||||
|
||||
data class GatewayLoadedImage(
|
||||
val bytes: ByteArray,
|
||||
val mimeType: String,
|
||||
)
|
||||
|
||||
/**
|
||||
* Role, scopes, commands, and permission snapshot sent with the connect frame.
|
||||
*/
|
||||
@@ -616,6 +623,38 @@ class GatewaySession(
|
||||
throw GatewayRequestRejected(res.error ?: ErrorShape("UNAVAILABLE", "request failed"))
|
||||
}
|
||||
|
||||
suspend fun loadImageArtifact(
|
||||
expectedEndpointStableId: String?,
|
||||
sessionKey: String,
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
): GatewayLoadedImage? {
|
||||
val conn = readyConnection(expectedEndpointStableId) ?: return null
|
||||
val params =
|
||||
buildJsonObject {
|
||||
put("sessionKey", JsonPrimitive(sessionKey))
|
||||
agentId?.trim()?.takeIf(String::isNotEmpty)?.let { put("agentId", JsonPrimitive(it)) }
|
||||
put("artifactId", JsonPrimitive(artifactId))
|
||||
}
|
||||
val response = conn.request(GatewayMethod.ArtifactsDownload.rawValue, params, timeoutMs = 15_000)
|
||||
if (!response.ok) {
|
||||
throw GatewayRequestRejected(response.error ?: ErrorShape("UNAVAILABLE", "artifact download failed"))
|
||||
}
|
||||
val ticketedPath =
|
||||
response.payloadJson
|
||||
?.let(::parseJsonOrNull)
|
||||
.asObjectOrNull()
|
||||
?.get("url")
|
||||
.asStringOrNull()
|
||||
?.trim()
|
||||
?.takeIf(String::isNotEmpty)
|
||||
?: return null
|
||||
val loaded = conn.loadTicketedImage(ticketedPath) ?: return null
|
||||
return synchronized(lifecycleLock) {
|
||||
loaded.takeIf { currentConnection === conn && conn.isReady() }
|
||||
}
|
||||
}
|
||||
|
||||
internal suspend fun requestForEndpoint(
|
||||
expectedEndpointStableId: String,
|
||||
method: String,
|
||||
@@ -830,6 +869,46 @@ class GatewaySession(
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun loadTicketedImage(ticketedPath: String): GatewayLoadedImage? =
|
||||
withContext(Dispatchers.IO) {
|
||||
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return@withContext null
|
||||
val rawPath = uri.rawPath ?: return@withContext null
|
||||
val rawQuery = uri.rawQuery ?: return@withContext null
|
||||
if (uri.isAbsolute || uri.rawAuthority != null || uri.rawFragment != null) return@withContext null
|
||||
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !rawQuery.contains("mediaTicket=")) {
|
||||
return@withContext null
|
||||
}
|
||||
val scheme = if (tlsConfig != null) "https" else "http"
|
||||
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
|
||||
val request = Request.Builder().url(url).header("Accept", "image/*")
|
||||
if (tlsConfig != null) {
|
||||
for ((name, value) in GatewayCustomHeaders.sanitized(customHeadersProvider?.invoke(endpoint.stableId).orEmpty())) {
|
||||
request.header(name, value)
|
||||
}
|
||||
}
|
||||
val call = client.newCall(request.build())
|
||||
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
|
||||
call.execute().use { response ->
|
||||
if (!response.isSuccessful) return@withContext null
|
||||
val body = response.body
|
||||
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
|
||||
if (!mimeType.startsWith("image/")) return@withContext null
|
||||
val maximumBytes = 12L * 1024L * 1024L
|
||||
val declaredLength = body.contentLength()
|
||||
if (declaredLength > maximumBytes) return@withContext null
|
||||
val buffer = Buffer()
|
||||
val source = body.source()
|
||||
var total = 0L
|
||||
while (true) {
|
||||
val read = source.read(buffer, minOf(8192L, maximumBytes + 1L - total))
|
||||
if (read == -1L) break
|
||||
total += read
|
||||
if (total > maximumBytes) return@withContext null
|
||||
}
|
||||
GatewayLoadedImage(bytes = buffer.readByteArray(), mimeType = mimeType)
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(DelicateCoroutinesApi::class)
|
||||
suspend fun sendRequestFrame(
|
||||
method: String,
|
||||
|
||||
@@ -171,11 +171,18 @@ internal fun decodeBase64Bitmap(
|
||||
maxDimension: Int = CHAT_DECODE_MAX_DIMENSION,
|
||||
): Bitmap? {
|
||||
if (base64.length > CHAT_IMAGE_MAX_BASE64_CHARS) return null
|
||||
val cacheKey = "$maxDimension:${base64.length}:${base64.hashCode()}"
|
||||
decodedBitmapCache.get(cacheKey)?.let { return it }
|
||||
|
||||
val bytes = Base64.decode(base64, Base64.DEFAULT)
|
||||
if (bytes.isEmpty()) return null
|
||||
return decodeImageBytes(bytes, maxDimension)
|
||||
}
|
||||
|
||||
/** Decodes already-authorized image bytes without base64 expansion. */
|
||||
internal fun decodeImageBytes(
|
||||
bytes: ByteArray,
|
||||
maxDimension: Int = CHAT_DECODE_MAX_DIMENSION,
|
||||
): Bitmap? {
|
||||
if (bytes.isEmpty() || bytes.size > 12 * 1024 * 1024) return null
|
||||
val cacheKey = "$maxDimension:${bytes.size}:${bytes.contentHashCode()}"
|
||||
decodedBitmapCache.get(cacheKey)?.let { return it }
|
||||
|
||||
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
|
||||
BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds)
|
||||
|
||||
@@ -10,6 +10,7 @@ import ai.openclaw.app.chat.MessageSpeechState
|
||||
import ai.openclaw.app.chat.OUTBOX_BRANCH_CHANGED_ERROR
|
||||
import ai.openclaw.app.chat.chatOutboxDisplayError
|
||||
import ai.openclaw.app.chat.normalizeVisibleChatMessageRole
|
||||
import ai.openclaw.app.gateway.GatewayLoadedImage
|
||||
import ai.openclaw.app.i18n.nativeString
|
||||
import ai.openclaw.app.i18n.nativeStringResource
|
||||
import ai.openclaw.app.tools.ToolDisplayRegistry
|
||||
@@ -83,6 +84,8 @@ import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import androidx.compose.ui.window.Dialog
|
||||
import androidx.compose.ui.window.DialogProperties
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.util.Locale
|
||||
|
||||
private data class ChatBubbleStyle(
|
||||
@@ -102,6 +105,8 @@ internal fun ChatMessageBubble(
|
||||
onForkMessage: (String) -> Unit = {},
|
||||
speechState: MessageSpeechState? = null,
|
||||
onToggleListen: ((String, String) -> Unit)? = null,
|
||||
imageResolverReady: Boolean = false,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage? = { null },
|
||||
) {
|
||||
val role = normalizeVisibleChatMessageRole(message.role) ?: return
|
||||
val style = bubbleStyle(role)
|
||||
@@ -111,7 +116,7 @@ internal fun ChatMessageBubble(
|
||||
message.content.filter { part ->
|
||||
when (part.type) {
|
||||
"text" -> !part.text.isNullOrBlank()
|
||||
"image" -> !part.base64.isNullOrBlank()
|
||||
"image" -> !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank()
|
||||
else -> part.isAudioAttachment()
|
||||
}
|
||||
}
|
||||
@@ -138,7 +143,12 @@ internal fun ChatMessageBubble(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
ChatBubbleContainer(style = style, roleLabel = roleLabel(role)) {
|
||||
ChatMessageBody(content = displayableContent, textColor = mobileText)
|
||||
ChatMessageBody(
|
||||
content = displayableContent,
|
||||
textColor = mobileText,
|
||||
imageResolverReady = imageResolverReady,
|
||||
loadImageArtifact = loadImageArtifact,
|
||||
)
|
||||
ChatMessageLinkPreview(messageId = message.id, role = role, content = displayableContent)
|
||||
messageSpeech?.let { speech ->
|
||||
MessageSpeechIndicator(
|
||||
@@ -223,6 +233,8 @@ private fun ChatBubbleContainer(
|
||||
private fun ChatMessageBody(
|
||||
content: List<ChatMessageContent>,
|
||||
textColor: Color,
|
||||
imageResolverReady: Boolean,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
for (part in content) {
|
||||
@@ -232,9 +244,17 @@ private fun ChatMessageBody(
|
||||
ChatMarkdown(text = text, textColor = textColor)
|
||||
}
|
||||
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
|
||||
part.type == "image" && !part.base64.isNullOrBlank() ->
|
||||
ChatBase64Image(base64 = part.base64, mimeType = part.mimeType)
|
||||
part.type == "image" && !part.artifactId.isNullOrBlank() ->
|
||||
ChatManagedImage(
|
||||
artifactId = part.artifactId,
|
||||
label = part.alt?.takeIf(String::isNotBlank) ?: part.fileName ?: nativeString("Image"),
|
||||
resolverReady = imageResolverReady,
|
||||
loadImage = loadImageArtifact,
|
||||
)
|
||||
else -> {
|
||||
val b64 = part.base64 ?: continue
|
||||
ChatBase64Image(base64 = b64, mimeType = part.mimeType)
|
||||
Text(part.fileName ?: nativeString("Attachment"), style = mobileCaption1, color = mobileTextSecondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -583,75 +603,138 @@ internal fun ChatBase64Image(
|
||||
mimeType: String?,
|
||||
) {
|
||||
val imageState = rememberBase64ImageState(base64)
|
||||
var previewVisible by rememberSaveable(base64) { mutableStateOf(false) }
|
||||
val image = imageState.image
|
||||
|
||||
if (image != null) {
|
||||
Surface(
|
||||
onClick = { previewVisible = true },
|
||||
shape = RoundedCornerShape(10.dp),
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
color = mobileCardSurface,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
ChatImagePreview(image = image, description = mimeType ?: nativeString("Attachment"), stateKey = base64)
|
||||
} else if (imageState.failed) {
|
||||
Text(nativeString("Unsupported attachment"), style = mobileCaption1, color = mobileTextSecondary)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun ChatManagedImage(
|
||||
artifactId: String,
|
||||
label: String,
|
||||
resolverReady: Boolean,
|
||||
loadImage: suspend (String) -> GatewayLoadedImage?,
|
||||
) {
|
||||
var image by remember(artifactId) { mutableStateOf<ImageBitmap?>(null) }
|
||||
var failed by remember(artifactId) { mutableStateOf(false) }
|
||||
var retryGeneration by rememberSaveable(artifactId) { mutableStateOf(0) }
|
||||
|
||||
LaunchedEffect(artifactId, resolverReady, retryGeneration) {
|
||||
if (!resolverReady) {
|
||||
failed = true
|
||||
image = null
|
||||
return@LaunchedEffect
|
||||
}
|
||||
failed = false
|
||||
image = null
|
||||
val loaded = runCatching { loadImage(artifactId) }.getOrNull()
|
||||
image =
|
||||
loaded?.let { value ->
|
||||
withContext(Dispatchers.Default) { decodeImageBytes(value.bytes)?.asImageBitmap() }
|
||||
}
|
||||
failed = image == null
|
||||
}
|
||||
|
||||
when {
|
||||
image != null -> ChatImagePreview(image = checkNotNull(image), description = label, stateKey = artifactId)
|
||||
failed ->
|
||||
Surface(
|
||||
onClick = { retryGeneration += 1 },
|
||||
shape = RoundedCornerShape(10.dp),
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
color = mobileCardSurface,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
Text(
|
||||
nativeString("Image unavailable · Tap to retry"),
|
||||
modifier = Modifier.padding(12.dp),
|
||||
style = mobileCaption1,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
else ->
|
||||
Text(
|
||||
nativeString("Loading image…"),
|
||||
modifier = Modifier.padding(12.dp),
|
||||
style = mobileCaption1,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ChatImagePreview(
|
||||
image: ImageBitmap,
|
||||
description: String,
|
||||
stateKey: String,
|
||||
) {
|
||||
var previewVisible by rememberSaveable(stateKey) { mutableStateOf(false) }
|
||||
Surface(
|
||||
onClick = { previewVisible = true },
|
||||
shape = RoundedCornerShape(10.dp),
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
color = mobileCardSurface,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
Box {
|
||||
Image(
|
||||
bitmap = image,
|
||||
contentDescription = description,
|
||||
contentScale = ContentScale.Fit,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
)
|
||||
Surface(
|
||||
modifier = Modifier.align(Alignment.BottomEnd).padding(8.dp).size(32.dp),
|
||||
shape = CircleShape,
|
||||
color = Color.Black.copy(alpha = 0.62f),
|
||||
contentColor = Color.White,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.OpenInFull,
|
||||
contentDescription = nativeString("Open image preview"),
|
||||
modifier = Modifier.size(17.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (previewVisible) {
|
||||
Dialog(
|
||||
onDismissRequest = { previewVisible = false },
|
||||
properties = DialogProperties(usePlatformDefaultWidth = false),
|
||||
) {
|
||||
Box {
|
||||
Box(
|
||||
modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.96f)).clickable { previewVisible = false },
|
||||
contentAlignment = Alignment.Center,
|
||||
) {
|
||||
Image(
|
||||
bitmap = image,
|
||||
contentDescription = mimeType ?: nativeString("Attachment"),
|
||||
contentDescription = nativeString("Image preview"),
|
||||
contentScale = ContentScale.Fit,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
modifier = Modifier.fillMaxSize().padding(20.dp),
|
||||
)
|
||||
Surface(
|
||||
modifier = Modifier.align(Alignment.BottomEnd).padding(8.dp).size(32.dp),
|
||||
onClick = { previewVisible = false },
|
||||
modifier = Modifier.align(Alignment.TopEnd).padding(16.dp).size(44.dp),
|
||||
shape = CircleShape,
|
||||
color = Color.Black.copy(alpha = 0.62f),
|
||||
contentColor = Color.White,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.OpenInFull,
|
||||
contentDescription = nativeString("Open image preview"),
|
||||
modifier = Modifier.size(17.dp),
|
||||
imageVector = Icons.Default.Close,
|
||||
contentDescription = nativeString("Close image preview"),
|
||||
modifier = Modifier.size(22.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (previewVisible) {
|
||||
Dialog(
|
||||
onDismissRequest = { previewVisible = false },
|
||||
properties = DialogProperties(usePlatformDefaultWidth = false),
|
||||
) {
|
||||
Box(
|
||||
modifier = Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.96f)).clickable { previewVisible = false },
|
||||
contentAlignment = Alignment.Center,
|
||||
) {
|
||||
Image(
|
||||
bitmap = image,
|
||||
contentDescription = nativeString("Image preview"),
|
||||
contentScale = ContentScale.Fit,
|
||||
modifier = Modifier.fillMaxSize().padding(20.dp),
|
||||
)
|
||||
Surface(
|
||||
onClick = { previewVisible = false },
|
||||
modifier = Modifier.align(Alignment.TopEnd).padding(16.dp).size(44.dp),
|
||||
shape = CircleShape,
|
||||
color = Color.Black.copy(alpha = 0.62f),
|
||||
contentColor = Color.White,
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Close,
|
||||
contentDescription = nativeString("Close image preview"),
|
||||
modifier = Modifier.size(22.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (imageState.failed) {
|
||||
Text(nativeString("Unsupported attachment"), style = mobileCaption1, color = mobileTextSecondary)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ import ai.openclaw.app.chat.questionsForSession
|
||||
import ai.openclaw.app.chat.resolveChatComposerOwner
|
||||
import ai.openclaw.app.chat.resolveGatewayDefaultAgentId
|
||||
import ai.openclaw.app.currentAppLanguage
|
||||
import ai.openclaw.app.gateway.GatewayLoadedImage
|
||||
import ai.openclaw.app.i18n.NativeText
|
||||
import ai.openclaw.app.i18n.joinedNativeText
|
||||
import ai.openclaw.app.i18n.nativeString
|
||||
@@ -739,6 +740,7 @@ fun ChatScreen(
|
||||
speechState = messageSpeechState,
|
||||
onToggleListen = viewModel::toggleChatMessageSpeech,
|
||||
resolveInlineWidgetResource = viewModel::resolveInlineWidgetResource,
|
||||
loadImageArtifact = viewModel::loadChatImageArtifact,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
|
||||
@@ -1255,6 +1257,7 @@ private fun ChatMessageList(
|
||||
speechState: MessageSpeechState?,
|
||||
onToggleListen: (String, String) -> Unit,
|
||||
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
val baseTimeline =
|
||||
@@ -1323,6 +1326,7 @@ private fun ChatMessageList(
|
||||
onToggleListen = onToggleListen,
|
||||
inlineWidgetResolverReady = healthOk,
|
||||
resolveInlineWidgetResource = resolveInlineWidgetResource,
|
||||
loadImageArtifact = loadImageArtifact,
|
||||
)
|
||||
is ChatTimelineItem.OutboxCommand ->
|
||||
ChatOutboxBubble(
|
||||
@@ -1366,6 +1370,7 @@ private fun ChatMessageList(
|
||||
onToggleListen = onToggleListen,
|
||||
inlineWidgetResolverReady = healthOk,
|
||||
resolveInlineWidgetResource = resolveInlineWidgetResource,
|
||||
loadImageArtifact = loadImageArtifact,
|
||||
)
|
||||
ChatTimelineItem.Thinking -> {
|
||||
val run = workingRun
|
||||
@@ -1616,18 +1621,26 @@ private fun ChatBubble(
|
||||
onToggleListen: (String, String) -> Unit,
|
||||
inlineWidgetResolverReady: Boolean,
|
||||
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
) {
|
||||
val normalizedRole = role.trim().lowercase(Locale.US)
|
||||
val isUser = normalizedRole == "user"
|
||||
var visibleImageCount = 0
|
||||
val displayableContent =
|
||||
content.filter { part ->
|
||||
when (part.type) {
|
||||
"text" -> !part.text.isNullOrBlank()
|
||||
"image" -> !part.base64.isNullOrBlank()
|
||||
"image" -> {
|
||||
val displayable = !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank()
|
||||
val visible = displayable && visibleImageCount < 4
|
||||
if (displayable) visibleImageCount += 1
|
||||
visible
|
||||
}
|
||||
"canvas" -> normalizedRole == "assistant" && part.widget != null
|
||||
else -> part.isAudioAttachment()
|
||||
}
|
||||
}
|
||||
val omittedImageCount = (visibleImageCount - 4).coerceAtLeast(0)
|
||||
if (displayableContent.isEmpty()) return
|
||||
|
||||
val messageText = chatMessagePlainText(displayableContent)
|
||||
@@ -1692,10 +1705,19 @@ private fun ChatBubble(
|
||||
part.type == "text" -> Unit
|
||||
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
|
||||
part.type == "image" ->
|
||||
ChatBase64Image(
|
||||
base64 = checkNotNull(part.base64),
|
||||
mimeType = part.mimeType,
|
||||
)
|
||||
if (!part.base64.isNullOrBlank()) {
|
||||
ChatBase64Image(
|
||||
base64 = part.base64,
|
||||
mimeType = part.mimeType,
|
||||
)
|
||||
} else {
|
||||
ChatManagedImage(
|
||||
artifactId = checkNotNull(part.artifactId),
|
||||
label = part.alt?.takeIf(String::isNotBlank) ?: part.fileName ?: nativeString("Image"),
|
||||
resolverReady = inlineWidgetResolverReady,
|
||||
loadImage = loadImageArtifact,
|
||||
)
|
||||
}
|
||||
part.type == "canvas" && normalizedRole == "assistant" ->
|
||||
ChatInlineWidget(
|
||||
preview = checkNotNull(part.widget),
|
||||
@@ -1705,6 +1727,13 @@ private fun ChatBubble(
|
||||
else -> Text(text = part.fileName ?: nativeString("Attachment"), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
|
||||
}
|
||||
}
|
||||
if (omittedImageCount > 0) {
|
||||
Text(
|
||||
text = nativeString("Additional images hidden: \${omittedImageCount}", omittedImageCount),
|
||||
style = ClawTheme.type.caption,
|
||||
color = ClawTheme.colors.textMuted,
|
||||
)
|
||||
}
|
||||
if (messageId != null) {
|
||||
ChatMessageLinkPreview(messageId = messageId, role = normalizedRole, content = displayableContent)
|
||||
}
|
||||
|
||||
+24
@@ -60,6 +60,30 @@ class ChatControllerMessageIdentityTest {
|
||||
assertEquals(listOf(ChatMessageContent(type = "text", text = "Hi there")), content)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun managedImagesParticipateInMessageIdentity() {
|
||||
fun message(artifactId: String) =
|
||||
ChatMessage(
|
||||
id = artifactId,
|
||||
role = "assistant",
|
||||
content =
|
||||
listOf(
|
||||
ChatMessageContent(
|
||||
type = "image",
|
||||
artifactId = artifactId,
|
||||
url = "/api/chat/media/outgoing/main/$artifactId/full",
|
||||
mimeType = "image/png",
|
||||
),
|
||||
),
|
||||
timestampMs = 1,
|
||||
)
|
||||
|
||||
assertNotEquals(
|
||||
messageIdentityKey(message("artifact_managed_image_11111111-1111-4111-8111-111111111111")),
|
||||
messageIdentityKey(message("artifact_managed_image_22222222-2222-4222-8222-222222222222")),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
fun liveHistoryDropsInternalRoleRows() =
|
||||
|
||||
+27
-3
@@ -276,14 +276,14 @@ class ChatMessageContentParsingTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun parsesImageBlocksOnlyWhenInlineContentExists() {
|
||||
fun parsesInlineAndManagedImageBlocks() {
|
||||
val image =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"image","mimeType":"image/png","fileName":"chart.png","content":"abc123"}""",
|
||||
)
|
||||
val managedImage =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"image","mimeType":"image/png","fileName":"chart.png","url":"/api/chat/media/outgoing/main/id"}""",
|
||||
"""{"type":"image","artifactId":"artifact_managed_image_11111111-1111-4111-8111-111111111111","mimeType":"image/png","fileName":"chart.png","url":"/api/chat/media/outgoing/main/id","openUrl":"/api/chat/media/outgoing/main/id","alt":"Chart","width":1200,"height":800,"sizeBytes":2048}""",
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
@@ -291,11 +291,35 @@ class ChatMessageContentParsingTest {
|
||||
parseChatMessageContent(image),
|
||||
)
|
||||
assertEquals(
|
||||
ChatMessageContent(type = "image", mimeType = "image/png", fileName = "chart.png", base64 = null),
|
||||
ChatMessageContent(
|
||||
type = "image",
|
||||
mimeType = "image/png",
|
||||
fileName = "chart.png",
|
||||
artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111",
|
||||
url = "/api/chat/media/outgoing/main/id",
|
||||
openUrl = "/api/chat/media/outgoing/main/id",
|
||||
alt = "Chart",
|
||||
width = 1200,
|
||||
height = 800,
|
||||
sizeBytes = 2048,
|
||||
),
|
||||
parseChatMessageContent(managedImage),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun derivesArtifactIdentityForShippedManagedImageBlocks() {
|
||||
val image =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"image","mimeType":"image/png","url":"/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full"}""",
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
"artifact_managed_image_11111111-1111-4111-8111-111111111111",
|
||||
parseChatMessageContent(image)?.artifactId,
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dropsOversizedInlineImageContentBeforeRendering() {
|
||||
val oversized = "A".repeat(CHAT_IMAGE_MAX_BASE64_CHARS + 1)
|
||||
|
||||
@@ -40,10 +40,18 @@ class RoomChatTranscriptCacheTest {
|
||||
)
|
||||
|
||||
@Test
|
||||
fun transcriptRoundTripKeepsTextRowsOnly() =
|
||||
fun transcriptRoundTripKeepsTextAndManagedReferencesWithoutBinaryParts() =
|
||||
runTest {
|
||||
val store = cache()
|
||||
val imagePart = ChatMessageContent(type = "image", mimeType = "image/png", fileName = "a.png", base64 = "AAAA")
|
||||
val managedImage =
|
||||
ChatMessageContent(
|
||||
type = "image",
|
||||
mimeType = "image/png",
|
||||
artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111",
|
||||
url = "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
|
||||
alt = "Managed image",
|
||||
)
|
||||
store.saveTranscript(
|
||||
gatewayId = "gateway-a",
|
||||
agentId = "main",
|
||||
@@ -51,19 +59,44 @@ class RoomChatTranscriptCacheTest {
|
||||
messages =
|
||||
listOf(
|
||||
message("hello", role = "user", timestampMs = 10, idempotencyKey = "run-1:user", extraParts = listOf(imagePart)),
|
||||
// Attachment-only messages have no cacheable text and are skipped entirely.
|
||||
// Inline binary-only messages remain disposable and are skipped entirely.
|
||||
ChatMessage(id = "img", role = "user", content = listOf(imagePart), timestampMs = 11),
|
||||
ChatMessage(id = "managed", role = "assistant", content = listOf(managedImage), timestampMs = 11),
|
||||
message("world", role = "assistant", timestampMs = 12),
|
||||
),
|
||||
)
|
||||
|
||||
val loaded = store.loadTranscript("gateway-a", "main", "main")
|
||||
|
||||
assertEquals(listOf("hello", "world"), loaded.map { it.content.single().text })
|
||||
assertTrue(loaded.all { message -> message.content.all { part -> part.type == "text" && part.base64 == null } })
|
||||
assertEquals(listOf("user", "assistant"), loaded.map { it.role })
|
||||
assertEquals(listOf(10L, 12L), loaded.map { it.timestampMs })
|
||||
assertEquals(listOf("run-1:user", null), loaded.map { it.idempotencyKey })
|
||||
assertEquals(listOf("hello", null, "world"), loaded.map { it.content.single().text })
|
||||
assertTrue(loaded.all { message -> message.content.all { part -> part.base64 == null } })
|
||||
assertEquals(managedImage.artifactId, loaded[1].content.single().artifactId)
|
||||
assertEquals(listOf("user", "assistant", "assistant"), loaded.map { it.role })
|
||||
assertEquals(listOf(10L, 11L, 12L), loaded.map { it.timestampMs })
|
||||
assertEquals(listOf("run-1:user", null, null), loaded.map { it.idempotencyKey })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun legacyStringArrayTranscriptRowsRemainReadable() =
|
||||
runTest {
|
||||
database.dao().insertMessages(
|
||||
listOf(
|
||||
CachedMessageEntity(
|
||||
gatewayId = "gateway-a",
|
||||
agentId = "main",
|
||||
sessionKey = "main",
|
||||
rowOrder = 0,
|
||||
role = "assistant",
|
||||
textPartsJson = """["legacy one","legacy two"]""",
|
||||
timestampMs = 10,
|
||||
idempotencyKey = null,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
val loaded = cache().loadTranscript("gateway-a", "main", "main").single()
|
||||
|
||||
assertEquals(listOf("legacy one", "legacy two"), loaded.content.map { it.text })
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+111
@@ -19,6 +19,8 @@ import okhttp3.mockwebserver.Dispatcher
|
||||
import okhttp3.mockwebserver.MockResponse
|
||||
import okhttp3.mockwebserver.MockWebServer
|
||||
import okhttp3.mockwebserver.RecordedRequest
|
||||
import okio.Buffer
|
||||
import org.junit.Assert.assertArrayEquals
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Assert.assertTrue
|
||||
@@ -60,6 +62,115 @@ private class NoopDeviceAuthStore : DeviceAuthTokenStore {
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(sdk = [34])
|
||||
class GatewaySessionCustomHeadersTest {
|
||||
@Test
|
||||
fun managedImageDownload_usesArtifactTicketWithoutGatewayBearer() =
|
||||
runBlocking {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
val json = Json { ignoreUnknownKeys = true }
|
||||
val connected = CompletableDeferred<Unit>()
|
||||
val imageRequest = CompletableDeferred<RecordedRequest>()
|
||||
val imageBytes = byteArrayOf(1, 2, 3, 4)
|
||||
val attachmentId = "11111111-1111-4111-8111-111111111111"
|
||||
val artifactId = "artifact_managed_image_$attachmentId"
|
||||
val imagePath = "/api/chat/media/outgoing/main/$attachmentId/full?mediaTicket=ticket"
|
||||
val server =
|
||||
MockWebServer().apply {
|
||||
dispatcher =
|
||||
object : Dispatcher() {
|
||||
override fun dispatch(request: RecordedRequest): MockResponse {
|
||||
if (request.path == imagePath) {
|
||||
imageRequest.complete(request)
|
||||
return MockResponse()
|
||||
.setHeader("Content-Type", "image/png")
|
||||
.setBody(Buffer().write(imageBytes))
|
||||
}
|
||||
return MockResponse().withWebSocketUpgrade(
|
||||
object : WebSocketListener() {
|
||||
override fun onOpen(
|
||||
webSocket: WebSocket,
|
||||
response: Response,
|
||||
) {
|
||||
webSocket.send(CONNECT_CHALLENGE_FRAME)
|
||||
}
|
||||
|
||||
override fun onMessage(
|
||||
webSocket: WebSocket,
|
||||
text: String,
|
||||
) {
|
||||
val frame = json.parseToJsonElement(text).jsonObject
|
||||
if (frame["type"]?.jsonPrimitive?.content != "req") return
|
||||
val id = frame["id"]?.jsonPrimitive?.content ?: return
|
||||
when (frame["method"]?.jsonPrimitive?.content) {
|
||||
"connect" ->
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"snapshot":{"sessionDefaults":{"mainSessionKey":"main"}}}}""",
|
||||
)
|
||||
"artifacts.download" ->
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
start()
|
||||
}
|
||||
val stableId = "manual|127.0.0.1|${server.port}"
|
||||
val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
val session =
|
||||
GatewaySession(
|
||||
scope = scope,
|
||||
identityStore = testDeviceIdentityStore(app),
|
||||
deviceAuthStore = NoopDeviceAuthStore(),
|
||||
onConnected = { if (!connected.isCompleted) connected.complete(Unit) },
|
||||
onDisconnected = {},
|
||||
onEvent = { _, _ -> },
|
||||
)
|
||||
|
||||
try {
|
||||
session.connect(
|
||||
endpoint = GatewayEndpoint(stableId, "test", "127.0.0.1", server.port, tlsEnabled = false),
|
||||
token = "bootstrap-token",
|
||||
bootstrapToken = null,
|
||||
password = null,
|
||||
options =
|
||||
GatewayConnectOptions(
|
||||
role = "operator",
|
||||
scopes = listOf("operator.read"),
|
||||
caps = emptyList(),
|
||||
commands = emptyList(),
|
||||
permissions = emptyMap(),
|
||||
client =
|
||||
GatewayClientInfo(
|
||||
id = "openclaw-android-test",
|
||||
displayName = "Android Test",
|
||||
version = "1.0.0-test",
|
||||
platform = "android",
|
||||
mode = "ui",
|
||||
instanceId = "android-test-instance",
|
||||
deviceFamily = "android",
|
||||
modelIdentifier = "test",
|
||||
),
|
||||
),
|
||||
tls = null,
|
||||
)
|
||||
withTimeout(TEST_TIMEOUT_MS) { connected.await() }
|
||||
|
||||
val loaded = session.loadImageArtifact(stableId, "main", "main", artifactId)
|
||||
assertArrayEquals(imageBytes, loaded?.bytes)
|
||||
assertEquals("image/png", loaded?.mimeType)
|
||||
val request = withTimeout(TEST_TIMEOUT_MS) { imageRequest.await() }
|
||||
assertNull(request.getHeader("Authorization"))
|
||||
assertEquals("image/*", request.getHeader("Accept"))
|
||||
} finally {
|
||||
session.disconnectAndJoin()
|
||||
scope.cancel()
|
||||
server.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun tlsUpgradeRequest_carriesLatestSanitizedHeadersForOnlyThisGateway() {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import ai.openclaw.app.chat.ChatMessage
|
||||
import ai.openclaw.app.chat.ChatMessageContent
|
||||
import android.os.Looper
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.Robolectric
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.Shadows.shadowOf
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
class ChatMessageViewsTest {
|
||||
@Test
|
||||
fun managedImageCompositionRequestsItsArtifact() {
|
||||
val artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111"
|
||||
val requested = mutableListOf<String>()
|
||||
val controller = Robolectric.buildActivity(ComponentActivity::class.java).setup()
|
||||
|
||||
try {
|
||||
controller.get().setContent {
|
||||
ChatMessageBubble(
|
||||
message =
|
||||
ChatMessage(
|
||||
id = "managed-image",
|
||||
role = "assistant",
|
||||
content =
|
||||
listOf(
|
||||
ChatMessageContent(
|
||||
type = "image",
|
||||
mimeType = "image/png",
|
||||
artifactId = artifactId,
|
||||
alt = "Managed image",
|
||||
),
|
||||
),
|
||||
timestampMs = 1,
|
||||
),
|
||||
imageResolverReady = true,
|
||||
loadImageArtifact = { requestedArtifactId ->
|
||||
requested += requestedArtifactId
|
||||
null
|
||||
},
|
||||
)
|
||||
}
|
||||
shadowOf(Looper.getMainLooper()).idle()
|
||||
|
||||
assertEquals(listOf(artifactId), requested)
|
||||
} finally {
|
||||
controller.pause().stop().destroy()
|
||||
shadowOf(Looper.getMainLooper()).idle()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
|
||||
private let globalAgentId: String?
|
||||
private let outboxGatewayID: String?
|
||||
private let sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)?
|
||||
private let imageArtifactLoader: IOSImageArtifactLoader?
|
||||
|
||||
var outboxRequiresSessionRoutingContract: Bool {
|
||||
true
|
||||
@@ -21,7 +22,8 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
|
||||
widgetGateway: GatewayNodeSession? = nil,
|
||||
globalAgentId: String? = nil,
|
||||
outboxGatewayID: String? = nil,
|
||||
sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? = nil)
|
||||
sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? = nil,
|
||||
imageArtifactLoader: IOSImageArtifactLoader? = nil)
|
||||
{
|
||||
self.gateway = gateway
|
||||
self.widgetGateway = widgetGateway
|
||||
@@ -30,6 +32,7 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
|
||||
let normalizedGatewayID = outboxGatewayID?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.outboxGatewayID = normalizedGatewayID?.isEmpty == false ? normalizedGatewayID : nil
|
||||
self.sessionMutationRequest = sessionMutationRequest
|
||||
self.imageArtifactLoader = imageArtifactLoader
|
||||
}
|
||||
|
||||
func acquireOutboxRouteLease() async -> OpenClawChatTransportRouteLeaseResult {
|
||||
@@ -562,6 +565,31 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
|
||||
})
|
||||
}
|
||||
|
||||
func loadImageArtifact(
|
||||
sessionKey: String,
|
||||
artifactId: String) async throws -> OpenClawChatLoadedImage?
|
||||
{
|
||||
guard let imageArtifactLoader,
|
||||
let route = await gateway.currentRoute(),
|
||||
let gatewayID = await gateway.currentGatewayID(ifCurrentRoute: route)
|
||||
else { return nil }
|
||||
let target = self.sessionTarget(for: sessionKey)
|
||||
let request = OpenClawChatGatewayRequests.artifactDownload(
|
||||
sessionKey: target.sessionKey,
|
||||
agentID: target.agentID,
|
||||
artifactId: artifactId)
|
||||
let data = try await gateway.request(request, ifCurrentRoute: route)
|
||||
let response = try JSONDecoder().decode(ArtifactsDownloadResult.self, from: data)
|
||||
guard let url = response.url?.trimmingCharacters(in: .whitespacesAndNewlines), !url.isEmpty
|
||||
else { return nil }
|
||||
guard await self.gateway.currentRoute() == route else { throw CancellationError() }
|
||||
let loaded = try await imageArtifactLoader.load(
|
||||
ticketedPath: url,
|
||||
expectedGatewayID: gatewayID)
|
||||
guard await self.gateway.currentRoute() == route else { throw CancellationError() }
|
||||
return loaded
|
||||
}
|
||||
|
||||
func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL? {
|
||||
await self.resolveInlineWidgetResource(
|
||||
path: path,
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import Foundation
|
||||
import OpenClawChatUI
|
||||
import OpenClawKit
|
||||
|
||||
struct IOSImageArtifactLoader: Sendable {
|
||||
struct Connection: Sendable {
|
||||
let config: GatewayConnectConfig
|
||||
let gatewayID: String
|
||||
let customHeaders: [String: String]
|
||||
}
|
||||
|
||||
enum LoadError: Error, Equatable {
|
||||
case invalidSource
|
||||
case invalidResponse
|
||||
case requestFailed(statusCode: Int)
|
||||
case unsupportedMediaType
|
||||
case payloadTooLarge
|
||||
}
|
||||
|
||||
typealias Request = @Sendable (URLRequest) async throws -> (Data, URLResponse)
|
||||
typealias RequestFactory = @Sendable (GatewayTLSParams, Int) -> Request
|
||||
typealias ConnectionProvider = @MainActor @Sendable () -> Connection?
|
||||
|
||||
static let maximumImageBytes = 12 * 1024 * 1024
|
||||
private static let managedImagePathPrefix = "/api/chat/media/outgoing/"
|
||||
private let connectionProvider: ConnectionProvider
|
||||
private let requestFactory: RequestFactory
|
||||
|
||||
init(connectionProvider: @escaping ConnectionProvider) {
|
||||
self.init(connectionProvider: connectionProvider) { tls, maximumBytes in
|
||||
let session = GatewayTLSPinningSession(params: tls)
|
||||
return { request in
|
||||
defer { session.finishTasksAndInvalidate() }
|
||||
return try await session.data(for: request, maximumBytes: maximumBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
init(
|
||||
connectionProvider: @escaping ConnectionProvider,
|
||||
requestFactory: @escaping RequestFactory)
|
||||
{
|
||||
self.connectionProvider = connectionProvider
|
||||
self.requestFactory = requestFactory
|
||||
}
|
||||
|
||||
func load(
|
||||
ticketedPath rawPath: String,
|
||||
expectedGatewayID: String) async throws -> OpenClawChatLoadedImage
|
||||
{
|
||||
let path = rawPath.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard let connection = await self.connectionProvider(),
|
||||
connection.gatewayID == expectedGatewayID,
|
||||
let url = Self.managedImageURL(config: connection.config, path: path)
|
||||
else { throw LoadError.invalidSource }
|
||||
|
||||
var request = URLRequest(url: url)
|
||||
request.timeoutInterval = 20
|
||||
request.setValue("image/*", forHTTPHeaderField: "Accept")
|
||||
if url.scheme?.lowercased() == "https" {
|
||||
for (name, value) in GatewayCustomHeaders.sanitized(connection.customHeaders) {
|
||||
request.setValue(value, forHTTPHeaderField: name)
|
||||
}
|
||||
}
|
||||
let tls = connection.config.tls ?? GatewayTLSParams(
|
||||
required: false,
|
||||
expectedFingerprint: nil,
|
||||
allowTOFU: false,
|
||||
storeKey: nil)
|
||||
let data: Data
|
||||
let response: URLResponse
|
||||
do {
|
||||
(data, response) = try await self.requestFactory(tls, Self.maximumImageBytes)(request)
|
||||
} catch is GatewayBoundedDataError {
|
||||
throw LoadError.payloadTooLarge
|
||||
}
|
||||
guard let http = response as? HTTPURLResponse else { throw LoadError.invalidResponse }
|
||||
guard (200..<300).contains(http.statusCode) else {
|
||||
throw LoadError.requestFailed(statusCode: http.statusCode)
|
||||
}
|
||||
guard let mimeType = http.mimeType?.lowercased(), mimeType.hasPrefix("image/") else {
|
||||
throw LoadError.unsupportedMediaType
|
||||
}
|
||||
guard data.count <= Self.maximumImageBytes else { throw LoadError.payloadTooLarge }
|
||||
return OpenClawChatLoadedImage(data: data, mimeType: mimeType)
|
||||
}
|
||||
|
||||
private static func managedImageURL(config: GatewayConnectConfig, path: String) -> URL? {
|
||||
guard path.hasPrefix(self.managedImagePathPrefix),
|
||||
let relative = URLComponents(string: path),
|
||||
relative.scheme == nil,
|
||||
relative.host == nil,
|
||||
relative.fragment == nil,
|
||||
relative.percentEncodedPath.hasPrefix(Self.managedImagePathPrefix),
|
||||
relative.queryItems?.contains(where: {
|
||||
$0.name == "mediaTicket" && $0.value?.isEmpty == false
|
||||
}) == true,
|
||||
var base = URLComponents(url: config.url, resolvingAgainstBaseURL: false),
|
||||
base.host != nil
|
||||
else { return nil }
|
||||
switch base.scheme?.lowercased() {
|
||||
case "wss", "https": base.scheme = "https"
|
||||
case "ws", "http": base.scheme = "http"
|
||||
default: return nil
|
||||
}
|
||||
base.percentEncodedPath = relative.percentEncodedPath
|
||||
base.percentEncodedQuery = relative.percentEncodedQuery
|
||||
base.fragment = nil
|
||||
return base.url
|
||||
}
|
||||
}
|
||||
@@ -619,11 +619,20 @@ final class NodeAppModel {
|
||||
if self.isAppleReviewDemoModeEnabled {
|
||||
return AppleReviewDemoChatTransport()
|
||||
}
|
||||
let imageArtifactLoader = IOSImageArtifactLoader { [weak self] in
|
||||
guard let config = self?.activeGatewayConnectConfig else { return nil }
|
||||
return IOSImageArtifactLoader.Connection(
|
||||
config: config,
|
||||
gatewayID: config.nodeOptions.deviceAuthGatewayID ?? config.effectiveStableID,
|
||||
customHeaders: GatewaySettingsStore.loadGatewayCustomHeaders(
|
||||
gatewayStableID: config.effectiveStableID))
|
||||
}
|
||||
return IOSGatewayChatTransport(
|
||||
gateway: self.operatorSession,
|
||||
widgetGateway: self.nodeGateway,
|
||||
globalAgentId: self.chatDeliveryAgentId,
|
||||
outboxGatewayID: outboxGatewayID)
|
||||
outboxGatewayID: outboxGatewayID,
|
||||
imageArtifactLoader: imageArtifactLoader)
|
||||
}
|
||||
|
||||
/// Gateway identity the transcript cache is scoped to: the active
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
import Testing
|
||||
@testable import OpenClaw
|
||||
|
||||
@Suite("iOS managed image artifact loader")
|
||||
struct IOSImageArtifactLoaderTests {
|
||||
@Test @MainActor func `loads ticketed image with proxy headers and without a gateway bearer`() async throws {
|
||||
let gatewayURL = try #require(URL(string: "wss://gateway.example"))
|
||||
let config = Self.config(url: gatewayURL)
|
||||
let loader = IOSImageArtifactLoader(
|
||||
connectionProvider: {
|
||||
IOSImageArtifactLoader.Connection(
|
||||
config: config,
|
||||
gatewayID: config.effectiveStableID,
|
||||
customHeaders: ["X-Proxy-Token": "proxy"])
|
||||
},
|
||||
requestFactory: { _, maximumBytes in
|
||||
#expect(maximumBytes == 12 * 1024 * 1024)
|
||||
return { request in
|
||||
#expect(request.url?.absoluteString ==
|
||||
"https://gateway.example/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full?mediaTicket=ticket")
|
||||
#expect(request.value(forHTTPHeaderField: "Authorization") == nil)
|
||||
#expect(request.value(forHTTPHeaderField: "X-Proxy-Token") == "proxy")
|
||||
let responseURL = try #require(request.url)
|
||||
let response = try #require(HTTPURLResponse(
|
||||
url: responseURL,
|
||||
statusCode: 200,
|
||||
httpVersion: nil,
|
||||
headerFields: ["Content-Type": "image/png"]))
|
||||
return (Data([1, 2, 3]), response)
|
||||
}
|
||||
})
|
||||
|
||||
let loaded = try await loader.load(
|
||||
ticketedPath:
|
||||
"/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full?mediaTicket=ticket",
|
||||
expectedGatewayID: config.effectiveStableID)
|
||||
|
||||
#expect(loaded.data == Data([1, 2, 3]))
|
||||
#expect(loaded.mimeType == "image/png")
|
||||
}
|
||||
|
||||
@Test @MainActor func `rejects absolute and unticketed paths before fetching`() async {
|
||||
let config = Self.config()
|
||||
let loader = IOSImageArtifactLoader(
|
||||
connectionProvider: {
|
||||
IOSImageArtifactLoader.Connection(
|
||||
config: config,
|
||||
gatewayID: config.effectiveStableID,
|
||||
customHeaders: [:])
|
||||
},
|
||||
requestFactory: { _, _ in
|
||||
Issue.record("invalid paths must not reach the network")
|
||||
return { _ in throw CancellationError() }
|
||||
})
|
||||
|
||||
await #expect(throws: IOSImageArtifactLoader.LoadError.invalidSource) {
|
||||
try await loader.load(
|
||||
ticketedPath: "https://example.com/image.png?mediaTicket=ticket",
|
||||
expectedGatewayID: config.effectiveStableID)
|
||||
}
|
||||
await #expect(throws: IOSImageArtifactLoader.LoadError.invalidSource) {
|
||||
try await loader.load(
|
||||
ticketedPath:
|
||||
"/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
|
||||
expectedGatewayID: config.effectiveStableID)
|
||||
}
|
||||
}
|
||||
|
||||
private static func config(
|
||||
url: URL = URL(string: "ws://127.0.0.1:18789")!) -> GatewayConnectConfig
|
||||
{
|
||||
GatewayConnectConfig(
|
||||
url: url,
|
||||
stableID: "manual|127.0.0.1|18789",
|
||||
tls: nil,
|
||||
token: nil,
|
||||
bootstrapToken: nil,
|
||||
password: nil,
|
||||
nodeOptions: GatewayConnectOptions(
|
||||
role: "node",
|
||||
scopes: [],
|
||||
caps: [],
|
||||
commands: [],
|
||||
permissions: [:],
|
||||
clientId: "ios",
|
||||
clientMode: "node",
|
||||
clientDisplayName: "Phone"))
|
||||
}
|
||||
}
|
||||
@@ -269,13 +269,65 @@ struct SwiftUIRenderSmokeTests {
|
||||
userMessageExpanded: false,
|
||||
onToggleUserMessageExpanded: {},
|
||||
inlineWidgetResolverReady: true,
|
||||
inlineWidgetResourceResolver: { _, _ in nil })
|
||||
inlineWidgetResourceResolver: { _, _ in nil },
|
||||
imageArtifactResolverReady: false,
|
||||
loadImageArtifact: { _ in nil })
|
||||
.environment(\.dynamicTypeSize, typeSize)
|
||||
|
||||
_ = Self.host(root, size: CGSize(width: 320, height: 420))
|
||||
}
|
||||
}
|
||||
|
||||
@Test @MainActor func `managed assistant image starts its artifact load`() async throws {
|
||||
let artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111"
|
||||
let message = OpenClawChatMessage(
|
||||
role: "assistant",
|
||||
content: [OpenClawChatMessageContent(
|
||||
type: "image",
|
||||
text: nil,
|
||||
mimeType: "image/png",
|
||||
fileName: nil,
|
||||
artifactId: artifactId,
|
||||
url: "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
|
||||
alt: "Managed preview",
|
||||
content: nil)],
|
||||
timestamp: 1)
|
||||
var requestedArtifactId: String?
|
||||
let root = ChatMessageBubble(
|
||||
message: message,
|
||||
style: .standard,
|
||||
markdownVariant: .standard,
|
||||
userAccent: nil,
|
||||
displayOptions: [],
|
||||
assistantName: "OpenClaw",
|
||||
assistantAvatarText: "OC",
|
||||
assistantAvatarTint: nil,
|
||||
showsAssistantAvatar: true,
|
||||
isClean: false,
|
||||
contextWindowTokens: nil,
|
||||
userMessageExpanded: false,
|
||||
onToggleUserMessageExpanded: {},
|
||||
inlineWidgetResolverReady: true,
|
||||
inlineWidgetResourceResolver: { _, _ in nil },
|
||||
imageArtifactResolverReady: true,
|
||||
loadImageArtifact: { requested in
|
||||
requestedArtifactId = requested
|
||||
return OpenClawChatLoadedImage(
|
||||
data: Data(base64Encoded:
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=")!,
|
||||
mimeType: "image/png")
|
||||
})
|
||||
let window = Self.host(root, size: CGSize(width: 393, height: 420))
|
||||
defer { window.isHidden = true }
|
||||
|
||||
let deadline = ContinuousClock().now.advanced(by: .seconds(2))
|
||||
while requestedArtifactId == nil, ContinuousClock().now < deadline {
|
||||
try await Task.sleep(for: .milliseconds(10))
|
||||
}
|
||||
|
||||
#expect(requestedArtifactId == artifactId)
|
||||
}
|
||||
|
||||
@Test @MainActor func `streaming assistant bubble builds mixed prose and code`() {
|
||||
let text = """
|
||||
Earlier prose stays visible.
|
||||
@@ -336,7 +388,9 @@ struct SwiftUIRenderSmokeTests {
|
||||
userMessageExpanded: false,
|
||||
onToggleUserMessageExpanded: {},
|
||||
inlineWidgetResolverReady: true,
|
||||
inlineWidgetResourceResolver: { _, _ in nil })
|
||||
inlineWidgetResourceResolver: { _, _ in nil },
|
||||
imageArtifactResolverReady: false,
|
||||
loadImageArtifact: { _ in nil })
|
||||
ChatStreamingAssistantBubble(
|
||||
text: text,
|
||||
markdownVariant: .standard,
|
||||
@@ -390,7 +444,9 @@ struct SwiftUIRenderSmokeTests {
|
||||
userMessageExpanded: false,
|
||||
onToggleUserMessageExpanded: {},
|
||||
inlineWidgetResolverReady: true,
|
||||
inlineWidgetResourceResolver: { _, _ in nil })
|
||||
inlineWidgetResourceResolver: { _, _ in nil },
|
||||
imageArtifactResolverReady: false,
|
||||
loadImageArtifact: { _ in nil })
|
||||
.environment(\.dynamicTypeSize, typeSize)
|
||||
|
||||
_ = Self.host(root, size: CGSize(width: 320, height: 280))
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import Foundation
|
||||
import OpenClawChatUI
|
||||
import OpenClawKit
|
||||
import OpenClawProtocol
|
||||
|
||||
private let gatewayManagedImagePathPrefix = "/api/chat/media/outgoing/"
|
||||
|
||||
extension GatewayConnection {
|
||||
func loadImageArtifact(
|
||||
sessionKey: String,
|
||||
agentID: String?,
|
||||
artifactId: String,
|
||||
ifCurrentServerLease lease: ServerLease) async throws -> OpenClawChatLoadedImage?
|
||||
{
|
||||
let request = OpenClawChatGatewayRequests.artifactDownload(
|
||||
sessionKey: sessionKey,
|
||||
agentID: agentID,
|
||||
artifactId: artifactId)
|
||||
let responseData = try await self.request(
|
||||
method: request.method,
|
||||
params: request.params,
|
||||
timeoutMs: request.timeoutMs,
|
||||
ifCurrentServerLease: lease)
|
||||
let response = try JSONDecoder().decode(ArtifactsDownloadResult.self, from: responseData)
|
||||
guard let ticketedPath = response.url?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
let url = Self.managedImageURL(gatewayURL: lease.route.url, ticketedPath: ticketedPath)
|
||||
else { return nil }
|
||||
|
||||
var urlRequest = URLRequest(url: url)
|
||||
urlRequest.timeoutInterval = 20
|
||||
urlRequest.setValue("image/*", forHTTPHeaderField: "Accept")
|
||||
// Native macOS has no per-Gateway proxy-header configuration surface today. If one is
|
||||
// added, carry its immutable snapshot on Route so the socket and ticket GET cannot diverge.
|
||||
let tls = lease.route.tls?.params ?? GatewayTLSParams(
|
||||
required: false,
|
||||
expectedFingerprint: nil,
|
||||
allowTOFU: false,
|
||||
storeKey: nil)
|
||||
let session = GatewayTLSPinningSession(params: tls)
|
||||
defer { session.finishTasksAndInvalidate() }
|
||||
let (data, urlResponse) = try await session.data(
|
||||
for: urlRequest,
|
||||
maximumBytes: 12 * 1024 * 1024)
|
||||
guard await self.isCurrentServerLease(lease) else {
|
||||
throw OpenClawChatTransportSendError.notDispatched
|
||||
}
|
||||
guard let http = urlResponse as? HTTPURLResponse,
|
||||
(200..<300).contains(http.statusCode),
|
||||
let mimeType = http.mimeType?.lowercased(),
|
||||
mimeType.hasPrefix("image/")
|
||||
else { return nil }
|
||||
return OpenClawChatLoadedImage(data: data, mimeType: mimeType)
|
||||
}
|
||||
|
||||
private static func managedImageURL(gatewayURL: URL, ticketedPath: String) -> URL? {
|
||||
guard ticketedPath.hasPrefix(gatewayManagedImagePathPrefix),
|
||||
let relative = URLComponents(string: ticketedPath),
|
||||
relative.scheme == nil,
|
||||
relative.host == nil,
|
||||
relative.fragment == nil,
|
||||
relative.queryItems?.contains(where: {
|
||||
$0.name == "mediaTicket" && $0.value?.isEmpty == false
|
||||
}) == true,
|
||||
var base = URLComponents(url: gatewayURL, resolvingAgainstBaseURL: false),
|
||||
base.host != nil
|
||||
else { return nil }
|
||||
switch base.scheme?.lowercased() {
|
||||
case "wss", "https": base.scheme = "https"
|
||||
case "ws", "http": base.scheme = "http"
|
||||
default: return nil
|
||||
}
|
||||
base.percentEncodedPath = relative.percentEncodedPath
|
||||
base.percentEncodedQuery = relative.percentEncodedQuery
|
||||
base.fragment = nil
|
||||
return base.url
|
||||
}
|
||||
}
|
||||
@@ -47,10 +47,10 @@ actor GatewayConnection {
|
||||
struct Route: Equatable, Sendable {
|
||||
fileprivate let generation: UInt64
|
||||
fileprivate let authority: UInt64?
|
||||
fileprivate let url: URL
|
||||
let url: URL
|
||||
fileprivate let token: String?
|
||||
fileprivate let password: String?
|
||||
fileprivate let tls: GatewayTLSRoute?
|
||||
let tls: GatewayTLSRoute?
|
||||
fileprivate let deviceAuthGatewayID: String?
|
||||
let activationOwnershipFingerprint: String?
|
||||
|
||||
@@ -67,7 +67,9 @@ actor GatewayConnection {
|
||||
/// One connected Gateway server, not merely an endpoint configuration.
|
||||
/// A reconnect at the same URL creates a different lease.
|
||||
struct ServerLease: Sendable {
|
||||
fileprivate let route: Route
|
||||
// Managed-image HTTP reuses this captured route from its focused extension file.
|
||||
// Carrying the snapshot forward prevents endpoint or TLS rediscovery after suspension.
|
||||
let route: Route
|
||||
fileprivate let socketGeneration: UInt64
|
||||
fileprivate let client: GatewayChannelActor
|
||||
}
|
||||
|
||||
@@ -566,6 +566,21 @@ struct MacGatewayChatTransport: OpenClawChatTransport {
|
||||
connection: self.connection)
|
||||
}
|
||||
|
||||
func loadImageArtifact(
|
||||
sessionKey: String,
|
||||
artifactId: String) async throws -> OpenClawChatLoadedImage?
|
||||
{
|
||||
guard let serverLease = await connection.captureServerLease() else {
|
||||
throw OpenClawChatTransportSendError.notDispatched
|
||||
}
|
||||
let target = self.sessionTarget(for: sessionKey)
|
||||
return try await self.connection.loadImageArtifact(
|
||||
sessionKey: target.sessionKey,
|
||||
agentID: target.agentID,
|
||||
artifactId: artifactId,
|
||||
ifCurrentServerLease: serverLease)
|
||||
}
|
||||
|
||||
var supportsSlashCommandCatalog: Bool {
|
||||
true
|
||||
}
|
||||
|
||||
@@ -80,6 +80,22 @@ public enum OpenClawChatGatewayRequests {
|
||||
OpenClawChatGatewayRequest(method: "models.list", timeoutMs: self.defaultTimeoutMs)
|
||||
}
|
||||
|
||||
public static func artifactDownload(
|
||||
sessionKey: String,
|
||||
agentID: String?,
|
||||
artifactId: String) -> OpenClawChatGatewayRequest
|
||||
{
|
||||
var params: [String: AnyCodable] = [
|
||||
"sessionKey": AnyCodable(sessionKey),
|
||||
"artifactId": AnyCodable(artifactId),
|
||||
]
|
||||
self.add(agentID, to: ¶ms, key: "agentId")
|
||||
return OpenClawChatGatewayRequest(
|
||||
method: "artifacts.download",
|
||||
params: params,
|
||||
timeoutMs: self.defaultTimeoutMs)
|
||||
}
|
||||
|
||||
public static func chatMetadata(
|
||||
sessionKey: String,
|
||||
fallbackAgentID: String?) -> OpenClawChatGatewayRequest
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import Foundation
|
||||
import ImageIO
|
||||
import SwiftUI
|
||||
|
||||
#if canImport(AppKit)
|
||||
import AppKit
|
||||
#elseif canImport(UIKit)
|
||||
import UIKit
|
||||
#endif
|
||||
|
||||
enum ChatMediaImageDecoder {
|
||||
static let maximumThumbnailPixels = 2048
|
||||
|
||||
static func decode(_ data: Data) -> OpenClawPlatformImage? {
|
||||
guard let source = CGImageSourceCreateWithData(data as CFData, nil),
|
||||
CGImageSourceGetCount(source) > 0
|
||||
else { return nil }
|
||||
let options: [CFString: Any] = [
|
||||
kCGImageSourceCreateThumbnailFromImageAlways: true,
|
||||
kCGImageSourceCreateThumbnailWithTransform: true,
|
||||
kCGImageSourceThumbnailMaxPixelSize: self.maximumThumbnailPixels,
|
||||
kCGImageSourceShouldCacheImmediately: true,
|
||||
]
|
||||
guard let cgImage = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary)
|
||||
else { return nil }
|
||||
#if canImport(AppKit)
|
||||
return NSImage(cgImage: cgImage, size: .zero)
|
||||
#elseif canImport(UIKit)
|
||||
return UIImage(cgImage: cgImage)
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
struct ChatMediaImageAttachment: View {
|
||||
private enum LoadState {
|
||||
case loading
|
||||
case loaded(OpenClawPlatformImage)
|
||||
case unavailable
|
||||
}
|
||||
|
||||
let artifactId: String
|
||||
let label: String
|
||||
let resolverReady: Bool
|
||||
let load: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
|
||||
|
||||
@State private var state: LoadState = .loading
|
||||
@State private var retryGeneration = 0
|
||||
@State private var showsFullImage = false
|
||||
|
||||
var body: some View {
|
||||
Group {
|
||||
switch self.state {
|
||||
case .loading:
|
||||
HStack(spacing: 8) {
|
||||
ProgressView()
|
||||
Text(String(localized: "Loading image…"))
|
||||
.font(OpenClawChatTypography.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
.frame(minHeight: 88)
|
||||
.frame(maxWidth: .infinity)
|
||||
case let .loaded(image):
|
||||
Button {
|
||||
self.showsFullImage = true
|
||||
} label: {
|
||||
OpenClawPlatformImageFactory.image(image)
|
||||
.resizable()
|
||||
.scaledToFit()
|
||||
.frame(maxHeight: 320)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous))
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: 12, style: .continuous)
|
||||
.strokeBorder(Color.white.opacity(0.12), lineWidth: 1))
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.accessibilityLabel(self.label)
|
||||
.accessibilityHint(String(localized: "Opens image preview"))
|
||||
.sheet(isPresented: self.$showsFullImage) {
|
||||
ZStack(alignment: .topTrailing) {
|
||||
Color.black.ignoresSafeArea()
|
||||
ScrollView([.horizontal, .vertical]) {
|
||||
OpenClawPlatformImageFactory.image(image)
|
||||
.resizable()
|
||||
.scaledToFit()
|
||||
.padding(20)
|
||||
}
|
||||
Button {
|
||||
self.showsFullImage = false
|
||||
} label: {
|
||||
Image(systemName: "xmark.circle.fill")
|
||||
.font(.title2)
|
||||
.symbolRenderingMode(.hierarchical)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.foregroundStyle(.white)
|
||||
.padding(16)
|
||||
.accessibilityLabel(String(localized: "Close image preview"))
|
||||
}
|
||||
}
|
||||
case .unavailable:
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "photo.badge.exclamationmark")
|
||||
Text(String(localized: "Image unavailable"))
|
||||
.font(OpenClawChatTypography.footnote)
|
||||
Spacer()
|
||||
Button {
|
||||
self.retryGeneration &+= 1
|
||||
} label: {
|
||||
Text(String(localized: "Retry"))
|
||||
.font(OpenClawChatTypography.footnote)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
.foregroundStyle(.secondary)
|
||||
.padding(10)
|
||||
.background(Color.black.opacity(0.04))
|
||||
.clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous))
|
||||
}
|
||||
}
|
||||
.task(id: "\(self.artifactId):\(self.resolverReady):\(self.retryGeneration)") {
|
||||
await self.loadImage()
|
||||
}
|
||||
}
|
||||
|
||||
private func loadImage() async {
|
||||
guard self.resolverReady else {
|
||||
self.state = .unavailable
|
||||
return
|
||||
}
|
||||
self.state = .loading
|
||||
do {
|
||||
guard let loaded = try await self.load(self.artifactId), !Task.isCancelled else {
|
||||
if !Task.isCancelled { self.state = .unavailable }
|
||||
return
|
||||
}
|
||||
let image = await Task.detached(priority: .userInitiated) {
|
||||
ChatMediaImageDecoder.decode(loaded.data)
|
||||
}.value
|
||||
guard !Task.isCancelled else { return }
|
||||
self.state = image.map(LoadState.loaded) ?? .unavailable
|
||||
} catch is CancellationError {
|
||||
return
|
||||
} catch {
|
||||
guard !Task.isCancelled else { return }
|
||||
self.state = .unavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -219,6 +219,8 @@ struct ChatMessageBubble: View {
|
||||
let inlineWidgetResourceResolver: @MainActor @Sendable (
|
||||
String,
|
||||
OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
|
||||
let imageArtifactResolverReady: Bool
|
||||
let loadImageArtifact: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
|
||||
|
||||
var body: some View {
|
||||
if self.isUser {
|
||||
@@ -261,7 +263,9 @@ struct ChatMessageBubble: View {
|
||||
userMessageExpanded: self.userMessageExpanded,
|
||||
onToggleUserMessageExpanded: self.onToggleUserMessageExpanded,
|
||||
inlineWidgetResolverReady: self.inlineWidgetResolverReady,
|
||||
inlineWidgetResourceResolver: self.inlineWidgetResourceResolver)
|
||||
inlineWidgetResourceResolver: self.inlineWidgetResourceResolver,
|
||||
imageArtifactResolverReady: self.imageArtifactResolverReady,
|
||||
loadImageArtifact: self.loadImageArtifact)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,6 +318,8 @@ private struct ChatMessageBody: View {
|
||||
let inlineWidgetResourceResolver: @MainActor @Sendable (
|
||||
String,
|
||||
OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
|
||||
let imageArtifactResolverReady: Bool
|
||||
let loadImageArtifact: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
|
||||
|
||||
var body: some View {
|
||||
let text = self.primaryText
|
||||
@@ -370,12 +376,24 @@ private struct ChatMessageBody: View {
|
||||
ChatLinkPreview(url: previewURL)
|
||||
}
|
||||
|
||||
if !self.inlineAttachments.isEmpty {
|
||||
ForEach(self.inlineAttachments.indices, id: \.self) { idx in
|
||||
AttachmentRow(att: self.inlineAttachments[idx], isUser: self.isUser)
|
||||
if !self.visibleInlineAttachments.isEmpty {
|
||||
ForEach(self.visibleInlineAttachments.indices, id: \.self) { idx in
|
||||
AttachmentRow(
|
||||
att: self.visibleInlineAttachments[idx],
|
||||
isUser: self.isUser,
|
||||
resolverReady: self.imageArtifactResolverReady,
|
||||
loadImage: self.loadImageArtifact)
|
||||
}
|
||||
}
|
||||
|
||||
if self.omittedImageAttachmentCount > 0 {
|
||||
Text(String(
|
||||
format: String(localized: "Additional images hidden: %lld"),
|
||||
Int64(self.omittedImageAttachmentCount)))
|
||||
.font(OpenClawChatTypography.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
|
||||
ForEach(self.inlineWidgets.indices, id: \.self) { idx in
|
||||
ChatInlineWidgetView(
|
||||
preview: self.inlineWidgets[idx],
|
||||
@@ -503,6 +521,19 @@ private struct ChatMessageBody: View {
|
||||
self.message.content.filter(\.isInlineAttachment)
|
||||
}
|
||||
|
||||
private var visibleInlineAttachments: [OpenClawChatMessageContent] {
|
||||
var imageCount = 0
|
||||
return self.inlineAttachments.filter { attachment in
|
||||
guard attachment.isImageAttachment else { return true }
|
||||
defer { imageCount += 1 }
|
||||
return imageCount < 4
|
||||
}
|
||||
}
|
||||
|
||||
private var omittedImageAttachmentCount: Int {
|
||||
max(0, self.inlineAttachments.filter(\.isImageAttachment).count - 4)
|
||||
}
|
||||
|
||||
private var inlineWidgets: [OpenClawChatCanvasPreview] {
|
||||
guard self.message.role.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "assistant"
|
||||
else { return [] }
|
||||
@@ -620,11 +651,25 @@ private struct ChatMessageBody: View {
|
||||
private struct AttachmentRow: View {
|
||||
let att: OpenClawChatMessageContent
|
||||
let isUser: Bool
|
||||
let resolverReady: Bool
|
||||
let loadImage: @MainActor @Sendable (String) async throws -> OpenClawChatLoadedImage?
|
||||
|
||||
var body: some View {
|
||||
if self.att.isImageAttachment, let artifactId = self.normalizedArtifactId {
|
||||
ChatMediaImageAttachment(
|
||||
artifactId: artifactId,
|
||||
label: self.attachmentLabel,
|
||||
resolverReady: self.resolverReady,
|
||||
load: self.loadImage)
|
||||
} else {
|
||||
self.fallbackRow
|
||||
}
|
||||
}
|
||||
|
||||
private var fallbackRow: some View {
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: self.isAudio ? "waveform" : "paperclip")
|
||||
Text(self.isAudio ? "Voice note" : (self.att.fileName ?? "Attachment"))
|
||||
Text(self.isAudio ? "Voice note" : self.attachmentLabel)
|
||||
.font(OpenClawChatTypography.footnote)
|
||||
.lineLimit(1)
|
||||
.foregroundStyle(self.isUser ? OpenClawChatTheme.userText : OpenClawChatTheme.assistantText)
|
||||
@@ -646,6 +691,26 @@ private struct AttachmentRow: View {
|
||||
private var isAudio: Bool {
|
||||
self.att.mimeType?.hasPrefix("audio/") == true
|
||||
}
|
||||
|
||||
private var normalizedArtifactId: String? {
|
||||
let value = self.att.artifactId?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return value.isEmpty ? nil : value
|
||||
}
|
||||
|
||||
private var attachmentLabel: String {
|
||||
let values = [self.att.alt, self.att.fileName]
|
||||
return values.lazy
|
||||
.compactMap { $0?.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||
.first { !$0.isEmpty } ?? String(localized: "Attachment")
|
||||
}
|
||||
}
|
||||
|
||||
extension OpenClawChatMessageContent {
|
||||
fileprivate var isImageAttachment: Bool {
|
||||
self.type?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "image" ||
|
||||
self.mimeType?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
.hasPrefix("image/") == true
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
|
||||
@@ -116,6 +116,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
|
||||
public let thinkingSignature: String?
|
||||
public let mimeType: String?
|
||||
public let fileName: String?
|
||||
public let artifactId: String?
|
||||
public let url: String?
|
||||
public let openUrl: String?
|
||||
public let alt: String?
|
||||
public let width: Int?
|
||||
public let height: Int?
|
||||
public let sizeBytes: Int?
|
||||
public let durationSeconds: Double?
|
||||
public let content: AnyCodable?
|
||||
public let preview: OpenClawChatCanvasPreview?
|
||||
@@ -144,6 +151,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
|
||||
thinkingSignature: String? = nil,
|
||||
mimeType: String?,
|
||||
fileName: String?,
|
||||
artifactId: String? = nil,
|
||||
url: String? = nil,
|
||||
openUrl: String? = nil,
|
||||
alt: String? = nil,
|
||||
width: Int? = nil,
|
||||
height: Int? = nil,
|
||||
sizeBytes: Int? = nil,
|
||||
durationSeconds: Double? = nil,
|
||||
content: AnyCodable?,
|
||||
preview: OpenClawChatCanvasPreview? = nil,
|
||||
@@ -159,6 +173,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
|
||||
self.thinkingSignature = thinkingSignature
|
||||
self.mimeType = mimeType
|
||||
self.fileName = fileName
|
||||
self.artifactId = artifactId
|
||||
self.url = url
|
||||
self.openUrl = openUrl
|
||||
self.alt = alt
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.sizeBytes = sizeBytes
|
||||
self.durationSeconds = durationSeconds
|
||||
self.content = content
|
||||
self.preview = preview
|
||||
@@ -176,6 +197,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
|
||||
case thinkingSignature
|
||||
case mimeType
|
||||
case fileName
|
||||
case artifactId
|
||||
case url
|
||||
case openUrl
|
||||
case alt
|
||||
case width
|
||||
case height
|
||||
case sizeBytes
|
||||
case durationSeconds
|
||||
case content
|
||||
case preview
|
||||
@@ -195,6 +223,15 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
|
||||
self.thinkingSignature = try container.decodeIfPresent(String.self, forKey: .thinkingSignature)
|
||||
self.mimeType = try container.decodeIfPresent(String.self, forKey: .mimeType)
|
||||
self.fileName = try container.decodeIfPresent(String.self, forKey: .fileName)
|
||||
let decodedURL = try container.decodeIfPresent(String.self, forKey: .url)
|
||||
self.url = decodedURL
|
||||
self.openUrl = try container.decodeIfPresent(String.self, forKey: .openUrl)
|
||||
self.artifactId = try container.decodeIfPresent(String.self, forKey: .artifactId)
|
||||
?? Self.managedImageArtifactId(from: decodedURL)
|
||||
self.alt = try container.decodeIfPresent(String.self, forKey: .alt)
|
||||
self.width = try container.decodeIfPresent(Int.self, forKey: .width)
|
||||
self.height = try container.decodeIfPresent(Int.self, forKey: .height)
|
||||
self.sizeBytes = try container.decodeIfPresent(Int.self, forKey: .sizeBytes)
|
||||
self.durationSeconds = try container.decodeIfPresent(Double.self, forKey: .durationSeconds)
|
||||
self.id = try container.decodeIfPresent(String.self, forKey: .id)
|
||||
self.name = try container.decodeIfPresent(String.self, forKey: .name)
|
||||
@@ -221,6 +258,13 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
|
||||
try container.encodeIfPresent(self.thinkingSignature, forKey: .thinkingSignature)
|
||||
try container.encodeIfPresent(self.mimeType, forKey: .mimeType)
|
||||
try container.encodeIfPresent(self.fileName, forKey: .fileName)
|
||||
try container.encodeIfPresent(self.artifactId, forKey: .artifactId)
|
||||
try container.encodeIfPresent(self.url, forKey: .url)
|
||||
try container.encodeIfPresent(self.openUrl, forKey: .openUrl)
|
||||
try container.encodeIfPresent(self.alt, forKey: .alt)
|
||||
try container.encodeIfPresent(self.width, forKey: .width)
|
||||
try container.encodeIfPresent(self.height, forKey: .height)
|
||||
try container.encodeIfPresent(self.sizeBytes, forKey: .sizeBytes)
|
||||
try container.encodeIfPresent(self.durationSeconds, forKey: .durationSeconds)
|
||||
try container.encodeIfPresent(self.content, forKey: .content)
|
||||
try container.encodeIfPresent(self.preview, forKey: .preview)
|
||||
@@ -230,6 +274,21 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
|
||||
try container.encodeIfPresent(self.details, forKey: .details)
|
||||
try container.encodeIfPresent(self.isError, forKey: .isError)
|
||||
}
|
||||
|
||||
private static func managedImageArtifactId(from rawURL: String?) -> String? {
|
||||
guard let rawURL,
|
||||
let components = URLComponents(string: rawURL),
|
||||
components.scheme == nil,
|
||||
components.host == nil
|
||||
else { return nil }
|
||||
let segments = components.percentEncodedPath.split(separator: "/", omittingEmptySubsequences: true)
|
||||
guard segments.count == 7,
|
||||
segments[0...3] == ["api", "chat", "media", "outgoing"],
|
||||
segments[6] == "full",
|
||||
let attachmentId = UUID(uuidString: String(segments[5]))?.uuidString.lowercased()
|
||||
else { return nil }
|
||||
return "artifact_managed_image_\(attachmentId)"
|
||||
}
|
||||
}
|
||||
|
||||
public struct OpenClawChatCanvasPreview: Codable, Hashable, Sendable {
|
||||
|
||||
@@ -1405,6 +1405,13 @@ extension OpenClawChatSQLiteTranscriptCache {
|
||||
thinkingSignature: nil,
|
||||
mimeType: item.mimeType,
|
||||
fileName: item.fileName,
|
||||
artifactId: item.artifactId,
|
||||
url: item.url,
|
||||
openUrl: item.openUrl,
|
||||
alt: item.alt,
|
||||
width: item.width,
|
||||
height: item.height,
|
||||
sizeBytes: item.sizeBytes,
|
||||
durationSeconds: item.durationSeconds,
|
||||
content: nil,
|
||||
id: item.id,
|
||||
|
||||
@@ -548,6 +548,16 @@ public struct OpenClawChatMetadataCapabilities: Codable, Sendable, Equatable {
|
||||
}
|
||||
}
|
||||
|
||||
public struct OpenClawChatLoadedImage: Sendable {
|
||||
public let data: Data
|
||||
public let mimeType: String
|
||||
|
||||
public init(data: Data, mimeType: String) {
|
||||
self.data = data
|
||||
self.mimeType = mimeType
|
||||
}
|
||||
}
|
||||
|
||||
/// One physical Gateway route for Swarm capability discovery and child paging.
|
||||
/// All pages use the captured route so a reconnect cannot combine two servers.
|
||||
public struct OpenClawChatSwarmRouteLease: Sendable {
|
||||
@@ -671,6 +681,7 @@ public protocol OpenClawChatTransport: Sendable {
|
||||
path: String,
|
||||
replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
|
||||
func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL?
|
||||
func loadImageArtifact(sessionKey: String, artifactId: String) async throws -> OpenClawChatLoadedImage?
|
||||
|
||||
func setActiveSessionKey(_ sessionKey: String) async throws
|
||||
func resetSession(sessionKey: String) async throws
|
||||
@@ -678,6 +689,13 @@ public protocol OpenClawChatTransport: Sendable {
|
||||
}
|
||||
|
||||
extension OpenClawChatTransport {
|
||||
public func loadImageArtifact(
|
||||
sessionKey _: String,
|
||||
artifactId _: String) async throws -> OpenClawChatLoadedImage?
|
||||
{
|
||||
nil
|
||||
}
|
||||
|
||||
public func isSwarmEnabled(sessionKey _: String) async throws -> Bool {
|
||||
false
|
||||
}
|
||||
|
||||
@@ -544,6 +544,13 @@ public struct OpenClawChatView: View {
|
||||
inlineWidgetResolverReady: self.viewModel.healthOK,
|
||||
inlineWidgetResourceResolver: { [weak viewModel] path, failedResource in
|
||||
await viewModel?.resolveInlineWidgetResource(path: path, replacing: failedResource)
|
||||
},
|
||||
imageArtifactResolverReady: self.viewModel.healthOK,
|
||||
loadImageArtifact: { [weak viewModel] artifactId in
|
||||
guard let viewModel else { return nil }
|
||||
return try await viewModel.transport.loadImageArtifact(
|
||||
sessionKey: viewModel.sessionKey,
|
||||
artifactId: artifactId)
|
||||
})
|
||||
.frame(
|
||||
maxWidth: .infinity,
|
||||
|
||||
+24
-2
@@ -25,6 +25,13 @@ extension OpenClawChatViewModel {
|
||||
thinkingSignature: content.thinkingSignature,
|
||||
mimeType: content.mimeType,
|
||||
fileName: content.fileName,
|
||||
artifactId: content.artifactId,
|
||||
url: content.url,
|
||||
openUrl: content.openUrl,
|
||||
alt: content.alt,
|
||||
width: content.width,
|
||||
height: content.height,
|
||||
sizeBytes: content.sizeBytes,
|
||||
durationSeconds: content.durationSeconds,
|
||||
content: content.content,
|
||||
id: content.id,
|
||||
@@ -56,7 +63,12 @@ extension OpenClawChatViewModel {
|
||||
let id = (item.id ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let name = (item.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let fileName = (item.fileName ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return [type, text, id, name, fileName].joined(separator: "\\u{001F}")
|
||||
let artifactId = (item.artifactId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let url = (item.url ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let openUrl = (item.openUrl ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let mimeType = (item.mimeType ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return [type, text, id, name, fileName, artifactId, url, openUrl, mimeType]
|
||||
.joined(separator: "\\u{001F}")
|
||||
}.joined(separator: "\\u{001E}")
|
||||
}
|
||||
|
||||
@@ -64,7 +76,10 @@ extension OpenClawChatViewModel {
|
||||
message.content.map { item in
|
||||
let type = (item.type ?? "text").trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
let text = (item.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return [type, text].joined(separator: "\\u{001F}")
|
||||
let artifactId = (item.artifactId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let url = (item.url ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let openUrl = (item.openUrl ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return [type, text, artifactId, url, openUrl].joined(separator: "\\u{001F}")
|
||||
}.joined(separator: "\\u{001E}")
|
||||
}
|
||||
|
||||
@@ -176,6 +191,13 @@ extension OpenClawChatViewModel {
|
||||
thinkingSignature: content.thinkingSignature,
|
||||
mimeType: content.mimeType,
|
||||
fileName: content.fileName,
|
||||
artifactId: content.artifactId,
|
||||
url: content.url,
|
||||
openUrl: content.openUrl,
|
||||
alt: content.alt,
|
||||
width: content.width,
|
||||
height: content.height,
|
||||
sizeBytes: content.sizeBytes,
|
||||
durationSeconds: localDuration,
|
||||
content: content.content,
|
||||
id: content.id,
|
||||
|
||||
@@ -710,6 +710,13 @@ public actor GatewayNodeSession {
|
||||
socketGeneration: socketGeneration)
|
||||
}
|
||||
|
||||
public func currentGatewayID(ifCurrentRoute route: GatewayNodeSessionRoute) -> String? {
|
||||
guard self.isCurrentRoute(route), self.channel != nil else { return nil }
|
||||
// iOS operator routes normalize this to the effective stable ID before connect.
|
||||
// Keep nil for unscoped clients rather than letting artifact HTTP bind to a guessed owner.
|
||||
return self.connectOptions?.deviceAuthGatewayID
|
||||
}
|
||||
|
||||
public func supportsServerCapability(
|
||||
_ capability: GatewayServerCapability,
|
||||
ifCurrentRoute expectedRoute: GatewayNodeSessionRoute) -> Bool?
|
||||
|
||||
@@ -81,6 +81,10 @@ public struct GatewayTLSValidationError: LocalizedError, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public enum GatewayBoundedDataError: Error, Equatable, Sendable {
|
||||
case responseTooLarge(maximumBytes: Int)
|
||||
}
|
||||
|
||||
protocol GatewayTLSFailureProviding: AnyObject {
|
||||
func consumeLastTLSFailure() -> GatewayTLSValidationFailure?
|
||||
}
|
||||
@@ -764,6 +768,42 @@ public final class GatewayTLSPinningSession: NSObject, WebSocketSessioning, URLS
|
||||
return WebSocketTaskBox(task: task)
|
||||
}
|
||||
|
||||
public func data(for request: URLRequest, maximumBytes: Int) async throws -> (Data, URLResponse) {
|
||||
self.registerExpectedAuthority(url: request.url)
|
||||
guard maximumBytes >= 0 else {
|
||||
throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes)
|
||||
}
|
||||
|
||||
let (bytes, response) = try await self.session.bytes(for: request)
|
||||
let expectedLength = response.expectedContentLength
|
||||
guard expectedLength < 0 || expectedLength <= Int64(maximumBytes) else {
|
||||
bytes.task.cancel()
|
||||
throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes)
|
||||
}
|
||||
|
||||
var data = Data()
|
||||
if expectedLength > 0 {
|
||||
data.reserveCapacity(Int(expectedLength))
|
||||
}
|
||||
do {
|
||||
for try await byte in bytes {
|
||||
guard data.count < maximumBytes else {
|
||||
bytes.task.cancel()
|
||||
throw GatewayBoundedDataError.responseTooLarge(maximumBytes: maximumBytes)
|
||||
}
|
||||
data.append(byte)
|
||||
}
|
||||
} catch {
|
||||
bytes.task.cancel()
|
||||
throw error
|
||||
}
|
||||
return (data, response)
|
||||
}
|
||||
|
||||
public func finishTasksAndInvalidate() {
|
||||
self.session.finishTasksAndInvalidate()
|
||||
}
|
||||
|
||||
public func urlSession(
|
||||
_ session: URLSession,
|
||||
didReceive challenge: URLAuthenticationChallenge,
|
||||
|
||||
@@ -11813,17 +11813,20 @@ public struct ArtifactsDownloadResult: Codable, Sendable {
|
||||
public let encoding: String?
|
||||
public let data: String?
|
||||
public let url: String?
|
||||
public let expiresat: String?
|
||||
|
||||
public init(
|
||||
artifact: ArtifactSummary,
|
||||
encoding: String? = nil,
|
||||
data: String? = nil,
|
||||
url: String? = nil)
|
||||
url: String? = nil,
|
||||
expiresat: String? = nil)
|
||||
{
|
||||
self.artifact = artifact
|
||||
self.encoding = encoding
|
||||
self.data = data
|
||||
self.url = url
|
||||
self.expiresat = expiresat
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
@@ -11831,6 +11834,7 @@ public struct ArtifactsDownloadResult: Codable, Sendable {
|
||||
case encoding
|
||||
case data
|
||||
case url
|
||||
case expiresat = "expiresAt"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import OpenClawChatUI
|
||||
|
||||
@Suite("Managed chat image attachments")
|
||||
struct ChatMessageMediaAttachmentTests {
|
||||
@Test func `decodes canonical managed image fields`() throws {
|
||||
let message = try JSONDecoder().decode(
|
||||
OpenClawChatMessage.self,
|
||||
from: Data(
|
||||
"""
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "image",
|
||||
"artifactId": "artifact_managed_image_11111111-1111-4111-8111-111111111111",
|
||||
"url": "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full",
|
||||
"openUrl": "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full",
|
||||
"alt": "Chart",
|
||||
"mimeType": "image/png",
|
||||
"width": 1200,
|
||||
"height": 800,
|
||||
"sizeBytes": 2048
|
||||
}]
|
||||
}
|
||||
""".utf8))
|
||||
|
||||
let image = try #require(message.content.first)
|
||||
#expect(image.artifactId == "artifact_managed_image_11111111-1111-4111-8111-111111111111")
|
||||
#expect(image.alt == "Chart")
|
||||
#expect(image.mimeType == "image/png")
|
||||
#expect(image.width == 1200)
|
||||
#expect(image.height == 800)
|
||||
#expect(image.sizeBytes == 2048)
|
||||
#expect(image.isInlineAttachment)
|
||||
}
|
||||
|
||||
@Test @MainActor func `distinct images never reconcile as the same final message`() {
|
||||
let first = Self.message(artifactId: "artifact_managed_image_11111111-1111-4111-8111-111111111111")
|
||||
let second = Self.message(artifactId: "artifact_managed_image_22222222-2222-4222-8222-222222222222")
|
||||
|
||||
#expect(
|
||||
OpenClawChatViewModel.finalMessageContentFingerprint(for: first) !=
|
||||
OpenClawChatViewModel.finalMessageContentFingerprint(for: second))
|
||||
#expect(
|
||||
OpenClawChatViewModel.messageContentFingerprint(for: first) !=
|
||||
OpenClawChatViewModel.messageContentFingerprint(for: second))
|
||||
}
|
||||
|
||||
@Test func `derives stable identity for shipped managed image blocks`() throws {
|
||||
let message = try JSONDecoder().decode(
|
||||
OpenClawChatMessage.self,
|
||||
from: Data(
|
||||
"""
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "image",
|
||||
"url": "/api/chat/media/outgoing/main/11111111-1111-4111-8111-111111111111/full",
|
||||
"mimeType": "image/png"
|
||||
}]
|
||||
}
|
||||
""".utf8))
|
||||
|
||||
#expect(
|
||||
message.content.first?.artifactId ==
|
||||
"artifact_managed_image_11111111-1111-4111-8111-111111111111")
|
||||
}
|
||||
|
||||
@Test func `transcript cache preserves references without image bytes`() throws {
|
||||
let message = Self.message(
|
||||
artifactId: "artifact_managed_image_11111111-1111-4111-8111-111111111111")
|
||||
let cached = try #require(OpenClawChatSQLiteTranscriptCache.cacheableMessages([message]).first)
|
||||
let image = try #require(cached.content.first)
|
||||
|
||||
#expect(image.artifactId == message.content.first?.artifactId)
|
||||
#expect(image.url == message.content.first?.url)
|
||||
#expect(image.content == nil)
|
||||
}
|
||||
|
||||
private static func message(artifactId: String) -> OpenClawChatMessage {
|
||||
OpenClawChatMessage(
|
||||
role: "assistant",
|
||||
content: [
|
||||
OpenClawChatMessageContent(
|
||||
type: "image",
|
||||
text: nil,
|
||||
mimeType: "image/png",
|
||||
fileName: nil,
|
||||
artifactId: artifactId,
|
||||
url: "/api/chat/media/outgoing/main/\(artifactId)/full",
|
||||
content: nil),
|
||||
],
|
||||
timestamp: nil)
|
||||
}
|
||||
}
|
||||
@@ -3310,6 +3310,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
|
||||
- H2: Choose scopes and pair the device
|
||||
- H2: Advertise client capabilities
|
||||
- H2: Recover state after reconnect
|
||||
- H2: Render generated image artifacts
|
||||
- H2: Use history metadata and stable anchors
|
||||
- H2: Subscribe instead of polling usage
|
||||
- H2: Backfill exec approvals
|
||||
|
||||
@@ -133,6 +133,30 @@ current WebSocket connection. It resets with a new connection. The `seq` inside
|
||||
an `agent` event payload is assigned per run and orders that run's lifecycle,
|
||||
assistant, plan, tool, and other stream events.
|
||||
|
||||
## Render generated image artifacts
|
||||
|
||||
Assistant-generated images arrive as canonical `type: "image"` content blocks.
|
||||
Managed blocks include a stable `artifactId`, a Gateway-relative `url`, MIME
|
||||
type, dimensions, size, and accessible alt text. Keep that reference in the
|
||||
transcript cache; do not persist downloaded bytes or temporary download URLs.
|
||||
|
||||
Resolve the image through the authenticated WebSocket connection:
|
||||
|
||||
1. Call `artifacts.download` with the current `sessionKey`, optional `agentId`,
|
||||
and the block's `artifactId`.
|
||||
2. Use the returned short-lived `url` before `expiresAt`. The URL is scoped to
|
||||
that exact transcript-backed artifact and does not contain a reusable Gateway
|
||||
or device credential.
|
||||
3. Fetch it from the Gateway origin using the same TLS pin and reverse-proxy
|
||||
headers as the active connection. Validate the response as an image and
|
||||
enforce a 12 MiB source limit plus a bounded decoded thumbnail.
|
||||
4. If the URL expires, repeat `artifacts.download` once. Reconnect or route
|
||||
changes cancel the old load rather than retargeting it to another Gateway.
|
||||
|
||||
Older image blocks without `artifactId` remain displayable by existing Control
|
||||
UI clients, but native clients should show a readable attachment fallback rather
|
||||
than forward a shared owner credential.
|
||||
|
||||
## Use history metadata and stable anchors
|
||||
|
||||
Rows returned by `chat.history` can carry an `__openclaw` metadata envelope:
|
||||
|
||||
@@ -301,6 +301,7 @@ The Android Chat tab supports session selection (default `main`, plus other exis
|
||||
- History: `chat.history` (display-normalized — inline directive tags, plain-text tool-call XML payloads (`<tool_call>`, `<function_call>`, `<tool_calls>`, `<function_calls>`, and truncated variants), and leaked ASCII/full-width model control tokens are stripped; silent-token assistant rows such as exact `NO_REPLY` / `no_reply` are omitted; oversized rows can be replaced with placeholders)
|
||||
- Send: `chat.send`
|
||||
- Durable sending: every send (text, picked images, and voice notes) is journaled to a per-gateway on-device outbox before any network attempt, so app termination cannot lose submitted input. Sends queued while offline deliver in order on reconnect with stable idempotency keys, and a send is retired only after the turn is visible in canonical `chat.history` — an acknowledgement alone is not treated as proof of delivery. Ambiguous outcomes (lost acknowledgement, app killed mid-send, gateway restart before the transcript write) surface as visible rows with explicit **Retry**/**Delete** instead of auto-resending. Slash commands never auto-replay across a reconnect; they park for explicit retry. The queue is bounded (50 messages and 48 MB of attachment bytes per gateway) and unsent rows expire after 48 hours. Composer drafts that were never submitted are not process-durable.
|
||||
- Image input works through the picker and Android Sharesheet. Assistant-generated images resolve through the paired Gateway connection, render inline with a full-screen preview, and retain only their small artifact references in the offline transcript cache. Downloads are capped at 12 MiB and decoded to bounded display bitmaps.
|
||||
- Push updates (best-effort): `chat.subscribe` -> `event:"chat"`
|
||||
- Listen: long-press an assistant message and choose **Listen** to hear it; audio renders via gateway `tts.speak` with the configured TTS provider chain, and on-device system TTS is used when the gateway cannot render audio. Playback stops on session switch, new chat, app backgrounding, or chat close.
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ Availability: iPhone app builds are distributed through Apple channels when enab
|
||||
- Keeps a small read-only offline cache of recent chat sessions and transcripts per paired gateway: cold opens paint the last known transcript immediately and refresh once the gateway responds, recent chats stay browsable while disconnected, and reset/forget purges the protected local cache.
|
||||
- Queues text messages sent while disconnected in a durable per-gateway outbox (up to 50): queued bubbles show in the transcript, flush in order on reconnect with idempotent retries, remain durable until canonical history confirms the send, retry with backoff before surfacing a retry/delete action, and expire instead of sending after 48 hours offline; reset/forget clears the queue with the cache.
|
||||
- Chat is the single text-and-voice surface. Chat actions can open the full Sessions screen without leaving Chat and can show or hide assistant reasoning and tool activity. Tap the microphone for draft dictation, open its menu to record a voice note, or use the inline Talk control for realtime voice; the Talk control animates from live microphone or playback level while listening or speaking.
|
||||
- Chat accepts images from the photo picker, camera, Files, paste, and the iOS share sheet. Assistant-generated images render inline from short-lived Gateway artifact URLs, open in a full-screen preview, and remain available after reconnect or history reload without storing image bytes in the transcript cache.
|
||||
- **Settings -> OpenClaw** opens a dedicated Gateway settings assistant when the operator connection has `operator.admin` and the Gateway supports `openclaw.chat`. Its setup conversation stays separate from ordinary Chat, redacts secret replies locally, and moves to Chat only after you tap **Open Chat**.
|
||||
- Speaks assistant messages on demand: long-press a message in Chat and choose **Listen**. The app plays supported gateway `tts.speak` clips with the configured TTS provider and falls back to on-device speech when gateway audio is unavailable or unplayable. Playback stops on session switch or backgrounding.
|
||||
|
||||
|
||||
@@ -13,6 +13,11 @@ Mac-hosted node tools such as `system.run`.
|
||||
|
||||
Use **Quick Chat** for a Spotlight-style main-session composer without opening a full window. Press Option-Space (⌥Space) by default, choose it from the menu bar menu, or record another shortcut in **Settings → General**.
|
||||
|
||||
The full native chat accepts image attachments through its picker, paste, and
|
||||
drag and drop. Assistant-generated images render inline through short-lived
|
||||
Gateway artifact URLs and open in a larger preview; iOS and macOS share the same
|
||||
bounded image model and renderer.
|
||||
|
||||
Only need the CLI and Gateway? Start with [Getting started](/start/getting-started).
|
||||
|
||||
## Download
|
||||
|
||||
@@ -417,7 +417,7 @@ The macOS app keeps its native link-browser sidebar for links clicked in the das
|
||||
- Re-sending with the same `idempotencyKey` returns `{ status: "in_flight" }` while running, and `{ status: "ok" }` after completion.
|
||||
- `chat.history` responses are size-bounded for UI safety. When transcript entries are too large, Gateway may truncate long text fields, omit heavy metadata blocks, and replace oversized messages with a placeholder (`[chat.history omitted: message too large]`).
|
||||
- When a visible assistant message was truncated in `chat.history`, the side reader can fetch the full display-normalized transcript entry on demand through `chat.message.get` by `sessionKey`, active `agentId` when needed, and transcript `messageId`. If the Gateway still cannot return more, the reader shows an explicit unavailable state instead of silently repeating the truncated preview.
|
||||
- Assistant/generated images are persisted as managed media references and served back through authenticated Gateway media URLs, so reloads do not depend on raw base64 image payloads staying in the chat history response.
|
||||
- Assistant/generated images are persisted as managed media references. New clients resolve their stable artifact ids through authenticated `artifacts.download` and receive short-lived, exact-resource media URLs, so reloads do not depend on raw base64 payloads or reusable credentials in image URLs.
|
||||
- When rendering `chat.history`, the Control UI strips display-only inline directive tags from visible assistant text (for example `[[reply_to_*]]` and `[[audio_as_voice]]`), plain-text tool-call XML payloads (including `<tool_call>...</tool_call>`, `<function_call>...</function_call>`, `<tool_calls>...</tool_calls>`, `<function_calls>...</function_calls>`, and truncated tool-call blocks), and leaked ASCII/full-width model control tokens. It omits assistant entries whose whole visible text is only the exact silent token `NO_REPLY` / `no_reply` or the heartbeat acknowledgement token `HEARTBEAT_OK`.
|
||||
- During an active send and the final history refresh, the chat view keeps local optimistic user/assistant messages visible if `chat.history` briefly returns an older snapshot; the canonical transcript replaces those local messages once the Gateway history catches up.
|
||||
- Live `chat` events are delivery state, while `chat.history` is rebuilt from the durable session transcript. After tool-final events the Control UI reloads history and merges only a small optimistic tail; the transcript boundary is documented in [WebChat](/web/webchat).
|
||||
@@ -663,6 +663,13 @@ When gateway auth is configured, assistant local-media previews use a two-step r
|
||||
|
||||
This keeps media rendering compatible with browser-native media elements without putting reusable gateway credentials in visible media URLs.
|
||||
|
||||
Generated images under `/api/chat/media/outgoing/...` use the same capability
|
||||
principle through `artifacts.download`. The authenticated WebSocket request
|
||||
authorizes the transcript artifact and returns a short-lived URL. The HTTP media
|
||||
route rechecks that the artifact still belongs to the transcript before serving
|
||||
bytes. The previous shared-owner bearer path remains available for older Control
|
||||
UI clients during the compatibility window.
|
||||
|
||||
## Approval links
|
||||
|
||||
Operator approval notifications can deep-link to a [standalone approval document](/web/urls#special-documents-and-startup-modes). The URL is stable for the lifetime of the approval and safe to forward between your own devices: it identifies the approval, never authorizes it.
|
||||
|
||||
@@ -68,6 +68,7 @@ export const ArtifactsDownloadResultSchema = closedObject({
|
||||
encoding: Type.Optional(Type.Literal("base64")),
|
||||
data: Type.Optional(Type.String()),
|
||||
url: Type.Optional(NonEmptyString),
|
||||
expiresAt: Type.Optional(NonEmptyString),
|
||||
});
|
||||
|
||||
// Wire types derive directly from local schema consts so public d.ts graphs never
|
||||
|
||||
@@ -55,10 +55,12 @@ vi.mock("./session-transcript-readers.js", () => ({
|
||||
|
||||
const {
|
||||
DEFAULT_MANAGED_IMAGE_ATTACHMENT_LIMITS,
|
||||
MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX,
|
||||
attachManagedOutgoingImagesToMessage,
|
||||
cleanupManagedOutgoingImageRecords,
|
||||
createManagedOutgoingImageBlocks,
|
||||
handleManagedOutgoingImageHttpRequest,
|
||||
resolveManagedOutgoingImageArtifactDownload,
|
||||
resolveManagedImageAttachmentLimits,
|
||||
} = await import("./managed-image-attachments.js");
|
||||
|
||||
@@ -101,8 +103,10 @@ async function expectPathMissing(targetPath: string): Promise<void> {
|
||||
|
||||
type ManagedImageBlock = {
|
||||
type?: string;
|
||||
artifactId?: string;
|
||||
alt?: string;
|
||||
mimeType?: string;
|
||||
sizeBytes?: number;
|
||||
url?: string;
|
||||
openUrl?: string;
|
||||
};
|
||||
@@ -319,6 +323,50 @@ describe("handleManagedOutgoingImageHttpRequest", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("serves an exact transcript image through a short-lived artifact ticket", async () => {
|
||||
const { attachmentId, sessionKey } = await createFixture(stateDir);
|
||||
const canonicalPath = `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/${attachmentId}/full`;
|
||||
loadSessionEntryMock.mockReturnValue({
|
||||
storePath: path.join(stateDir, "gateway-sessions.json"),
|
||||
entry: { sessionId: "sess-1", sessionFile: "session.jsonl" },
|
||||
});
|
||||
resolveSessionHistoryTranscriptPathMock.mockResolvedValue("session.jsonl");
|
||||
readSessionMessagesMock.mockResolvedValue([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "image", url: canonicalPath, openUrl: canonicalPath }],
|
||||
__openclaw: { id: "msg-1" },
|
||||
},
|
||||
]);
|
||||
|
||||
const download = await resolveManagedOutgoingImageArtifactDownload({
|
||||
sessionKey,
|
||||
artifactId: `${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`,
|
||||
stateDir,
|
||||
});
|
||||
expect(download?.url).toContain("mediaTicket=");
|
||||
|
||||
vi.clearAllMocks();
|
||||
const { result } = await requestManagedImage({
|
||||
stateDir,
|
||||
pathName: download?.url ?? "",
|
||||
denyAuth: true,
|
||||
});
|
||||
|
||||
expect(result.statusCode).toBe(200);
|
||||
expect(result.body.toString("utf-8")).toBe("original-image");
|
||||
expect(authorizeGatewayHttpRequestOrReplyMock).not.toHaveBeenCalled();
|
||||
|
||||
const wrongAttachmentId = "22222222-2222-4222-8222-222222222222";
|
||||
const wrong = await requestManagedImage({
|
||||
stateDir,
|
||||
pathName: (download?.url ?? "").replace(attachmentId, wrongAttachmentId),
|
||||
denyAuth: true,
|
||||
});
|
||||
expect(wrong.result.statusCode).toBe(401);
|
||||
expect(authorizeGatewayHttpRequestOrReplyMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps serving and deleting an original after the configured media root changes", async () => {
|
||||
const fixture = await createFixture(stateDir);
|
||||
const externalConfigDir = tempDirs.make("managed-image-moved-config-");
|
||||
@@ -653,6 +701,8 @@ describe("createManagedOutgoingImageBlocks", () => {
|
||||
expect(String(block.url)).toMatch(/\/full$/);
|
||||
|
||||
const attachmentId = requireAttachmentIdFromUrl(block.url);
|
||||
expect(block.artifactId).toBe(`${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`);
|
||||
expect(block.sizeBytes).toBe(Buffer.from(TINY_PNG_BASE64, "base64").byteLength);
|
||||
const record = readManagedImageRecord(attachmentId, stateDir);
|
||||
expect(record?.original.mediaSubdir).toBe(MANAGED_OUTGOING_ORIGINALS_SUBDIR);
|
||||
expect(record?.original.mediaId).toMatch(/\.png$/);
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
// Gateway managed image attachment store.
|
||||
// Validates, stores, serves, and cleans up outgoing image attachments.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import path from "node:path";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import {
|
||||
asDateTimestampMs,
|
||||
resolveTimestampMsToIsoString,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import { resolveDefaultAgentId } from "../agents/agent-scope-config.js";
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
@@ -17,6 +21,7 @@ import {
|
||||
readImageProbeFromHeader,
|
||||
} from "../media/media-services.js";
|
||||
import { getMediaDir, MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "../media/store.js";
|
||||
import { safeEqualSecret } from "../security/secret-equal.js";
|
||||
import type { AuthRateLimiter } from "./auth-rate-limit.js";
|
||||
import type { ResolvedGatewayAuth } from "./auth.js";
|
||||
import { sendJson, sendMethodNotAllowed, sendMissingScopeForbidden } from "./http-common.js";
|
||||
@@ -44,8 +49,12 @@ import {
|
||||
|
||||
const OUTGOING_IMAGE_ROUTE_PREFIX = "/api/chat/media/outgoing";
|
||||
const DEFAULT_TRANSIENT_OUTGOING_IMAGE_TTL_MS = 15 * 60 * 1000;
|
||||
const MANAGED_OUTGOING_IMAGE_TICKET_SCOPE = "managed-outgoing-image";
|
||||
export const MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS = 5 * 60 * 1000;
|
||||
export const MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX = "artifact_managed_image_";
|
||||
const MANAGED_OUTGOING_ATTACHMENT_ID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const managedOutgoingImageTicketSecret = randomBytes(32);
|
||||
|
||||
export const DEFAULT_MANAGED_IMAGE_ATTACHMENT_LIMITS = {
|
||||
maxBytes: 12 * 1024 * 1024,
|
||||
@@ -86,6 +95,24 @@ type SessionManagedOutgoingAttachmentIndexCacheEntry = {
|
||||
size: number;
|
||||
index: SessionManagedOutgoingAttachmentIndex;
|
||||
};
|
||||
|
||||
type ManagedOutgoingImageTicketPayload = {
|
||||
scope: typeof MANAGED_OUTGOING_IMAGE_TICKET_SCOPE;
|
||||
sessionKey: string;
|
||||
attachmentId: string;
|
||||
variant: "full";
|
||||
exp: number;
|
||||
};
|
||||
|
||||
export type ManagedOutgoingImageArtifactDownload = {
|
||||
artifactId: string;
|
||||
sessionKey: string;
|
||||
title: string;
|
||||
mimeType?: string;
|
||||
sizeBytes?: number;
|
||||
url: string;
|
||||
expiresAt: string;
|
||||
};
|
||||
type SessionManagedOutgoingAttachmentTranscriptStat = Omit<
|
||||
SessionManagedOutgoingAttachmentIndexCacheEntry,
|
||||
"index"
|
||||
@@ -310,6 +337,91 @@ function buildOutgoingVariantUrl(sessionKey: string, attachmentId: string, varia
|
||||
return `${OUTGOING_IMAGE_ROUTE_PREFIX}/${encodeURIComponent(sessionKey)}/${attachmentId}/${variant}`;
|
||||
}
|
||||
|
||||
function buildManagedOutgoingImageArtifactId(attachmentId: string): string {
|
||||
return `${MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX}${attachmentId}`;
|
||||
}
|
||||
|
||||
export function parseManagedOutgoingImageArtifactId(value: string): string | null {
|
||||
if (!value.startsWith(MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX)) {
|
||||
return null;
|
||||
}
|
||||
const attachmentId = value.slice(MANAGED_OUTGOING_IMAGE_ARTIFACT_ID_PREFIX.length);
|
||||
return MANAGED_OUTGOING_ATTACHMENT_ID_RE.test(attachmentId) ? attachmentId : null;
|
||||
}
|
||||
|
||||
function signManagedOutgoingImageTicketPayload(encodedPayload: string): string {
|
||||
return createHmac("sha256", managedOutgoingImageTicketSecret)
|
||||
.update(encodedPayload)
|
||||
.digest("base64url");
|
||||
}
|
||||
|
||||
function createManagedOutgoingImageTicket(params: {
|
||||
sessionKey: string;
|
||||
attachmentId: string;
|
||||
nowMs?: number;
|
||||
}): { ticket: string; expiresAt: string } | null {
|
||||
const now = asDateTimestampMs(params.nowMs ?? Date.now());
|
||||
if (now === undefined) {
|
||||
return null;
|
||||
}
|
||||
const exp = asDateTimestampMs(now + MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS);
|
||||
if (exp === undefined) {
|
||||
return null;
|
||||
}
|
||||
const payload: ManagedOutgoingImageTicketPayload = {
|
||||
scope: MANAGED_OUTGOING_IMAGE_TICKET_SCOPE,
|
||||
sessionKey: params.sessionKey,
|
||||
attachmentId: params.attachmentId,
|
||||
variant: "full",
|
||||
exp,
|
||||
};
|
||||
const encodedPayload = Buffer.from(JSON.stringify(payload), "utf8").toString("base64url");
|
||||
const signature = signManagedOutgoingImageTicketPayload(encodedPayload);
|
||||
return {
|
||||
ticket: `v1.${encodedPayload}.${signature}`,
|
||||
expiresAt: resolveTimestampMsToIsoString(exp),
|
||||
};
|
||||
}
|
||||
|
||||
function verifyManagedOutgoingImageTicket(params: {
|
||||
ticket: string | null;
|
||||
sessionKey: string;
|
||||
attachmentId: string;
|
||||
nowMs?: number;
|
||||
}): boolean {
|
||||
const now = asDateTimestampMs(params.nowMs ?? Date.now());
|
||||
if (now === undefined) {
|
||||
return false;
|
||||
}
|
||||
const parts = params.ticket?.split(".");
|
||||
if (!parts || parts.length !== 3 || parts[0] !== "v1") {
|
||||
return false;
|
||||
}
|
||||
const [, encodedPayload, signature] = parts;
|
||||
if (!encodedPayload || !signature) {
|
||||
return false;
|
||||
}
|
||||
if (!safeEqualSecret(signature, signManagedOutgoingImageTicketPayload(encodedPayload))) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(encodedPayload, "base64url").toString("utf8"),
|
||||
) as Partial<ManagedOutgoingImageTicketPayload>;
|
||||
return (
|
||||
payload.scope === MANAGED_OUTGOING_IMAGE_TICKET_SCOPE &&
|
||||
payload.sessionKey === params.sessionKey &&
|
||||
payload.attachmentId === params.attachmentId &&
|
||||
payload.variant === "full" &&
|
||||
typeof payload.exp === "number" &&
|
||||
Number.isFinite(payload.exp) &&
|
||||
payload.exp >= now
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function deriveAltText(source: string, index: number) {
|
||||
const fallback = `Generated image ${index + 1}`;
|
||||
try {
|
||||
@@ -508,12 +620,14 @@ function buildManagedImageBlock(record: ManagedImageRecord): ManagedImageBlock {
|
||||
const fullUrl = buildOutgoingVariantUrl(record.sessionKey, record.attachmentId, "full");
|
||||
return {
|
||||
type: "image",
|
||||
artifactId: buildManagedOutgoingImageArtifactId(record.attachmentId),
|
||||
url: fullUrl,
|
||||
openUrl: fullUrl,
|
||||
alt: record.alt,
|
||||
mimeType: record.original.contentType,
|
||||
width: record.original.width,
|
||||
height: record.original.height,
|
||||
sizeBytes: record.original.sizeBytes,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -565,7 +679,7 @@ function parseManagedOutgoingRoute(value: string) {
|
||||
sessionKey: decodeURIComponent(
|
||||
expectDefined(match[1], "managed image attachments regex capture 1"),
|
||||
),
|
||||
attachmentId: match[2],
|
||||
attachmentId: expectDefined(match[2], "managed image attachments regex capture 2"),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
@@ -792,6 +906,74 @@ async function recordMatchesTranscriptMessage(
|
||||
);
|
||||
}
|
||||
|
||||
async function resolveManagedOutgoingImageArtifactDownloadForRecord(
|
||||
record: ManagedImageRecord,
|
||||
): Promise<ManagedOutgoingImageArtifactDownload | null> {
|
||||
if (!(await recordMatchesTranscriptMessage(record))) {
|
||||
return null;
|
||||
}
|
||||
const ticket = createManagedOutgoingImageTicket({
|
||||
sessionKey: record.sessionKey,
|
||||
attachmentId: record.attachmentId,
|
||||
});
|
||||
if (!ticket) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const stat = await fs.stat(resolveManagedImageOriginalPath(record));
|
||||
if (!stat.isFile()) {
|
||||
return null;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const canonicalUrl = buildOutgoingVariantUrl(record.sessionKey, record.attachmentId, "full");
|
||||
const params = new URLSearchParams({ mediaTicket: ticket.ticket });
|
||||
return {
|
||||
artifactId: buildManagedOutgoingImageArtifactId(record.attachmentId),
|
||||
sessionKey: record.sessionKey,
|
||||
title: record.alt,
|
||||
...(record.original.contentType ? { mimeType: record.original.contentType } : {}),
|
||||
...(record.original.sizeBytes != null ? { sizeBytes: record.original.sizeBytes } : {}),
|
||||
url: `${canonicalUrl}?${params.toString()}`,
|
||||
expiresAt: ticket.expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolve one transcript-backed image to a short-lived HTTP capability. */
|
||||
export async function resolveManagedOutgoingImageArtifactDownload(params: {
|
||||
sessionKey: string;
|
||||
artifactId: string;
|
||||
stateDir?: string;
|
||||
}): Promise<ManagedOutgoingImageArtifactDownload | null> {
|
||||
const attachmentId = parseManagedOutgoingImageArtifactId(params.artifactId);
|
||||
if (!attachmentId) {
|
||||
return null;
|
||||
}
|
||||
const record = readManagedImageRecord(attachmentId, params.stateDir);
|
||||
if (!record || record.sessionKey !== params.sessionKey) {
|
||||
return null;
|
||||
}
|
||||
return await resolveManagedOutgoingImageArtifactDownloadForRecord(record);
|
||||
}
|
||||
|
||||
/** Upgrade legacy managed-image URLs that predate stable artifact ids. */
|
||||
export async function resolveManagedOutgoingImageUrlDownload(params: {
|
||||
sessionKey: string;
|
||||
url: string;
|
||||
stateDir?: string;
|
||||
}): Promise<ManagedOutgoingImageArtifactDownload | null> {
|
||||
const parsed = parseManagedOutgoingRoute(params.url);
|
||||
if (!parsed || parsed.sessionKey !== params.sessionKey) {
|
||||
return null;
|
||||
}
|
||||
const record = readManagedImageRecord(parsed.attachmentId, params.stateDir);
|
||||
if (!record || record.sessionKey !== params.sessionKey) {
|
||||
return null;
|
||||
}
|
||||
return await resolveManagedOutgoingImageArtifactDownloadForRecord(record);
|
||||
}
|
||||
|
||||
export async function attachManagedOutgoingImagesToMessage(params: {
|
||||
messageId: string;
|
||||
blocks?: readonly Record<string, unknown>[];
|
||||
@@ -1051,25 +1233,6 @@ export async function handleManagedOutgoingImageHttpRequest(
|
||||
return true;
|
||||
}
|
||||
|
||||
const requestAuth = await authorizeGatewayHttpRequestOrReply({
|
||||
req,
|
||||
res,
|
||||
auth: opts.auth,
|
||||
trustedProxies: opts.trustedProxies,
|
||||
allowRealIpFallback: opts.allowRealIpFallback,
|
||||
rateLimiter: opts.rateLimiter,
|
||||
});
|
||||
if (!requestAuth) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const requestedScopes = resolveOpenAiCompatibleHttpOperatorScopes(req, requestAuth);
|
||||
const scopeAuth = authorizeOperatorScopesForMethod("chat.history", requestedScopes);
|
||||
if (!scopeAuth.allowed) {
|
||||
sendMissingScopeForbidden(res, scopeAuth.missingScope);
|
||||
return true;
|
||||
}
|
||||
|
||||
const encodedSessionKey = match[1];
|
||||
const attachmentId = match[2];
|
||||
if (!encodedSessionKey || !attachmentId) {
|
||||
@@ -1086,23 +1249,49 @@ export async function handleManagedOutgoingImageHttpRequest(
|
||||
sendStatus(res, 404, "not found");
|
||||
return true;
|
||||
}
|
||||
const hasValidMediaTicket = verifyManagedOutgoingImageTicket({
|
||||
ticket: requestUrl.searchParams.get("mediaTicket"),
|
||||
sessionKey,
|
||||
attachmentId,
|
||||
});
|
||||
if (!hasValidMediaTicket) {
|
||||
const requestAuth = await authorizeGatewayHttpRequestOrReply({
|
||||
req,
|
||||
res,
|
||||
auth: opts.auth,
|
||||
trustedProxies: opts.trustedProxies,
|
||||
allowRealIpFallback: opts.allowRealIpFallback,
|
||||
rateLimiter: opts.rateLimiter,
|
||||
});
|
||||
if (!requestAuth) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const requestedScopes = resolveOpenAiCompatibleHttpOperatorScopes(req, requestAuth);
|
||||
const scopeAuth = authorizeOperatorScopesForMethod("chat.history", requestedScopes);
|
||||
if (!scopeAuth.allowed) {
|
||||
sendMissingScopeForbidden(res, scopeAuth.missingScope);
|
||||
return true;
|
||||
}
|
||||
// The reusable shared-secret route remains for older Control UI clients.
|
||||
// Ticketed clients prove the exact transcript attachment instead of
|
||||
// forwarding an owner credential through another HTTP stack.
|
||||
if (!resolveOpenAiCompatibleHttpSenderIsOwner(req, requestAuth)) {
|
||||
sendJson(res, 403, {
|
||||
ok: false,
|
||||
error: {
|
||||
type: "forbidden",
|
||||
message: "owner access required",
|
||||
},
|
||||
});
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const record = readManagedImageRecord(attachmentId, opts.stateDir);
|
||||
if (!record || record.sessionKey !== sessionKey) {
|
||||
sendStatus(res, 404, "not found");
|
||||
return true;
|
||||
}
|
||||
// Requester-session headers are client-declared, so media bytes require
|
||||
// authenticated owner/admin context rather than trusting a URL-scoped header.
|
||||
if (!resolveOpenAiCompatibleHttpSenderIsOwner(req, requestAuth)) {
|
||||
sendJson(res, 403, {
|
||||
ok: false,
|
||||
error: {
|
||||
type: "forbidden",
|
||||
message: "owner access required",
|
||||
},
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (!(await recordMatchesTranscriptMessage(record))) {
|
||||
sendStatus(res, 404, "not found");
|
||||
return true;
|
||||
@@ -1123,7 +1312,14 @@ export async function handleManagedOutgoingImageHttpRequest(
|
||||
res.statusCode = 200;
|
||||
res.setHeader("content-type", record.original.contentType || "application/octet-stream");
|
||||
res.setHeader("content-length", String(body.byteLength));
|
||||
res.setHeader("cache-control", "private, max-age=31536000, immutable");
|
||||
res.setHeader("x-content-type-options", "nosniff");
|
||||
res.setHeader("referrer-policy", "no-referrer");
|
||||
res.setHeader(
|
||||
"cache-control",
|
||||
hasValidMediaTicket
|
||||
? `private, max-age=${MANAGED_OUTGOING_IMAGE_TICKET_TTL_MS / 1000}, immutable`
|
||||
: "private, max-age=31536000, immutable",
|
||||
);
|
||||
res.setHeader(
|
||||
"content-disposition",
|
||||
`inline; filename="${safeAttachmentFilename(record.original.filename)}"`,
|
||||
|
||||
@@ -7,6 +7,8 @@ import { artifactsHandlers } from "./artifacts.js";
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
getTaskSessionLookupByIdForStatus: vi.fn(),
|
||||
loadSessionEntry: vi.fn(),
|
||||
resolveManagedArtifactDownload: vi.fn(),
|
||||
resolveManagedUrlDownload: vi.fn(),
|
||||
visitSessionMessagesAsync: vi.fn(),
|
||||
resolveSessionKeyForRun: vi.fn(),
|
||||
}));
|
||||
@@ -44,6 +46,17 @@ vi.mock("../server-session-key.js", async () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../managed-image-attachments.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../managed-image-attachments.js")>(
|
||||
"../managed-image-attachments.js",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
resolveManagedOutgoingImageArtifactDownload: hoisted.resolveManagedArtifactDownload,
|
||||
resolveManagedOutgoingImageUrlDownload: hoisted.resolveManagedUrlDownload,
|
||||
};
|
||||
});
|
||||
|
||||
function createResponder() {
|
||||
const calls: Array<{ ok: boolean; payload?: unknown; error?: unknown }> = [];
|
||||
return {
|
||||
@@ -211,6 +224,8 @@ describe("artifacts RPC handlers", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
hoisted.resolveSessionKeyForRun.mockReset();
|
||||
hoisted.resolveManagedArtifactDownload.mockResolvedValue(null);
|
||||
hoisted.resolveManagedUrlDownload.mockResolvedValue(null);
|
||||
hoisted.getTaskSessionLookupByIdForStatus.mockReturnValue(undefined);
|
||||
hoisted.loadSessionEntry.mockReturnValue({
|
||||
storePath: "/tmp/sessions.json",
|
||||
@@ -360,6 +375,48 @@ describe("artifacts RPC handlers", () => {
|
||||
expectFields(downloadPayload.artifact, { id: artifactId });
|
||||
});
|
||||
|
||||
it("preserves managed artifact identity and returns a ticketed download URL", async () => {
|
||||
const artifactId = "artifact_managed_image_11111111-1111-4111-8111-111111111111";
|
||||
mockedMessages([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
artifactId,
|
||||
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/11111111-1111-4111-8111-111111111111/full",
|
||||
alt: "chart.png",
|
||||
mimeType: "image/png",
|
||||
sizeBytes: 14,
|
||||
},
|
||||
],
|
||||
__openclaw: { seq: 2 },
|
||||
},
|
||||
]);
|
||||
hoisted.resolveManagedArtifactDownload.mockResolvedValue({
|
||||
artifactId,
|
||||
sessionKey: "agent:main:main",
|
||||
title: "chart.png",
|
||||
mimeType: "image/png",
|
||||
sizeBytes: 14,
|
||||
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/id/full?mediaTicket=ticket",
|
||||
expiresAt: "2026-07-28T05:00:00.000Z",
|
||||
});
|
||||
|
||||
const listed = await listArtifacts({ sessionKey: "agent:main:main" });
|
||||
expectFields(expectFirstArtifact(listed.calls), { id: artifactId, sizeBytes: 14 });
|
||||
|
||||
const downloaded = await downloadArtifact({ sessionKey: "agent:main:main", artifactId });
|
||||
expectFields(expectOkPayload(downloaded.calls), {
|
||||
url: "/api/chat/media/outgoing/agent%3Amain%3Amain/id/full?mediaTicket=ticket",
|
||||
expiresAt: "2026-07-28T05:00:00.000Z",
|
||||
});
|
||||
expect(hoisted.resolveManagedArtifactDownload).toHaveBeenCalledWith({
|
||||
sessionKey: "agent:main:main",
|
||||
artifactId,
|
||||
});
|
||||
});
|
||||
|
||||
it("can scan artifact summaries without retaining inline data", async () => {
|
||||
mockedMessages([
|
||||
{
|
||||
|
||||
@@ -22,6 +22,11 @@ import {
|
||||
toAgentStoreSessionKey,
|
||||
} from "../../routing/session-key.js";
|
||||
import { getTaskSessionLookupByIdForStatus } from "../../tasks/task-status-access.js";
|
||||
import {
|
||||
parseManagedOutgoingImageArtifactId,
|
||||
resolveManagedOutgoingImageArtifactDownload,
|
||||
resolveManagedOutgoingImageUrlDownload,
|
||||
} from "../managed-image-attachments.js";
|
||||
import { resolveSessionKeyForRun } from "../server-session-key.js";
|
||||
import {
|
||||
resolveSessionStoreAgentId,
|
||||
@@ -411,13 +416,17 @@ function collectArtifactsFromMessage(params: {
|
||||
asNonEmptyString(block.filename) ??
|
||||
asNonEmptyString(block.alt) ??
|
||||
`${type} ${params.artifacts.length + 1}`;
|
||||
const id = artifactId({
|
||||
sessionKey: params.sessionKey,
|
||||
messageSeq,
|
||||
contentIndex,
|
||||
title,
|
||||
type,
|
||||
});
|
||||
const declaredArtifactId = asNonEmptyString(block.artifactId);
|
||||
const id =
|
||||
declaredArtifactId && parseManagedOutgoingImageArtifactId(declaredArtifactId)
|
||||
? declaredArtifactId
|
||||
: artifactId({
|
||||
sessionKey: params.sessionKey,
|
||||
messageSeq,
|
||||
contentIndex,
|
||||
title,
|
||||
type,
|
||||
});
|
||||
const includeData = params.downloadArtifactId
|
||||
? params.downloadArtifactId === id
|
||||
: params.includeDownloadData !== false;
|
||||
@@ -639,6 +648,37 @@ export const artifactsHandlers: GatewayRequestHandlers = {
|
||||
if (!requireQueryable(params, respond)) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
params.sessionKey &&
|
||||
!params.runId &&
|
||||
!params.taskId &&
|
||||
parseManagedOutgoingImageArtifactId(params.artifactId)
|
||||
) {
|
||||
const resolved = resolveQuerySession(params, context.getRuntimeConfig?.());
|
||||
const managed = resolved
|
||||
? await resolveManagedOutgoingImageArtifactDownload({
|
||||
sessionKey: resolved.sessionKey,
|
||||
artifactId: params.artifactId,
|
||||
})
|
||||
: null;
|
||||
if (managed) {
|
||||
respond(true, {
|
||||
artifact: {
|
||||
id: managed.artifactId,
|
||||
type: "image",
|
||||
title: managed.title,
|
||||
...(managed.mimeType ? { mimeType: managed.mimeType } : {}),
|
||||
...(managed.sizeBytes !== undefined ? { sizeBytes: managed.sizeBytes } : {}),
|
||||
sessionKey: managed.sessionKey,
|
||||
source: "session-transcript",
|
||||
download: { mode: "url" as const },
|
||||
},
|
||||
url: managed.url,
|
||||
expiresAt: managed.expiresAt,
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
const { artifact } = await findArtifact(params, context.getRuntimeConfig?.(), {
|
||||
downloadArtifactId: params.artifactId,
|
||||
});
|
||||
@@ -662,12 +702,24 @@ export const artifactsHandlers: GatewayRequestHandlers = {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const managedUrl =
|
||||
artifact.download.mode === "url" && artifact.url && artifact.sessionKey
|
||||
? await resolveManagedOutgoingImageUrlDownload({
|
||||
sessionKey: artifact.sessionKey,
|
||||
url: artifact.url,
|
||||
})
|
||||
: null;
|
||||
respond(true, {
|
||||
artifact: toSummary(artifact),
|
||||
...(artifact.download.mode === "bytes"
|
||||
? { encoding: "base64" as const, data: artifact.data }
|
||||
: {}),
|
||||
...(artifact.download.mode === "url" ? { url: artifact.url } : {}),
|
||||
...(artifact.download.mode === "url"
|
||||
? {
|
||||
url: managedUrl?.url ?? artifact.url,
|
||||
...(managedUrl ? { expiresAt: managedUrl.expiresAt } : {}),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
@@ -464,6 +464,7 @@ export type ArtifactDownloadResult = {
|
||||
encoding?: "base64";
|
||||
data?: string;
|
||||
url?: string;
|
||||
expiresAt?: string;
|
||||
};
|
||||
|
||||
export type SessionRunStatus = "running" | "done" | "failed" | "killed" | "timeout";
|
||||
|
||||
@@ -109,6 +109,83 @@ suite.define(() => {
|
||||
}
|
||||
});
|
||||
|
||||
it("renders a managed image through an artifact-scoped ticket", async () => {
|
||||
const context = await suite.newBrowserContext({
|
||||
locale: "en-US",
|
||||
serviceWorkers: "block",
|
||||
viewport: { height: 900, width: 1280 },
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const attachmentId = crypto.randomUUID();
|
||||
const artifactId = `artifact_managed_image_${attachmentId}`;
|
||||
const imageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${attachmentId}/full`;
|
||||
const ticketedUrl = `${imageUrl}?mediaTicket=ticket-e2e`;
|
||||
await page.route("**/api/chat/media/outgoing/**", async (route) => {
|
||||
const request = route.request();
|
||||
expect(new URL(request.url()).searchParams.get("mediaTicket")).toBe("ticket-e2e");
|
||||
expect(request.headers().authorization).toBeUndefined();
|
||||
await route.fulfill({
|
||||
contentType: "image/png",
|
||||
body: Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
),
|
||||
});
|
||||
});
|
||||
const gateway = await installMockGateway(page, {
|
||||
historyMessages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
artifactId,
|
||||
url: imageUrl,
|
||||
alt: "Ticketed generated image",
|
||||
mimeType: "image/png",
|
||||
width: 1,
|
||||
height: 1,
|
||||
},
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
methodResponses: {
|
||||
"artifacts.download": {
|
||||
artifact: {
|
||||
id: artifactId,
|
||||
type: "image",
|
||||
title: "Ticketed generated image",
|
||||
mimeType: "image/png",
|
||||
download: { mode: "url" },
|
||||
},
|
||||
url: ticketedUrl,
|
||||
expiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(`${suite.server.baseUrl}chat`);
|
||||
const image = page.getByAltText("Ticketed generated image");
|
||||
await image.waitFor({ state: "visible", timeout: 10_000 });
|
||||
await expect
|
||||
.poll(() =>
|
||||
image.evaluate((element) =>
|
||||
element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0,
|
||||
),
|
||||
)
|
||||
.toBe(1);
|
||||
const request = await gateway.waitForRequest("artifacts.download");
|
||||
expect(request.params).toMatchObject({
|
||||
sessionKey: "agent:main:main",
|
||||
artifactId,
|
||||
});
|
||||
} finally {
|
||||
await suite.closeBrowserContext(context);
|
||||
}
|
||||
});
|
||||
|
||||
it("renders a canonical inbound image through the ticketed media route", async () => {
|
||||
const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim();
|
||||
const context = await suite.newBrowserContext({
|
||||
|
||||
@@ -1,6 +1,31 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { SessionObserverDigest } from "../../../packages/gateway-protocol/src/schema/sessions.js";
|
||||
import { isCriticalObserverHealth, ObserverDigestHistory } from "./observer-digest.ts";
|
||||
import {
|
||||
isCriticalObserverHealth,
|
||||
ObserverDigestHistory,
|
||||
projectSessionObserverDigest,
|
||||
} from "./observer-digest.ts";
|
||||
|
||||
describe("projectSessionObserverDigest", () => {
|
||||
it("binds a session-row projection to its owning session", () => {
|
||||
expect(
|
||||
projectSessionObserverDigest("agent:main:projected", {
|
||||
runId: "run-1",
|
||||
revision: 2,
|
||||
updatedAt: 3,
|
||||
headline: "Projected",
|
||||
health: "on-track",
|
||||
}),
|
||||
).toEqual({
|
||||
sessionKey: "agent:main:projected",
|
||||
runId: "run-1",
|
||||
revision: 2,
|
||||
updatedAt: 3,
|
||||
headline: "Projected",
|
||||
health: "on-track",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("isCriticalObserverHealth", () => {
|
||||
it("recognizes only health states that require operator attention", () => {
|
||||
|
||||
@@ -13,6 +13,24 @@ type ProjectedObserverDigest = Pick<
|
||||
"agentId" | "runId" | "headline" | "health" | "updatedAt" | "revision"
|
||||
>;
|
||||
|
||||
export function projectSessionObserverDigest(
|
||||
sessionKey: string,
|
||||
digest: ProjectedObserverDigest | null | undefined,
|
||||
): SessionObserverDigest | null {
|
||||
if (!digest) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
sessionKey,
|
||||
...(digest.agentId ? { agentId: digest.agentId } : {}),
|
||||
runId: digest.runId,
|
||||
revision: digest.revision,
|
||||
updatedAt: digest.updatedAt,
|
||||
headline: digest.headline,
|
||||
health: digest.health,
|
||||
};
|
||||
}
|
||||
|
||||
export function isCriticalObserverHealth(health: unknown): health is "stuck" | "waiting-on-user" {
|
||||
return health === "stuck" || health === "waiting-on-user";
|
||||
}
|
||||
|
||||
@@ -115,6 +115,7 @@ export {
|
||||
export {
|
||||
ObserverDigestHistory,
|
||||
pickFreshestObserverDigest,
|
||||
projectSessionObserverDigest,
|
||||
resolveChatPaneObserverRunId,
|
||||
} from "../../lib/observer-digest.ts";
|
||||
export { isWorkboardEnabledInConfigSnapshot } from "../../lib/plugin-activation.ts";
|
||||
@@ -175,6 +176,7 @@ export { sendSessionObserverVisibility } from "./chat-observer.ts";
|
||||
export {
|
||||
applySelectedSessionProjection,
|
||||
dismissChatError,
|
||||
resolveChatArtifactDownload,
|
||||
resolveAssistantAttachmentAuthToken,
|
||||
SessionParticipationTracker,
|
||||
} from "./chat-pane-state.ts";
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
openSessionWorkspaceFile,
|
||||
parseCatalogSessionKey,
|
||||
pickFreshestObserverDigest,
|
||||
projectSessionObserverDigest,
|
||||
readPresenceEntries,
|
||||
refreshChatCommands,
|
||||
refreshPageChat,
|
||||
@@ -30,6 +31,7 @@ import {
|
||||
resolveActiveRunOutputTokens,
|
||||
resolveChatProjectionRunId,
|
||||
resolveAssistantAttachmentAuthToken,
|
||||
resolveChatArtifactDownload,
|
||||
resolveChatAgentId,
|
||||
resolveChatAvatarUrl,
|
||||
resolveControlUiFollowUpMode,
|
||||
@@ -54,7 +56,6 @@ import {
|
||||
workspaceResultConflictFromPlacement,
|
||||
type BoardViewCallbacks,
|
||||
type ChatProps,
|
||||
type SessionObserverDigest,
|
||||
type SidebarSide,
|
||||
type SidebarSlotId,
|
||||
} from "./chat-pane-deps.ts";
|
||||
@@ -79,19 +80,10 @@ export class ChatPaneRender extends ChatPaneHeaderRender {
|
||||
return html`<main class="app-shell app-shell--booting" aria-busy="true"></main>`;
|
||||
}
|
||||
const selectedSession = selectedChatSessionRow(state);
|
||||
const projectedObserverDigest: SessionObserverDigest | null = selectedSession?.observerDigest
|
||||
? {
|
||||
sessionKey: selectedSession.key,
|
||||
...(selectedSession.observerDigest.agentId
|
||||
? { agentId: selectedSession.observerDigest.agentId }
|
||||
: {}),
|
||||
runId: selectedSession.observerDigest.runId,
|
||||
revision: selectedSession.observerDigest.revision,
|
||||
updatedAt: selectedSession.observerDigest.updatedAt,
|
||||
headline: selectedSession.observerDigest.headline,
|
||||
health: selectedSession.observerDigest.health,
|
||||
}
|
||||
: null;
|
||||
const projectedObserverDigest = projectSessionObserverDigest(
|
||||
selectedSession?.key ?? state.sessionKey,
|
||||
selectedSession?.observerDigest,
|
||||
);
|
||||
const observerDigest = pickFreshestObserverDigest(
|
||||
state.observerDigest,
|
||||
projectedObserverDigest,
|
||||
@@ -559,6 +551,7 @@ export class ChatPaneRender extends ChatPaneHeaderRender {
|
||||
allowExternalEmbedUrls: state.allowExternalEmbedUrls,
|
||||
chatMessageMaxWidth: state.settings.chatMessageMaxWidth,
|
||||
assistantAttachmentAuthToken: resolveAssistantAttachmentAuthToken(state as never),
|
||||
resolveArtifactDownload: (params) => resolveChatArtifactDownload(state, params),
|
||||
basePath: state.basePath,
|
||||
gatewayUrl: state.settings.gatewayUrl,
|
||||
};
|
||||
|
||||
@@ -2,7 +2,11 @@ import { describe, expect, it } from "vitest";
|
||||
import type { SessionsListResult } from "../../api/types.ts";
|
||||
import { reconcileSessionHistory } from "../../lib/sessions/reconcile.ts";
|
||||
import { areUiSessionKeysEquivalent } from "../../lib/sessions/session-key.ts";
|
||||
import { applySelectedSessionProjection, SessionParticipationTracker } from "./chat-pane-state.ts";
|
||||
import {
|
||||
applySelectedSessionProjection,
|
||||
resolveChatArtifactDownload,
|
||||
SessionParticipationTracker,
|
||||
} from "./chat-pane-state.ts";
|
||||
|
||||
function projectionState(): Parameters<typeof applySelectedSessionProjection>[0] {
|
||||
return {
|
||||
@@ -80,6 +84,44 @@ describe("applySelectedSessionProjection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveChatArtifactDownload", () => {
|
||||
it("returns a trimmed ticket without exposing a gateway bearer credential", async () => {
|
||||
const requests: Array<{ method: string; params: unknown }> = [];
|
||||
const result = await resolveChatArtifactDownload(
|
||||
{
|
||||
connected: true,
|
||||
client: {
|
||||
request: async (method: string, params: unknown) => {
|
||||
requests.push({ method, params });
|
||||
return {
|
||||
artifact: {
|
||||
id: "artifact-1",
|
||||
type: "image",
|
||||
title: "image",
|
||||
download: { mode: "url" },
|
||||
},
|
||||
url: " /api/chat/media/outgoing/main/image/full?mediaTicket=ticket ",
|
||||
expiresAt: " 2026-07-28T00:00:00.000Z ",
|
||||
};
|
||||
},
|
||||
} as never,
|
||||
},
|
||||
{ sessionKey: "agent:main:main", artifactId: "artifact-1" },
|
||||
);
|
||||
|
||||
expect(requests).toEqual([
|
||||
{
|
||||
method: "artifacts.download",
|
||||
params: { sessionKey: "agent:main:main", artifactId: "artifact-1" },
|
||||
},
|
||||
]);
|
||||
expect(result).toEqual({
|
||||
url: "/api/chat/media/outgoing/main/image/full?mediaTicket=ticket",
|
||||
expiresAt: "2026-07-28T00:00:00.000Z",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("SessionParticipationTracker", () => {
|
||||
const resolve = (
|
||||
tracker: SessionParticipationTracker,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { GatewaySessionRow } from "../../api/types.ts";
|
||||
import type { GatewayBrowserClient } from "../../api/gateway.ts";
|
||||
import type { ArtifactDownloadResult, GatewaySessionRow } from "../../api/types.ts";
|
||||
import { resolveControlUiAuthToken } from "../../app/control-ui-auth.ts";
|
||||
|
||||
type SelectedSessionProjectionState = {
|
||||
@@ -83,6 +84,25 @@ export function resolveAssistantAttachmentAuthToken(state: {
|
||||
return resolveControlUiAuthToken(state);
|
||||
}
|
||||
|
||||
export async function resolveChatArtifactDownload(
|
||||
state: { connected: boolean; client?: GatewayBrowserClient | null },
|
||||
params: { sessionKey: string; artifactId: string },
|
||||
): Promise<{ url: string; expiresAt?: string } | null> {
|
||||
if (!state.connected || !state.client) {
|
||||
return null;
|
||||
}
|
||||
const result = await state.client.request<ArtifactDownloadResult | null>(
|
||||
"artifacts.download",
|
||||
params,
|
||||
);
|
||||
const url = typeof result?.url === "string" ? result.url.trim() : "";
|
||||
if (!url) {
|
||||
return null;
|
||||
}
|
||||
const expiresAt = typeof result?.expiresAt === "string" ? result.expiresAt.trim() : undefined;
|
||||
return { url, ...(expiresAt ? { expiresAt } : {}) };
|
||||
}
|
||||
|
||||
export function dismissChatError(state: {
|
||||
chatError?: string | null;
|
||||
lastError: string | null;
|
||||
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
import type { ChatComposerDisabledBanner } from "./components/chat-composer-types.ts";
|
||||
import { isChatRunWorking, renderChatComposer } from "./components/chat-composer.ts";
|
||||
import { inlineChatImageFromEvent, openInlineChatImage } from "./components/chat-image-lightbox.ts";
|
||||
import type { ArtifactDownloadResolver } from "./components/chat-message-media.ts";
|
||||
import { renderChatPullRequests } from "./components/chat-pull-requests.ts";
|
||||
import type { SessionRailMode } from "./components/chat-session-rail.ts";
|
||||
import { renderChatSessionSuggestions } from "./components/chat-session-suggestions.ts";
|
||||
@@ -176,6 +177,7 @@ export type ChatProps = {
|
||||
userAvatar?: string | null;
|
||||
localMediaPreviewRoots?: string[];
|
||||
assistantAttachmentAuthToken?: string | null;
|
||||
resolveArtifactDownload?: ArtifactDownloadResolver;
|
||||
autoExpandToolCalls?: boolean;
|
||||
attachments?: ChatAttachment[];
|
||||
getAttachments?: () => ChatAttachment[];
|
||||
@@ -331,6 +333,7 @@ export function renderChat(props: ChatProps) {
|
||||
fullMessageAgentId: props.fullMessageAgentId,
|
||||
localMediaPreviewRoots: props.localMediaPreviewRoots,
|
||||
assistantAttachmentAuthToken: props.assistantAttachmentAuthToken,
|
||||
resolveArtifactDownload: props.resolveArtifactDownload,
|
||||
canvasPluginSurfaceUrl: props.canvasPluginSurfaceUrl,
|
||||
embedSandboxMode: props.embedSandboxMode,
|
||||
allowExternalEmbedUrls: props.allowExternalEmbedUrls,
|
||||
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
extractTranscriptAttachments,
|
||||
schedulePairingQrExpiryRefresh,
|
||||
type AttachmentItem,
|
||||
type ArtifactDownloadResolver,
|
||||
type PairingQrExpiryNotice,
|
||||
} from "./chat-message-media.ts";
|
||||
import type { SidebarContent } from "./chat-sidebar.ts";
|
||||
@@ -176,6 +177,7 @@ export function renderGroupedMessage(
|
||||
basePath?: string;
|
||||
localMediaPreviewRoots?: readonly string[];
|
||||
assistantAttachmentAuthToken?: string | null;
|
||||
resolveArtifactDownload?: ArtifactDownloadResolver;
|
||||
onAssistantAttachmentLoaded?: () => void;
|
||||
onRequestOpenImage?: () => number;
|
||||
onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void;
|
||||
@@ -209,6 +211,7 @@ export function renderGroupedMessage(
|
||||
onRequestUpdate: opts.onRequestUpdate,
|
||||
onRequestOpenImage: opts.onRequestOpenImage,
|
||||
onOpenImage: opts.onOpenImage,
|
||||
resolveArtifactDownload: opts.resolveArtifactDownload,
|
||||
};
|
||||
schedulePairingQrExpiryRefresh(messageKey, message, opts.onRequestUpdate);
|
||||
const images = resolveRenderableMessageImages(extractImages(message), imageRenderOptions);
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
resolveMessageActionDetails,
|
||||
type MessageReplyTarget,
|
||||
} from "./chat-message-markdown.ts";
|
||||
import type { ArtifactDownloadResolver } from "./chat-message-media.ts";
|
||||
import {
|
||||
renderStreamGroupParts,
|
||||
type StreamGroupOptions,
|
||||
@@ -76,6 +77,7 @@ type RenderMessageGroupOptions = {
|
||||
basePath?: string;
|
||||
localMediaPreviewRoots?: readonly string[];
|
||||
assistantAttachmentAuthToken?: string | null;
|
||||
resolveArtifactDownload?: ArtifactDownloadResolver;
|
||||
canvasPluginSurfaceUrl?: string | null;
|
||||
embedSandboxMode?: EmbedSandboxMode;
|
||||
allowExternalEmbedUrls?: boolean;
|
||||
@@ -126,6 +128,7 @@ function buildGroupedMessageRenderOptions(
|
||||
basePath: opts.basePath,
|
||||
localMediaPreviewRoots: opts.localMediaPreviewRoots,
|
||||
assistantAttachmentAuthToken: opts.assistantAttachmentAuthToken,
|
||||
resolveArtifactDownload: opts.resolveArtifactDownload,
|
||||
embedSandboxMode: opts.embedSandboxMode,
|
||||
allowExternalEmbedUrls: opts.allowExternalEmbedUrls,
|
||||
};
|
||||
|
||||
@@ -67,10 +67,11 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
|
||||
const requestVersion = opts?.onRequestOpenImage?.();
|
||||
const managedSource = isManagedOutgoingImageSource(img.displayUrl);
|
||||
const cacheKey = managedSource
|
||||
? resolveManagedOutgoingImageBlobUrlCacheKey(img.displayUrl, opts)
|
||||
? resolveManagedOutgoingImageBlobUrlCacheKey(img.displayUrl, opts, img.artifactId)
|
||||
: undefined;
|
||||
const previewIsCurrent =
|
||||
!managedSource || readManagedOutgoingImageBlobUrl(img.displayUrl, opts) === previewUrl;
|
||||
!managedSource ||
|
||||
readManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId) === previewUrl;
|
||||
if (previewIsCurrent) {
|
||||
const release =
|
||||
opts?.onOpenImage && cacheKey ? retainManagedImageBlobUrl(cacheKey) : undefined;
|
||||
@@ -82,7 +83,7 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
|
||||
// Re-resolve before opening so the modal never receives a revoked URL.
|
||||
if (!opts?.onOpenImage) {
|
||||
const pendingWindow = reserveExternalWindowForDeferredNavigation();
|
||||
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts)
|
||||
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId)
|
||||
.then((freshUrl) => {
|
||||
const safeUrl = freshUrl
|
||||
? resolveSafeExternalUrl(freshUrl, window.location.href, { allowDataImage: true })
|
||||
@@ -98,7 +99,7 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
|
||||
.catch(() => pendingWindow?.close());
|
||||
return;
|
||||
}
|
||||
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts)
|
||||
void resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId)
|
||||
.then((freshUrl) => {
|
||||
if (!freshUrl) {
|
||||
return;
|
||||
@@ -133,12 +134,14 @@ export function renderMessageImages(images: RenderableImageBlock[], opts?: Image
|
||||
if (!isManagedOutgoingImageSource(img.displayUrl)) {
|
||||
return renderImageElement(img, img.displayUrl);
|
||||
}
|
||||
const preview = resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts).then((previewUrl) => {
|
||||
if (!previewUrl) {
|
||||
return nothing;
|
||||
}
|
||||
return renderImageElement(img, previewUrl);
|
||||
});
|
||||
const preview = resolveManagedOutgoingImageBlobUrl(img.displayUrl, opts, img.artifactId).then(
|
||||
(previewUrl) => {
|
||||
if (!previewUrl) {
|
||||
return nothing;
|
||||
}
|
||||
return renderImageElement(img, previewUrl);
|
||||
},
|
||||
);
|
||||
return until(preview, nothing);
|
||||
};
|
||||
|
||||
@@ -175,24 +178,29 @@ function resolveManagedOutgoingImageRequesterSessionKey(source: string): string
|
||||
function resolveManagedOutgoingImageBlobUrlCacheKey(
|
||||
source: string,
|
||||
opts?: ImageRenderOptions,
|
||||
artifactId?: string,
|
||||
): string {
|
||||
const authToken = opts?.authToken?.trim() ?? "";
|
||||
return `${source}::${authToken}`;
|
||||
return `${source}::${authToken}::${artifactId?.trim() ?? ""}`;
|
||||
}
|
||||
|
||||
function readManagedOutgoingImageBlobUrl(
|
||||
source: string,
|
||||
opts?: ImageRenderOptions,
|
||||
artifactId?: string,
|
||||
): string | undefined {
|
||||
return readManagedImageBlobUrl(resolveManagedOutgoingImageBlobUrlCacheKey(source, opts));
|
||||
return readManagedImageBlobUrl(
|
||||
resolveManagedOutgoingImageBlobUrlCacheKey(source, opts, artifactId),
|
||||
);
|
||||
}
|
||||
|
||||
async function resolveManagedOutgoingImageBlobUrl(
|
||||
source: string,
|
||||
opts?: ImageRenderOptions,
|
||||
artifactId?: string,
|
||||
): Promise<string | null> {
|
||||
const authToken = opts?.authToken?.trim() ?? "";
|
||||
const cacheKey = resolveManagedOutgoingImageBlobUrlCacheKey(source, opts);
|
||||
const cacheKey = resolveManagedOutgoingImageBlobUrlCacheKey(source, opts, artifactId);
|
||||
const cached = readManagedImageBlobUrl(cacheKey);
|
||||
if (cached) {
|
||||
return cached;
|
||||
@@ -204,11 +212,18 @@ async function resolveManagedOutgoingImageBlobUrl(
|
||||
if (!pending) {
|
||||
pending = (async () => {
|
||||
const requesterSessionKey = resolveManagedOutgoingImageRequesterSessionKey(source);
|
||||
const artifactDownload =
|
||||
requesterSessionKey && artifactId && opts?.resolveArtifactDownload
|
||||
? await opts
|
||||
.resolveArtifactDownload({ sessionKey: requesterSessionKey, artifactId })
|
||||
.catch(() => null)
|
||||
: null;
|
||||
const requestUrl = artifactDownload?.url ?? source;
|
||||
const headers = new Headers({ Accept: "image/*" });
|
||||
if (authToken) {
|
||||
if (!artifactDownload && authToken) {
|
||||
headers.set("Authorization", `Bearer ${authToken}`);
|
||||
}
|
||||
if (requesterSessionKey) {
|
||||
if (!artifactDownload && requesterSessionKey) {
|
||||
headers.set("x-openclaw-requester-session-key", requesterSessionKey);
|
||||
}
|
||||
const controller = new AbortController();
|
||||
@@ -220,7 +235,7 @@ async function resolveManagedOutgoingImageBlobUrl(
|
||||
try {
|
||||
// Managed media is a Gateway API at the origin root. Rebasing it under
|
||||
// the Control UI mount path serves the HTML shell instead of image bytes.
|
||||
const res = await fetch(source, {
|
||||
const res = await fetch(requestUrl, {
|
||||
method: "GET",
|
||||
headers,
|
||||
credentials: "same-origin",
|
||||
|
||||
@@ -17,12 +17,18 @@ const pairingQrExpiryRefreshTimers = new Map<string, PairingQrExpiryRefreshTimer
|
||||
|
||||
export type ImageBlock = {
|
||||
url: string;
|
||||
artifactId?: string;
|
||||
openUrl?: string;
|
||||
alt?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
|
||||
export type ArtifactDownloadResolver = (params: {
|
||||
sessionKey: string;
|
||||
artifactId: string;
|
||||
}) => Promise<{ url: string; expiresAt?: string } | null>;
|
||||
|
||||
export type ImageRenderOptions = {
|
||||
localMediaPreviewRoots?: readonly string[];
|
||||
basePath?: string;
|
||||
@@ -30,6 +36,7 @@ export type ImageRenderOptions = {
|
||||
onRequestUpdate?: () => void;
|
||||
onRequestOpenImage?: () => number;
|
||||
onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void;
|
||||
resolveArtifactDownload?: ArtifactDownloadResolver;
|
||||
};
|
||||
|
||||
export type RenderableImageBlock = ImageBlock & {
|
||||
@@ -209,6 +216,7 @@ export function extractImages(message: unknown): ImageBlock[] {
|
||||
// Handle source object format from optimistic user sends.
|
||||
const source = b.source as Record<string, unknown> | undefined;
|
||||
const imageMeta = {
|
||||
artifactId: typeof b.artifactId === "string" ? b.artifactId : undefined,
|
||||
alt: typeof b.alt === "string" ? b.alt : undefined,
|
||||
openUrl: typeof b.openUrl === "string" ? b.openUrl : undefined,
|
||||
width: typeof b.width === "number" ? b.width : undefined,
|
||||
|
||||
@@ -3638,6 +3638,52 @@ describe("grouped chat rendering", () => {
|
||||
activeItem?.release?.();
|
||||
});
|
||||
|
||||
it("prefers an artifact ticket without forwarding the gateway bearer", async () => {
|
||||
const artifactId = `artifact_managed_image_${crypto.randomUUID()}`;
|
||||
const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
|
||||
const ticketedUrl = `${managedChatImageUrl}?mediaTicket=ticket`;
|
||||
const resolveArtifactDownload = vi.fn(async () => ({
|
||||
url: ticketedUrl,
|
||||
expiresAt: "2026-07-28T05:00:00.000Z",
|
||||
}));
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
expect(url).toBe(ticketedUrl);
|
||||
const headers = new Headers(init?.headers);
|
||||
expect(headers.get("Authorization")).toBeNull();
|
||||
expect(headers.get("x-openclaw-requester-session-key")).toBeNull();
|
||||
return { ok: true, blob: async () => new Blob(["png"], { type: "image/png" }) };
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
|
||||
|
||||
const container = document.createElement("div");
|
||||
renderAssistantMessage(
|
||||
container,
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
artifactId,
|
||||
url: managedChatImageUrl,
|
||||
alt: "Ticketed image",
|
||||
},
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{
|
||||
showToolCalls: false,
|
||||
assistantAttachmentAuthToken: "must-not-be-forwarded",
|
||||
resolveArtifactDownload,
|
||||
},
|
||||
);
|
||||
|
||||
await vi.waitFor(() => expect(container.querySelector(".chat-message-image")).not.toBeNull());
|
||||
expect(resolveArtifactDownload).toHaveBeenCalledWith({
|
||||
sessionKey: "agent:main:main",
|
||||
artifactId,
|
||||
});
|
||||
});
|
||||
|
||||
it("aborts a stalled managed outgoing image fetch after the deadline", async () => {
|
||||
vi.useFakeTimers();
|
||||
const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`;
|
||||
|
||||
@@ -75,6 +75,7 @@ import { getToolTitlesVersion } from "../tool-titles.ts";
|
||||
import { renderBackgroundTasksStatusRow } from "./chat-background-tasks-status.ts";
|
||||
import type { BackgroundTasksProps } from "./chat-background-tasks.ts";
|
||||
import { renderChatDivider, renderChatNotice } from "./chat-divider.ts";
|
||||
import type { ArtifactDownloadResolver } from "./chat-message-media.ts";
|
||||
import {
|
||||
dismissConfirmedActionPopovers,
|
||||
getAssistantAttachmentAvailabilityRenderVersion,
|
||||
@@ -149,6 +150,7 @@ type ChatThreadProps = {
|
||||
fullMessageAgentId?: string;
|
||||
localMediaPreviewRoots?: string[];
|
||||
assistantAttachmentAuthToken?: string | null;
|
||||
resolveArtifactDownload?: ArtifactDownloadResolver;
|
||||
canvasPluginSurfaceUrl?: string | null;
|
||||
embedSandboxMode?: EmbedSandboxMode;
|
||||
allowExternalEmbedUrls?: boolean;
|
||||
@@ -1545,6 +1547,7 @@ function renderChatThreadContents(
|
||||
basePath: props.basePath,
|
||||
localMediaPreviewRoots: props.localMediaPreviewRoots ?? [],
|
||||
assistantAttachmentAuthToken: props.assistantAttachmentAuthToken ?? null,
|
||||
resolveArtifactDownload: props.resolveArtifactDownload,
|
||||
canvasPluginSurfaceUrl: props.canvasPluginSurfaceUrl,
|
||||
embedSandboxMode: props.embedSandboxMode ?? "scripts",
|
||||
allowExternalEmbedUrls: props.allowExternalEmbedUrls ?? false,
|
||||
|
||||
Reference in New Issue
Block a user