feat: add native inline widget support (#109212)

* feat: add native inline widget support

* refactor: simplify plugin surface refresh

* fix: preserve plugin surface refresh API

* fix: reload widgets after WebKit termination

* fix: harden native widget refresh

* fix: bound native widget recovery

* docs: defer native widget release note

* chore: refresh native i18n inventory

* fix: harden native widget delivery

* refactor: simplify native widget API surface

* fix: serialize native widget capability refresh

* fix: harden native widget recovery

* fix: recover native widget capabilities

* style: format widget refresh role selection

* fix: correct widget refresh formatting

* fix: harden cross-platform widget recovery

* fix: bind widget trust and recovery to routes

* chore: refresh native i18n inventory

* chore: regenerate Android gateway protocol

* test: remove unused release workflow read
This commit is contained in:
Peter Steinberger
2026-07-16 19:24:41 -07:00
committed by GitHub
parent 730cfd774d
commit 28a3540f32
105 changed files with 4267 additions and 405 deletions
File diff suppressed because it is too large Load Diff
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "الأداة غير متاحة"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "تم استخدام %@ من الرموز"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "الأداة غير متاحة"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget nicht verfügbar"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ Token verwendet"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget nicht verfügbar"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget no disponible"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ tokens utilizados"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget no disponible"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "ویجت در دسترس نیست"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ توکن استفاده شده است"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "ویجت در دسترس نیست"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget indisponible"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ jetons utilisés"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget indisponible"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "विजेट उपलब्ध नहीं है"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ टोकन उपयोग किए गए"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "विजेट उपलब्ध नहीं है"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget tidak tersedia"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ token telah digunakan"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget tidak tersedia"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget non disponibile"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ token utilizzati"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget non disponibile"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "ウィジェットを利用できません"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@トークンを使用"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "ウィジェットを利用できません"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "위젯을 사용할 수 없음"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@개 토큰 사용됨"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "위젯을 사용할 수 없음"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget niet beschikbaar"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ tokens gebruikt"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget niet beschikbaar"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widżet niedostępny"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "Wykorzystano %@ tokenów"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widżet niedostępny"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget indisponível"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ tokens usados"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget indisponível"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Виджет недоступен"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "Использовано токенов: %@"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Виджет недоступен"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widgeten är inte tillgänglig"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ token används"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widgeten är inte tillgänglig"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "วิดเจ็ตไม่พร้อมใช้งาน"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "ใช้โทเค็นไป %@ รายการ"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "วิดเจ็ตไม่พร้อมใช้งาน"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Widget kullanılamıyor"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "%@ token kullanıldı"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Widget kullanılamıyor"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Віджет недоступний"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "Використано токенів: %@"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Віджет недоступний"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "Tiện ích không khả dụng"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "Đã sử dụng %@ token"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "Tiện ích không khả dụng"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "小组件不可用"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "已使用 %@ 个 token"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "小组件不可用"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
+10
View File
@@ -7113,6 +7113,11 @@
"source": "$text ",
"translated": "$text "
},
{
"id": "native.android.d1712183ebf86197",
"source": "Widget unavailable",
"translated": "小工具無法使用"
},
{
"id": "native.android.cd2033b0a82a72dd",
"source": "$index.",
@@ -22618,6 +22623,11 @@
"source": "%@ tokens used",
"translated": "已使用 %@ 個權杖"
},
{
"id": "native.apple.2c865ac4a69645d3",
"source": "Widget unavailable",
"translated": "小工具無法使用"
},
{
"id": "native.apple.769b7dcea4708c40",
"source": "Image",
@@ -8,6 +8,7 @@ import ai.openclaw.app.chat.ChatPendingToolCall
import ai.openclaw.app.chat.ChatPlanStep
import ai.openclaw.app.chat.ChatSessionEntry
import ai.openclaw.app.chat.ChatThinkingLevelSelection
import ai.openclaw.app.chat.ChatWidgetResource
import ai.openclaw.app.chat.MessageSpeechState
import ai.openclaw.app.chat.OutgoingAttachment
import ai.openclaw.app.chat.defaultChatThinkingLevelSelection
@@ -895,6 +896,11 @@ class MainViewModel private constructor(
fun isTrustedCanvasActionUrl(rawUrl: String?): Boolean = ensureRuntime().isTrustedCanvasActionUrl(rawUrl)
internal suspend fun resolveInlineWidgetResource(
path: String,
failedResource: ChatWidgetResource?,
) = ensureRuntime().resolveInlineWidgetResource(path, failedResource)
fun requestCanvasRehydrate(source: String = "screen_tab") {
ensureRuntime().requestCanvasRehydrate(source = source, force = true)
}
@@ -13,6 +13,10 @@ import ai.openclaw.app.chat.ChatPlanStep
import ai.openclaw.app.chat.ChatSessionEntry
import ai.openclaw.app.chat.ChatThinkingLevelSelection
import ai.openclaw.app.chat.ChatTranscriptCache
import ai.openclaw.app.chat.ChatWidgetResource
import ai.openclaw.app.chat.ChatWidgetSurface
import ai.openclaw.app.chat.ChatWidgetSurfaceUrls
import ai.openclaw.app.chat.ChatWidgetUrlResolver
import ai.openclaw.app.chat.MainSessionBinding
import ai.openclaw.app.chat.MessageSpeechClient
import ai.openclaw.app.chat.MessageSpeechController
@@ -105,6 +109,7 @@ import android.os.SystemClock
import android.util.Base64
import android.util.Log
import androidx.core.content.ContextCompat
import androidx.webkit.WebViewFeature
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
@@ -467,6 +472,7 @@ class NodeRuntime private constructor(
@Volatile private var connectingEndpointStableId: String? = null
private val gatewayDataScopeLock = Any()
private val gatewaySwitchMutex = Mutex()
private val inlineWidgetRefreshMutex = Mutex()
private val gatewayLifecycleIntentLock = Any()
private val gatewayLifecycleIntentSeq = AtomicLong()
private var gatewayDataGeneration = 0L
@@ -720,6 +726,7 @@ class NodeRuntime private constructor(
hasRecordAudioPermission() &&
isVoiceWakeWordsReadyForCurrentGateway()
},
inlineWidgetsAvailable = { WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE) },
manualTls = { endpoint ->
prefs.gatewayRegistry.entries.value
.firstOrNull { it.stableId == endpoint.stableId }
@@ -4106,6 +4113,52 @@ class NodeRuntime private constructor(
fun isTrustedCanvasActionUrl(rawUrl: String?): Boolean = a2uiHandler.isTrustedCanvasActionUrl(rawUrl)
internal suspend fun resolveInlineWidgetResource(
path: String,
failedResource: ChatWidgetResource?,
): ChatWidgetResource? {
fun GatewaySession.currentWidgetSurface(): ChatWidgetSurface? =
currentCanvasHostRoute()?.let { route ->
ChatWidgetSurface(
url = route.url,
tlsFingerprintSha256 = route.tlsFingerprintSha256,
)
}
fun currentSurfaceUrls(): ChatWidgetSurfaceUrls =
ChatWidgetSurfaceUrls(
node = nodeSession.currentWidgetSurface(),
operator = operatorSession.currentWidgetSurface(),
)
// Initial loads may use the operator fallback; failures rotate the preferred live route.
if (failedResource == null) return ChatWidgetUrlResolver.resolvePreferred(currentSurfaceUrls(), path, excluding = null)
return inlineWidgetRefreshMutex.withLock {
// Serialize both role sessions so sibling widgets cannot invalidate each other's new token.
ChatWidgetUrlResolver.resolveAfterFailure(
target = path,
failedResource = failedResource,
currentSurfaceUrls = ::currentSurfaceUrls,
refreshNodeSurface = { observedUrl ->
nodeSession.refreshCanvasHostRouteIfCurrent(observedUrl)?.let { route ->
ChatWidgetSurface(
url = route.url,
tlsFingerprintSha256 = route.tlsFingerprintSha256,
)
}
},
refreshOperatorSurface = { observedUrl ->
operatorSession.refreshCanvasHostRouteIfCurrent(observedUrl)?.let { route ->
ChatWidgetSurface(
url = route.url,
tlsFingerprintSha256 = route.tlsFingerprintSha256,
)
}
},
)
}
}
fun loadChat(sessionKey: String) {
val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() }
chat.load(key)
@@ -3787,6 +3787,30 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
)
}
"canvas" -> {
val preview = obj["preview"].asObjectOrNull() ?: return null
val sandbox = preview["sandbox"].asStringOrNull() ?: return null
if (preview["kind"].asStringOrNull() != "canvas" ||
preview["surface"].asStringOrNull() != "assistant_message" ||
preview["render"].asStringOrNull() != "url" ||
(sandbox != "scripts" && sandbox != "strict")
) {
return null
}
val path = preview["url"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: return null
if (!ChatWidgetUrlResolver.supportsTarget(path)) return null
ChatMessageContent(
type = "canvas",
widget =
ChatWidgetPreview(
title = preview["title"].asStringOrNull(),
path = path,
preferredHeight = preview["preferredHeight"].asLongOrNull()?.coerceIn(160, 1200)?.toInt(),
sandbox = sandbox,
),
)
}
else -> null
}
}
@@ -36,8 +36,19 @@ data class ChatMessageContent(
val fileName: String? = null,
val base64: String? = null,
val durationMs: Long? = null,
val widget: ChatWidgetPreview? = null,
)
data class ChatWidgetPreview(
val title: String?,
val path: String,
val preferredHeight: Int?,
val sandbox: String,
) {
val height: Int
get() = (preferredHeight ?: 320).coerceIn(160, 1200)
}
/**
* Tool call placeholder shown while a gateway run is still streaming.
*/
@@ -0,0 +1,193 @@
package ai.openclaw.app.chat
import java.net.URI
import java.net.URLDecoder
internal object ChatWidgetUrlResolver {
private const val DOCUMENTS_PATH = "/__openclaw__/canvas/documents"
fun resolve(
surfaceUrl: String?,
target: String,
): String? {
val surface = parseCapabilitySurface(surfaceUrl) ?: return null
val relative = parseRelativeTarget(target) ?: return null
val joined =
buildString {
append(surface.scheme.lowercase())
append("://")
append(surface.rawAuthority)
append(surface.rawPath.trimEnd('/'))
append(relative.rawPath)
relative.rawQuery?.let { append('?').append(it) }
relative.rawFragment?.let { append('#').append(it) }
}
return runCatching { URI(joined) }.getOrNull()?.toASCIIString()
}
fun supportsTarget(target: String): Boolean = parseRelativeTarget(target) != null
private fun resolve(
surface: ChatWidgetSurface,
target: String,
role: ChatWidgetSurfaceRole,
attemptedRoles: Set<ChatWidgetSurfaceRole>,
): ChatWidgetResource? =
resolve(surface.url, target)?.let { url ->
ChatWidgetResource(
url = url,
tlsFingerprintSha256 = surface.tlsFingerprintSha256,
surfaceRole = role,
attemptedSurfaceRoles = attemptedRoles,
)
}
fun resolvePreferred(
surfaces: ChatWidgetSurfaceUrls,
target: String,
excluding: ChatWidgetResource?,
blockedRoles: Set<ChatWidgetSurfaceRole> = emptySet(),
attemptedRoles: Set<ChatWidgetSurfaceRole> = emptySet(),
): ChatWidgetResource? =
sequenceOf(
ChatWidgetSurfaceRole.NODE to surfaces.node,
ChatWidgetSurfaceRole.OPERATOR to surfaces.operator,
).filter { (role) -> role !in blockedRoles }
.mapNotNull { (role, surface) -> surface?.let { resolve(it, target, role, attemptedRoles) } }
.firstOrNull { isReplacement(it, excluding) }
suspend fun resolveAfterFailure(
target: String,
failedResource: ChatWidgetResource,
currentSurfaceUrls: () -> ChatWidgetSurfaceUrls,
refreshNodeSurface: suspend (String?) -> ChatWidgetSurface?,
refreshOperatorSurface: suspend (String?) -> ChatWidgetSurface?,
): ChatWidgetResource? {
val observed = currentSurfaceUrls()
val blockedRoles = failedResource.attemptedSurfaceRoles
if (failedResource.surfaceRole == ChatWidgetSurfaceRole.LEGACY && ChatWidgetSurfaceRole.LEGACY in blockedRoles) {
return null
}
val attemptedRoles = blockedRoles + failedResource.surfaceRole
if (ChatWidgetSurfaceRole.NODE !in blockedRoles) {
observed.node
?.let { resolve(it, target, ChatWidgetSurfaceRole.NODE, attemptedRoles) }
?.takeIf { isReplacement(it, failedResource) }
?.let { return it }
val refreshed =
refreshNodeSurface(observed.node?.url)?.let {
resolve(it, target, ChatWidgetSurfaceRole.NODE, attemptedRoles)
}
if (refreshed != null && isReplacement(refreshed, failedResource)) return refreshed
}
// A nil refresh can mean its route lease lost a reconnect race. Re-read
// both roles so a replacement connection wins over the stale observation.
val afterNodeRefresh = currentSurfaceUrls()
resolvePreferred(
afterNodeRefresh,
target,
excluding = failedResource,
blockedRoles = blockedRoles,
attemptedRoles = attemptedRoles,
)?.let { return it }
if (ChatWidgetSurfaceRole.OPERATOR !in blockedRoles) {
val refreshedOperator =
refreshOperatorSurface(afterNodeRefresh.operator?.url)?.let {
resolve(it, target, ChatWidgetSurfaceRole.OPERATOR, attemptedRoles)
}
if (refreshedOperator != null && isReplacement(refreshedOperator, failedResource)) return refreshedOperator
}
return resolvePreferred(
currentSurfaceUrls(),
target,
excluding = failedResource,
blockedRoles = blockedRoles,
attemptedRoles = attemptedRoles,
)
}
private fun isReplacement(
candidate: ChatWidgetResource,
failedResource: ChatWidgetResource?,
): Boolean {
if (failedResource == null) return true
return if (
failedResource.surfaceRole == ChatWidgetSurfaceRole.LEGACY &&
failedResource.tlsFingerprintSha256 == null
) {
candidate.url != failedResource.url
} else {
candidate.url != failedResource.url ||
candidate.tlsFingerprintSha256 != failedResource.tlsFingerprintSha256
}
}
private fun parseCapabilitySurface(raw: String?): URI? {
val parsed = raw?.trim()?.takeIf(String::isNotEmpty)?.let { runCatching { URI(it) }.getOrNull() } ?: return null
val scheme = parsed.scheme?.lowercase()
if (scheme != "http" && scheme != "https") return null
if (parsed.host.isNullOrBlank() || parsed.rawUserInfo != null || parsed.rawQuery != null || parsed.rawFragment != null) return null
val segments = parsed.rawPath.split('/').filter(String::isNotEmpty)
if (segments.size < 3 || segments[segments.lastIndex - 2] != "__openclaw__" || segments[segments.lastIndex - 1] != "cap") return null
if (decodeRepeatedly(segments.last())?.isEmpty() != false) return null
return parsed
}
private fun parseRelativeTarget(raw: String): URI? {
val target = raw.trim()
if (!target.startsWith('/')) return null
val parsed = runCatching { URI(target) }.getOrNull() ?: return null
if (parsed.isAbsolute || parsed.rawAuthority != null || !isCanonicalPath(parsed.rawPath)) return null
if (!parsed.rawPath.startsWith("$DOCUMENTS_PATH/")) return null
return parsed
}
private fun isCanonicalPath(path: String): Boolean {
val segments = path.split('/')
if (segments.firstOrNull()?.isNotEmpty() == true) return false
return segments.drop(1).all { encoded ->
if (encoded.isEmpty()) return@all false
val decoded = decodeRepeatedly(encoded) ?: return@all false
decoded != "." && decoded != ".." && !decoded.contains('/') && !decoded.contains('\\')
}
}
private fun decodeRepeatedly(raw: String): String? {
var value = raw
repeat(8) {
val decoded =
runCatching {
URLDecoder.decode(value.replace("+", "%2B"), Charsets.UTF_8.name())
}.getOrNull() ?: return null
if (decoded == value) return decoded
value = decoded
}
return null
}
}
internal data class ChatWidgetSurfaceUrls(
val node: ChatWidgetSurface?,
val operator: ChatWidgetSurface?,
)
internal data class ChatWidgetSurface(
val url: String,
val tlsFingerprintSha256: String?,
)
internal enum class ChatWidgetSurfaceRole {
NODE,
OPERATOR,
LEGACY,
}
internal data class ChatWidgetResource(
val url: String,
val tlsFingerprintSha256: String?,
val surfaceRole: ChatWidgetSurfaceRole = ChatWidgetSurfaceRole.LEGACY,
val attemptedSurfaceRoles: Set<ChatWidgetSurfaceRole> = emptySet(),
)
@@ -379,6 +379,7 @@ enum class GatewayMethod(
MigrationsMemoryApply("migrations.memory.apply"),
UiCommand("ui.command"),
ApprovalHistory("approval.history"),
PluginSurfaceRefresh("plugin.surface.refresh"),
}
enum class GatewayEvent(
@@ -160,6 +160,11 @@ internal enum class NodeEventSendOutcome {
FAILED,
}
internal data class GatewayCanvasHostRoute(
val url: String,
val tlsFingerprintSha256: String?,
)
/**
* WebSocket RPC session that maintains gateway connection lifecycle, auth, events, and node invokes.
*/
@@ -224,8 +229,19 @@ class GatewaySession(
/** One ready physical WebSocket captured before queued work starts waiting. */
internal class RequestLease internal constructor(
val endpointStableId: String,
private val isCurrentImpl: () -> Boolean = { true },
private val commitIfCurrentImpl: ((block: () -> Unit) -> Boolean)? = null,
private val requestImpl: suspend (method: String, paramsJson: String?, timeoutMs: Long) -> String,
) {
fun isCurrent(): Boolean = isCurrentImpl()
fun commitIfCurrent(block: () -> Unit): Boolean {
commitIfCurrentImpl?.let { return it(block) }
if (!isCurrentImpl()) return false
block()
return true
}
suspend fun request(
method: String,
paramsJson: String?,
@@ -377,6 +393,101 @@ class GatewaySession(
internal fun isReady(): Boolean = readyConnection() != null
internal fun currentCanvasHostUrl(): String? = pluginSurfaceUrls["canvas"]
internal fun currentCanvasHostRoute(): GatewayCanvasHostRoute? =
synchronized(lifecycleLock) {
val connection = readyConnection() ?: return@synchronized null
val url = pluginSurfaceUrls["canvas"] ?: return@synchronized null
// TOFU can learn the certificate during the WebSocket handshake. Read
// the live TLS config so the widget document inherits that exact trust.
val fingerprint =
connection.tlsConfig
?.effectiveFingerprintSha256
?.let(::normalizeGatewayTlsFingerprint)
?.takeIf { it.length == 64 }
GatewayCanvasHostRoute(
url = url,
tlsFingerprintSha256 =
gatewayTlsFingerprintForCanvasSurface(
fingerprint = fingerprint,
surfaceUrl = url,
endpoint = connection.endpoint,
isTlsConnection = connection.tlsConfig != null,
),
)
}
internal suspend fun refreshCanvasHostUrl(): String? = refreshCanvasHostUrl(observedSurfaceUrl = null, requireObservedMatch = false)
internal suspend fun refreshCanvasHostUrlIfCurrent(observedSurfaceUrl: String?): String? = refreshCanvasHostUrl(observedSurfaceUrl = observedSurfaceUrl, requireObservedMatch = true)
internal suspend fun refreshCanvasHostRouteIfCurrent(observedSurfaceUrl: String?): GatewayCanvasHostRoute? {
refreshCanvasHostUrlIfCurrent(observedSurfaceUrl)
// Pair the URL with the currently installed connection after suspension;
// a reconnect can replace both the capability and its certificate pin.
return currentCanvasHostRoute()
}
private suspend fun refreshCanvasHostUrl(
observedSurfaceUrl: String?,
requireObservedMatch: Boolean,
): String? {
val (lease, target, requestObservedSurfaceUrl) =
synchronized(lifecycleLock) {
val current = pluginSurfaceUrls["canvas"]
if (requireObservedMatch && current != observedSurfaceUrl) return current
val capturedLease = captureRequestLease() ?: return null
val capturedTarget =
desired
?.takeIf { it.endpoint.stableId == capturedLease.endpointStableId }
?: return null
Triple(capturedLease, capturedTarget, current)
}
val refreshMethod =
when (
target.options.role
.trim()
.lowercase(Locale.ROOT)
) {
"node" -> GatewayMethod.NodePluginSurfaceRefresh
"operator" -> GatewayMethod.PluginSurfaceRefresh
else -> return null
}
val response =
runCatching {
lease.request(
refreshMethod.rawValue,
buildJsonObject {
put("surface", JsonPrimitive("canvas"))
requestObservedSurfaceUrl?.let { put("observedUrl", JsonPrimitive(it)) }
}.toString(),
timeoutMs = 8_000,
)
}.getOrNull() ?: return null
val raw =
parseJsonOrNull(response)
.asObjectOrNull()
?.get("pluginSurfaceUrls")
.asObjectOrNull()
?.get("canvas")
.asStringOrNull()
val refreshed = normalizeCanvasHostUrl(raw, target.endpoint, isTlsConnection = target.tls != null) ?: return null
var result: String? = null
val committed =
lease.commitIfCurrent {
val current = pluginSurfaceUrls["canvas"]
result =
if (requireObservedMatch && current != observedSurfaceUrl) {
current
} else {
pluginSurfaceUrls = pluginSurfaceUrls + ("canvas" to refreshed)
refreshed
}
}
return if (committed) result else null
}
/** Current physical connection identity, including events sent during connect publication. */
internal fun currentEndpointStableId(): String? = currentConnection?.endpoint?.stableId
@@ -490,16 +601,30 @@ class GatewaySession(
}
/** Captures the current physical connection; requests never resolve a replacement socket. */
internal fun captureRequestLease(expectedEndpointStableId: String? = null): RequestLease? {
val conn = readyConnection(expectedEndpointStableId) ?: return null
return RequestLease(endpointStableId = conn.endpoint.stableId) { method, paramsJson, timeoutMs ->
val res = requestDetailed(conn, method, paramsJson, timeoutMs)
if (!res.ok) {
throw GatewayRequestRejected(res.error ?: ErrorShape("UNAVAILABLE", "request failed"))
internal fun captureRequestLease(expectedEndpointStableId: String? = null): RequestLease? =
synchronized(lifecycleLock) {
val conn = readyConnection(expectedEndpointStableId) ?: return@synchronized null
RequestLease(
endpointStableId = conn.endpoint.stableId,
isCurrentImpl = { currentConnection === conn && conn.isReady() },
commitIfCurrentImpl = { block ->
synchronized(lifecycleLock) {
if (currentConnection !== conn || !conn.isReady()) {
false
} else {
block()
true
}
}
},
) { method, paramsJson, timeoutMs ->
val res = requestDetailed(conn, method, paramsJson, timeoutMs)
if (!res.ok) {
throw GatewayRequestRejected(res.error ?: ErrorShape("UNAVAILABLE", "request failed"))
}
res.payloadJson ?: ""
}
res.payloadJson ?: ""
}
}
/** Sends an RPC request and returns the structured success/error payload. */
suspend fun requestDetailed(
@@ -616,6 +741,10 @@ class GatewaySession(
@Volatile
private var connectRequestId: String? = null
val tlsConfig: GatewayTlsConfig? =
buildGatewayTlsConfig(tls) { fingerprint ->
onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint)
}
private val client: OkHttpClient = buildClient()
private val listener = Listener()
private var socket: WebSocket? = null
@@ -820,10 +949,6 @@ class GatewaySession(
.writeTimeout(60, java.util.concurrent.TimeUnit.SECONDS)
.readTimeout(0, java.util.concurrent.TimeUnit.SECONDS)
.pingInterval(30, java.util.concurrent.TimeUnit.SECONDS)
val tlsConfig =
buildGatewayTlsConfig(tls) { fingerprint ->
onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint)
}
if (tlsConfig != null) {
builder.sslSocketFactory(tlsConfig.sslSocketFactory, tlsConfig.trustManager)
builder.hostnameVerifier(tlsConfig.hostnameVerifier)
@@ -1701,6 +1826,23 @@ internal fun shouldPauseGatewayReconnectAfterAuthFailure(
}
/** Builds the gateway WebSocket URL from endpoint authority and TLS policy. */
internal fun gatewayTlsFingerprintForCanvasSurface(
fingerprint: String?,
surfaceUrl: String,
endpoint: GatewayEndpoint,
isTlsConnection: Boolean,
): String? {
if (!isTlsConnection || fingerprint == null) return null
val surface = runCatching { java.net.URI(surfaceUrl) }.getOrNull() ?: return null
if (!surface.scheme.equals("https", ignoreCase = true)) return null
val surfaceHost = surface.host?.trim()?.trimEnd('.') ?: return null
val gatewayHost = endpoint.host.trim().trimEnd('.')
val surfacePort = surface.port.takeIf { it > 0 } ?: 443
return fingerprint.takeIf {
surfaceHost.equals(gatewayHost, ignoreCase = true) && surfacePort == endpoint.port
}
}
internal fun buildGatewayWebSocketUrl(
host: String,
port: Int,
@@ -37,12 +37,16 @@ data class GatewayTlsParams(
val stableId: String,
)
/** SSL primitives installed into OkHttp when a gateway needs TLS pinning/TOFU. */
data class GatewayTlsConfig(
/** SSL primitives and accepted route trust installed into OkHttp. */
class GatewayTlsConfig internal constructor(
val sslSocketFactory: SSLSocketFactory,
val trustManager: X509TrustManager,
val hostnameVerifier: HostnameVerifier,
)
private val effectiveFingerprint: AtomicReference<String?>,
) {
val effectiveFingerprintSha256: String?
get() = effectiveFingerprint.get()
}
/** Distinguishes non-TLS endpoints from unreachable endpoints during probing. */
enum class GatewayTlsProbeFailure {
@@ -82,8 +86,14 @@ internal fun buildGatewayTlsConfig(
params.expectedFingerprint
?.let(::normalizeGatewayTlsFingerprint)
?.takeIf { it.isNotBlank() }
val effectiveFingerprint = AtomicReference(expected)
val usesPlatformTrust = expected == null && !params.allowTOFU
fun recordAcceptedFingerprint(chain: Array<X509Certificate>) {
val certificate = chain.firstOrNull() ?: return
effectiveFingerprint.set(sha256Hex(certificate.encoded))
}
@SuppressLint("CustomX509TrustManager")
val trustManager =
object : X509ExtendedTrustManager() {
@@ -128,6 +138,7 @@ internal fun buildGatewayTlsConfig(
if (fingerprint != expected) {
throw CertificateException("gateway TLS fingerprint mismatch")
}
effectiveFingerprint.set(fingerprint)
return
}
if (params.allowTOFU) {
@@ -135,9 +146,11 @@ internal fun buildGatewayTlsConfig(
// caller persists the fingerprint against the endpoint's stable id,
// and later connects must come back through the pinned branch above.
onStore?.invoke(fingerprint)
effectiveFingerprint.set(fingerprint)
return
}
defaultTrust.checkServerTrusted(chain, authType)
effectiveFingerprint.set(fingerprint)
}
override fun checkServerTrusted(
@@ -148,6 +161,7 @@ internal fun buildGatewayTlsConfig(
if (usesPlatformTrust && defaultTrust is X509ExtendedTrustManager) {
// Preserve the connected hostname for Android's domain-aware platform trust manager.
defaultTrust.checkServerTrusted(chain, authType, socket)
recordAcceptedFingerprint(chain)
} else {
checkServerTrusted(chain, authType)
}
@@ -160,6 +174,7 @@ internal fun buildGatewayTlsConfig(
) {
if (usesPlatformTrust && defaultTrust is X509ExtendedTrustManager) {
defaultTrust.checkServerTrusted(chain, authType, engine)
recordAcceptedFingerprint(chain)
} else {
checkServerTrusted(chain, authType)
}
@@ -181,6 +196,7 @@ internal fun buildGatewayTlsConfig(
sslSocketFactory = context.socketFactory,
trustManager = trustManager,
hostnameVerifier = verifier,
effectiveFingerprint = effectiveFingerprint,
)
}
@@ -1327,6 +1327,7 @@ internal val nativeStringResourceIds: Map<String, Int> =
"When connected, the gateway can wake the phone with a silent push instead of holding an always-on session." to R.string.native_1dbf9d5979606697,
"Where do I get a setup code?" to R.string.native_c34dd5eb8b0af56e,
"While Using" to R.string.native_44173ba2044fb22f,
"Widget unavailable" to R.string.native_41659b48f2dc4c36,
"Working" to R.string.native_a92f0449a9f7235b,
"Working directory" to R.string.native_865e85c6fbf9d3b2,
"Working · \$pendingRunCount active runs" to R.string.native_75c4bbf668b72cbf,
@@ -27,6 +27,7 @@ class ConnectionManager(
private val photosAvailable: () -> Boolean,
private val installedAppsSharingEnabled: () -> Boolean,
private val voiceWakeAvailable: () -> Boolean,
private val inlineWidgetsAvailable: () -> Boolean,
private val manualTls: (GatewayEndpoint) -> Boolean,
) {
companion object {
@@ -48,6 +49,8 @@ class ConnectionManager(
"operator.write",
)
internal const val INLINE_WIDGETS_CLIENT_CAPABILITY = "inline-widgets"
internal fun operatorScopesForStoredDeviceToken(storedScopes: List<String>): List<String> {
val normalized =
storedScopes
@@ -218,7 +221,7 @@ class ConnectionManager(
GatewayConnectOptions(
role = "operator",
scopes = scopes,
caps = emptyList(),
caps = if (inlineWidgetsAvailable()) listOf(INLINE_WIDGETS_CLIENT_CAPABILITY) else emptyList(),
commands = emptyList(),
permissions = emptyMap(),
client = buildClientInfo(clientId = "openclaw-android", clientMode = "ui"),
@@ -0,0 +1,459 @@
package ai.openclaw.app.ui.chat
import ai.openclaw.app.chat.ChatWidgetPreview
import ai.openclaw.app.chat.ChatWidgetResource
import ai.openclaw.app.chat.ChatWidgetSurfaceRole
import ai.openclaw.app.gateway.GatewayTlsParams
import ai.openclaw.app.gateway.buildGatewayTlsConfig
import ai.openclaw.app.gateway.normalizeGatewayTlsFingerprint
import ai.openclaw.app.i18n.nativeString
import ai.openclaw.app.ui.design.ClawTheme
import android.annotation.SuppressLint
import android.os.Handler
import android.os.Looper
import android.view.ViewGroup
import android.webkit.RenderProcessGoneDetail
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebSettings
import android.webkit.WebView
import android.webkit.WebViewClient
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.key
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.webkit.ProfileStore
import androidx.webkit.WebViewCompat
import androidx.webkit.WebViewFeature
import kotlinx.coroutines.launch
import okhttp3.OkHttpClient
import okhttp3.Request
import okio.BufferedSource
import java.io.ByteArrayInputStream
import java.net.URI
import java.util.Locale
import java.util.UUID
import java.util.concurrent.TimeUnit
private const val INLINE_WIDGET_PROFILE_PREFIX = "openclaw-inline-widget-"
private const val INLINE_WIDGET_DOCUMENT_MAX_BYTES = 2L * 1024 * 1024
private const val INLINE_WIDGET_FETCH_TIMEOUT_SECONDS = 8L
private const val HTTP_HEADER_ACCEPT = "Accept"
private const val HTTP_HEADER_CACHE_CONTROL = "Cache-Control"
@Composable
internal fun ChatInlineWidget(
preview: ChatWidgetPreview,
resolverReady: Boolean,
resolveResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
) {
var resolvedResource by remember(preview.path) { mutableStateOf<ChatWidgetResource?>(null) }
var unavailable by remember(preview.path) { mutableStateOf(false) }
var recoveryAttempts by remember(preview.path) { mutableStateOf(0) }
var refreshInFlight by remember(preview.path) { mutableStateOf(false) }
var refreshRequestId by remember(preview.path) { mutableStateOf<UUID?>(null) }
val scope = rememberCoroutineScope()
val isolatedProfileSupported = remember { WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE) }
fun handleFailure(
resource: ChatWidgetResource,
rendererGone: Boolean,
) {
if (resolvedResource != resource) return
if (rendererGone) {
// onRenderProcessGone destroys the unusable WebView immediately. Drop
// Compose's resource reference even when its replacement is in flight.
resolvedResource = null
unavailable = false
}
if (refreshInFlight) return
if (recoveryAttempts >= ChatWidgetSurfaceRole.entries.size) {
refreshRequestId = null
resolvedResource = null
unavailable = true
return
}
recoveryAttempts += 1
refreshInFlight = true
val requestId = UUID.randomUUID()
refreshRequestId = requestId
scope.launch {
val replacement = resolveResource(preview.path, resource)
if (refreshRequestId != requestId) return@launch
refreshRequestId = null
resolvedResource = replacement
unavailable = replacement == null
refreshInFlight = false
}
}
LaunchedEffect(preview.path, resolverReady) {
refreshRequestId = null
refreshInFlight = false
if (!resolverReady) return@LaunchedEffect
resolvedResource = resolveResource(preview.path, null)
unavailable = resolvedResource == null
recoveryAttempts = 0
}
Column(modifier = Modifier.fillMaxWidth()) {
preview.title?.trim()?.takeIf(String::isNotEmpty)?.let { title ->
Text(
text = title,
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
modifier = Modifier.padding(bottom = 6.dp),
)
}
when {
resolvedResource != null && isolatedProfileSupported -> {
val resource = checkNotNull(resolvedResource)
val allowsScripts = preview.sandbox == "scripts"
// Resource identity owns the WebView, isolated profile, and cleanup handle together.
key(resource, allowsScripts) {
Surface(
modifier = Modifier.fillMaxWidth().height(preview.height.dp),
shape = RoundedCornerShape(10.dp),
border = BorderStroke(1.dp, ClawTheme.colors.border),
color = ClawTheme.colors.surface,
) {
InlineWidgetWebView(
resource = resource,
allowsScripts = allowsScripts,
onFailure = { handleFailure(resource, rendererGone = false) },
onRendererGone = { handleFailure(resource, rendererGone = true) },
)
}
}
}
unavailable || resolvedResource != null ->
Text(
text = nativeString("Widget unavailable"),
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
)
else ->
Box(modifier = Modifier.fillMaxWidth().height(44.dp), contentAlignment = Alignment.Center) {
CircularProgressIndicator(color = ClawTheme.colors.textMuted)
}
}
}
}
@SuppressLint("SetJavaScriptEnabled")
@Suppress("DEPRECATION")
@Composable
private fun InlineWidgetWebView(
resource: ChatWidgetResource,
allowsScripts: Boolean,
onFailure: () -> Unit,
onRendererGone: () -> Unit,
) {
val profileName = remember(resource, allowsScripts) { "$INLINE_WIDGET_PROFILE_PREFIX${UUID.randomUUID()}" }
val handle = remember(profileName) { InlineWidgetWebViewHandle() }
AndroidView(
modifier = Modifier.fillMaxWidth(),
factory = { context ->
pruneStaleInlineWidgetProfiles()
val webView = WebView(context)
if (WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) {
WebViewCompat.setProfile(webView, profileName)
} else {
error("isolated WebView profiles are unavailable")
}
val client =
InlineWidgetWebViewClient(
resource = resource,
onFailure = onFailure,
onRendererGone = onRendererGone,
)
handle.bind(client)
webView.apply {
settings.setAllowContentAccess(false)
settings.setAllowFileAccess(false)
settings.setAllowFileAccessFromFileURLs(false)
settings.setAllowUniversalAccessFromFileURLs(false)
settings.setSafeBrowsingEnabled(true)
settings.javaScriptEnabled = allowsScripts
settings.domStorageEnabled = false
settings.cacheMode = WebSettings.LOAD_NO_CACHE
settings.mixedContentMode = WebSettings.MIXED_CONTENT_NEVER_ALLOW
settings.javaScriptCanOpenWindowsAutomatically = false
settings.setSupportMultipleWindows(false)
isHorizontalScrollBarEnabled = false
webViewClient = client
loadUrl(resource.url)
}
},
onRelease = { webView ->
handle.release(webView)
deleteInlineWidgetProfile(profileName)
},
)
}
private class InlineWidgetWebViewHandle {
private var client: InlineWidgetWebViewClient? = null
fun bind(client: InlineWidgetWebViewClient) {
this.client = client
}
fun release(view: WebView) {
client?.release(view)
client = null
}
}
private fun pruneStaleInlineWidgetProfiles() {
if (!WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) return
val store = ProfileStore.getInstance()
store.allProfileNames
.asSequence()
.filter { it.startsWith(INLINE_WIDGET_PROFILE_PREFIX) }
.forEach { profileName -> runCatching { store.deleteProfile(profileName) } }
}
private fun deleteInlineWidgetProfile(profileName: String) {
if (!WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) return
val store = ProfileStore.getInstance()
val deleted = runCatching { store.deleteProfile(profileName) }.getOrDefault(false)
if (!deleted) {
Handler(Looper.getMainLooper()).post {
if (WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) {
runCatching { ProfileStore.getInstance().deleteProfile(profileName) }
}
}
}
}
private class InlineWidgetWebViewClient(
private val resource: ChatWidgetResource,
private val onFailure: () -> Unit,
private val onRendererGone: () -> Unit,
) : WebViewClient() {
private val pinnedClient = resource.tlsFingerprintSha256?.let(::buildPinnedWidgetClient)
private var released = false
fun release(view: WebView) {
if (released) return
released = true
view.stopLoading()
closePinnedClient()
view.webViewClient = WebViewClient()
view.removeAllViews()
view.destroy()
}
private fun releaseAfterRendererGone(view: WebView): Boolean {
if (released) return false
released = true
// A renderer-less WebView is unusable. Remove and destroy it before
// starting asynchronous route recovery; onRelease becomes a no-op.
(view.parent as? ViewGroup)?.removeView(view)
closePinnedClient()
view.destroy()
return true
}
private fun closePinnedClient() {
pinnedClient?.dispatcher?.cancelAll()
pinnedClient?.connectionPool?.evictAll()
}
override fun onPageCommitVisible(
view: WebView,
url: String,
) {
// Compose can retain the pre-navigation WebView layer until the first damage event.
// Invalidate once committed so the initial widget frame paints without user input.
view.postInvalidateOnAnimation()
}
override fun shouldOverrideUrlLoading(
view: WebView,
request: WebResourceRequest,
): Boolean =
request.isForMainFrame &&
(!request.method.equals("GET", ignoreCase = true) || !sameDocument(resource.url, request.url.toString()))
override fun shouldInterceptRequest(
view: WebView,
request: WebResourceRequest,
): WebResourceResponse? {
val scheme = request.url.scheme?.lowercase()
if (scheme != "http" && scheme != "https") return null
val allowed =
request.isForMainFrame &&
request.method.equals("GET", ignoreCase = true) &&
sameDocument(resource.url, request.url.toString())
if (!allowed) return blockedWidgetResponse()
if (resource.tlsFingerprintSha256 == null) return null
if (scheme != "https" || pinnedClient == null) return failedWidgetResponse()
return fetchPinnedWidgetDocument(client = pinnedClient, url = request.url.toString())
}
override fun onReceivedError(
view: WebView,
request: WebResourceRequest,
error: WebResourceError,
) {
if (request.isForMainFrame) onFailure()
}
override fun onReceivedHttpError(
view: WebView,
request: WebResourceRequest,
errorResponse: WebResourceResponse,
) {
if (request.isForMainFrame && errorResponse.statusCode >= 400) onFailure()
}
override fun onRenderProcessGone(
view: WebView,
detail: RenderProcessGoneDetail,
): Boolean {
if (releaseAfterRendererGone(view)) onRendererGone()
return true
}
}
private fun buildPinnedWidgetClient(rawFingerprint: String): OkHttpClient? {
val fingerprint = normalizeGatewayTlsFingerprint(rawFingerprint)
if (fingerprint.length != 64) return null
val tls =
buildGatewayTlsConfig(
GatewayTlsParams(
required = true,
expectedFingerprint = fingerprint,
allowTOFU = false,
stableId = "inline-widget",
),
) ?: return null
return OkHttpClient
.Builder()
.sslSocketFactory(tls.sslSocketFactory, tls.trustManager)
.hostnameVerifier(tls.hostnameVerifier)
.followRedirects(false)
.followSslRedirects(false)
.retryOnConnectionFailure(false)
.cache(null)
.callTimeout(INLINE_WIDGET_FETCH_TIMEOUT_SECONDS, TimeUnit.SECONDS)
.build()
}
private fun fetchPinnedWidgetDocument(
client: OkHttpClient,
url: String,
): WebResourceResponse =
try {
val request =
Request
.Builder()
.url(url)
.header(HTTP_HEADER_ACCEPT, "text/html")
.header(HTTP_HEADER_CACHE_CONTROL, "no-cache")
.get()
.build()
client.newCall(request).execute().use { response ->
if (!response.isSuccessful) return failedWidgetResponse()
val body = response.body
val contentType = body.contentType() ?: return failedWidgetResponse()
val mimeType = "${contentType.type}/${contentType.subtype}".lowercase(Locale.US)
if (mimeType != "text/html") return failedWidgetResponse()
val contentLength = body.contentLength()
if (contentLength > INLINE_WIDGET_DOCUMENT_MAX_BYTES) return failedWidgetResponse()
val bytes =
readBoundedWidgetDocument(
source = body.source(),
maxBytes = INLINE_WIDGET_DOCUMENT_MAX_BYTES.toInt(),
) ?: return failedWidgetResponse()
val responseHeaders =
listOf(
"Cache-Control",
"Content-Security-Policy",
"Permissions-Policy",
"Referrer-Policy",
"X-Content-Type-Options",
).mapNotNull { name -> response.header(name)?.let { name to it } }.toMap()
WebResourceResponse(
mimeType,
contentType.charset(Charsets.UTF_8)?.name() ?: Charsets.UTF_8.name(),
response.code,
response.message.ifBlank { "OK" },
responseHeaders,
ByteArrayInputStream(bytes),
)
}
} catch (_: Exception) {
failedWidgetResponse()
}
internal fun readBoundedWidgetDocument(
source: BufferedSource,
maxBytes: Int,
): ByteArray? {
require(maxBytes in 0 until Int.MAX_VALUE)
val buffer = ByteArray(maxBytes + 1)
var offset = 0
while (offset < buffer.size) {
val read = source.read(buffer, offset, buffer.size - offset)
if (read == -1) break
if (read == 0) return null
offset += read
}
if (offset > maxBytes) return null
return buffer.copyOf(offset)
}
private fun sameDocument(
expected: String,
candidate: String,
): Boolean {
val expectedUri = runCatching { URI(expected) }.getOrNull() ?: return false
val candidateUri = runCatching { URI(candidate) }.getOrNull() ?: return false
return expectedUri.scheme == candidateUri.scheme &&
expectedUri.rawAuthority == candidateUri.rawAuthority &&
expectedUri.rawPath == candidateUri.rawPath &&
expectedUri.rawQuery == candidateUri.rawQuery
}
private fun blockedWidgetResponse(): WebResourceResponse =
WebResourceResponse(
"text/plain",
"UTF-8",
403,
"Blocked",
mapOf("Cache-Control" to "no-store"),
ByteArrayInputStream(ByteArray(0)),
)
private fun failedWidgetResponse(): WebResourceResponse =
WebResourceResponse(
"text/plain",
"UTF-8",
502,
"Widget unavailable",
mapOf("Cache-Control" to "no-store"),
ByteArrayInputStream(ByteArray(0)),
)
@@ -14,6 +14,7 @@ import ai.openclaw.app.chat.ChatPlanStepStatus
import ai.openclaw.app.chat.ChatSessionEntry
import ai.openclaw.app.chat.ChatThinkingLevelOption
import ai.openclaw.app.chat.ChatThinkingLevelSelection
import ai.openclaw.app.chat.ChatWidgetResource
import ai.openclaw.app.chat.MessageSpeechPhase
import ai.openclaw.app.chat.MessageSpeechState
import ai.openclaw.app.chat.VoiceNoteRecorderState
@@ -420,6 +421,7 @@ fun ChatScreen(
onReplyMessage = viewModel::setChatReplyDraft,
speechState = messageSpeechState,
onToggleListen = viewModel::toggleChatMessageSpeech,
resolveInlineWidgetResource = viewModel::resolveInlineWidgetResource,
modifier = Modifier.weight(1f),
)
@@ -810,6 +812,7 @@ private fun ChatMessageList(
onReplyMessage: (String) -> Unit,
speechState: MessageSpeechState?,
onToggleListen: (String, String) -> Unit,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
modifier: Modifier = Modifier,
) {
val timeline =
@@ -849,6 +852,8 @@ private fun ChatMessageList(
onReplyMessage = onReplyMessage,
speechState = speechState,
onToggleListen = onToggleListen,
inlineWidgetResolverReady = healthOk,
resolveInlineWidgetResource = resolveInlineWidgetResource,
)
is ChatTimelineItem.OutboxCommand ->
ChatOutboxBubble(
@@ -867,6 +872,8 @@ private fun ChatMessageList(
onReplyMessage = onReplyMessage,
speechState = null,
onToggleListen = onToggleListen,
inlineWidgetResolverReady = healthOk,
resolveInlineWidgetResource = resolveInlineWidgetResource,
)
ChatTimelineItem.Thinking -> ChatThinkingBubble()
}
@@ -1047,6 +1054,8 @@ private fun ChatBubble(
onReplyMessage: (String) -> Unit,
speechState: MessageSpeechState?,
onToggleListen: (String, String) -> Unit,
inlineWidgetResolverReady: Boolean,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
) {
val normalizedRole = role.trim().lowercase(Locale.US)
val isUser = normalizedRole == "user"
@@ -1055,6 +1064,7 @@ private fun ChatBubble(
when (part.type) {
"text" -> !part.text.isNullOrBlank()
"image" -> !part.base64.isNullOrBlank()
"canvas" -> normalizedRole == "assistant" && part.widget != null
else -> part.isAudioAttachment()
}
}
@@ -1112,6 +1122,12 @@ private fun ChatBubble(
base64 = checkNotNull(part.base64),
mimeType = part.mimeType,
)
part.type == "canvas" && normalizedRole == "assistant" ->
ChatInlineWidget(
preview = checkNotNull(part.widget),
resolverReady = inlineWidgetResolverReady,
resolveResource = resolveInlineWidgetResource,
)
else -> Text(text = part.fileName ?: nativeString("Attachment"), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
}
}
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"ميزة التعرّف على الكلام على الجهاز غير متاحة."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"المرفقات كبيرة جدًا بحيث لا يمكن وضعها في قائمة الانتظار ضمن رسالة واحدة؛ أزل بعضها وحاول مجددًا."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"يوجد نموذج واحد مُعدّ. حدّث لإعادة التحقق من التوفر."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"الأداة غير متاحة"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"يلزم اتصال آمن لهذا المضيف."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"لا توجد عمليات أتمتة مطابقة."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"يمكن للهاتف الوصول إلى Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Die Spracherkennung auf dem Gerät ist nicht verfügbar."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Die Anhänge sind zu groß, um sie für eine einzelne Nachricht in die Warteschlange zu stellen. Entfernen Sie einige und versuchen Sie es erneut."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 Modell konfiguriert. Aktualisieren Sie, um die Verfügbarkeit erneut zu prüfen."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget nicht verfügbar"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Für diesen Host ist eine sichere Verbindung erforderlich."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Keine passenden Automatisierungen."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Smartphone kann die Gateway erreichen"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"El reconocimiento de voz en el dispositivo no está disponible."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Los archivos adjuntos son demasiado grandes para ponerlos en cola en un solo mensaje; elimina algunos e inténtalo de nuevo."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 modelo configurado. Actualiza para volver a comprobar la disponibilidad."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget no disponible"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Se requiere una conexión segura para este host."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"No hay automatizaciones coincidentes."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"El teléfono puede acceder al Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"تشخیص گفتار روی دستگاه در دسترس نیست."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"پیوست‌ها برای قرار گرفتن در صف یک پیام بیش از حد بزرگ هستند؛ تعدادی را حذف و دوباره تلاش کنید."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"۱ مدل پیکربندی شده است. برای بررسی دوباره موجود بودن، تازه‌سازی کنید."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"ویجت در دسترس نیست"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"برای این میزبان، اتصال امن الزامی است."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"خودکارسازی منطبقی یافت نشد."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"تلفن می‌تواند به Gateway دسترسی داشته باشد"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"La reconnaissance vocale sur lappareil nest pas disponible."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Les pièces jointes sont trop volumineuses pour être mises en file dattente dans un seul message ; supprimez-en certaines et réessayez."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 modèle configuré. Actualisez pour vérifier à nouveau la disponibilité."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget indisponible"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Une connexion sécurisée est requise pour cet hôte."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Aucune automatisation correspondante."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Le téléphone peut joindre la Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"डिवाइस पर स्पीच रिकग्निशन उपलब्ध नहीं है।"</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"एक संदेश के लिए अटैचमेंट कतार में लगाने हेतु बहुत बड़े हैं; कुछ हटाकर फिर से कोशिश करें।"</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 मॉडल कॉन्फ़िगर किया गया है। उपलब्धता दोबारा जाँचने के लिए रीफ़्रेश करें।"</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"विजेट उपलब्ध नहीं है"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"इस होस्ट के लिए सुरक्षित कनेक्शन आवश्यक है।"</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"कोई मेल खाता ऑटोमेशन नहीं मिला।"</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"फ़ोन Gateway तक पहुँच सकता है"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Pengenalan ucapan di perangkat tidak tersedia."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Lampiran terlalu besar untuk dimasukkan ke antrean dalam satu pesan; hapus beberapa lampiran lalu coba lagi."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 model dikonfigurasi. Segarkan untuk memeriksa kembali ketersediaan."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget tidak tersedia"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Koneksi aman diperlukan untuk host ini."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Tidak ada otomatisasi yang cocok."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Ponsel dapat menjangkau Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Il riconoscimento vocale sul dispositivo non è disponibile."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Gli allegati sono troppo grandi per essere messi in coda in un solo messaggio; rimuovine alcuni e riprova."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 modello configurato. Aggiorna per verificare nuovamente la disponibilità."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget non disponibile"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Per questo host è necessaria una connessione sicura."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Nessuna automazione corrispondente."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Il telefono può raggiungere il Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"デバイス上の音声認識は利用できません。"</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"添付ファイルが大きすぎるため、1件のメッセージとしてキューに追加できません。一部を削除して、もう一度お試しください。"</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"設定済みのモデルが1件あります。更新して利用可能か再確認してください。"</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"ウィジェットを利用できません"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"このホストには安全な接続が必要です。"</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"一致する自動化はありません。"</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"スマートフォンが Gateway に到達できること"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"기기 내 음성 인식을 사용할 수 없습니다."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"첨부 파일이 너무 커서 하나의 메시지로 대기열에 추가할 수 없습니다. 일부를 제거한 후 다시 시도하세요."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"구성된 모델이 1개 있습니다. 새로 고쳐 사용 가능 여부를 다시 확인하세요."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"위젯을 사용할 수 없음"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"이 호스트에는 보안 연결이 필요합니다."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"일치하는 자동화가 없습니다."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"휴대폰에서 Gateway에 연결할 수 있음"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Spraakherkenning op het apparaat is niet beschikbaar."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"De bijlagen zijn te groot om voor één bericht in de wachtrij te plaatsen; verwijder er enkele en probeer het opnieuw."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 geconfigureerd model. Vernieuw om de beschikbaarheid opnieuw te controleren."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget niet beschikbaar"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Voor deze host is een beveiligde verbinding vereist."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Geen overeenkomende automatiseringen."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Telefoon kan de Gateway bereiken"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Rozpoznawanie mowy na urządzeniu jest niedostępne."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Załączniki są zbyt duże, aby dodać je do kolejki w jednej wiadomości; usuń część z nich i spróbuj ponownie."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Skonfigurowano 1 model. Odśwież, aby ponownie sprawdzić dostępność."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widżet niedostępny"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Ten host wymaga bezpiecznego połączenia."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Brak pasujących automatyzacji."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Telefon może połączyć się z Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"O reconhecimento de fala no dispositivo não está disponível."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Os anexos são grandes demais para serem colocados na fila em uma única mensagem; remova alguns e tente novamente."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 modelo configurado. Atualize para verificar novamente a disponibilidade."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget indisponível"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"É necessária uma conexão segura para este host."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Nenhuma automação correspondente."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"O telefone pode acessar o Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Распознавание речи на устройстве недоступно."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Вложения слишком велики, чтобы поставить их в очередь в одном сообщении; удалите некоторые из них и попробуйте ещё раз."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Настроена 1 модель. Обновите, чтобы повторно проверить доступность."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Виджет недоступен"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Для этого хоста требуется защищённое соединение."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Подходящие автоматизации не найдены."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Телефон может подключиться к Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Taligenkänning på enheten är inte tillgänglig."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Bilagorna är för stora för att köas i ett enda meddelande. Ta bort några och försök igen."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 konfigurerad modell. Uppdatera för att kontrollera tillgängligheten igen."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widgeten är inte tillgänglig"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"En säker anslutning krävs för den här värden."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Inga matchande automatiseringar."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Telefonen kan nå Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"การรู้จำเสียงพูดบนอุปกรณ์ไม่พร้อมใช้งาน"</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"ไฟล์แนบมีขนาดใหญ่เกินกว่าจะเพิ่มลงในคิวสำหรับข้อความเดียวได้ โปรดนำบางไฟล์ออกแล้วลองอีกครั้ง"</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"มีโมเดลที่กำหนดค่าไว้ 1 รายการ รีเฟรชเพื่อตรวจสอบความพร้อมใช้งานอีกครั้ง"</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"วิดเจ็ตไม่พร้อมใช้งาน"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"โฮสต์นี้ต้องใช้การเชื่อมต่อที่ปลอดภัย"</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"ไม่พบระบบอัตโนมัติที่ตรงกัน"</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"โทรศัพท์สามารถเข้าถึง Gateway ได้"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Cihaz üzerinde konuşma tanıma kullanılamıyor."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Ekler tek bir mesaj için kuyruğa alınamayacak kadar büyük; bazılarını kaldırıp tekrar deneyin."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 yapılandırılmış model var. Kullanılabilirliği yeniden kontrol etmek için yenileyin."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget kullanılamıyor"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Bu ana makine için güvenli bağlantı gereklidir."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Eşleşen otomasyon yok."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Telefon Gateway\'ye erişebiliyor"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Розпізнавання мовлення на пристрої недоступне."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Вкладення завеликі, щоб додати їх до черги в одному повідомленні; видаліть деякі з них і спробуйте ще раз."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Налаштовано 1 модель. Оновіть, щоб повторно перевірити доступність."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Віджет недоступний"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Для цього хоста потрібне захищене з’єднання."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Відповідних автоматизацій не знайдено."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Телефон може підключитися до Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Tính năng nhận dạng giọng nói trên thiết bị không khả dụng."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Các tệp đính kèm quá lớn để đưa vào hàng đợi trong một tin nhắn; hãy xóa bớt rồi thử lại."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 mô hình đã được cấu hình. Làm mới để kiểm tra lại tính khả dụng."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Tiện ích không khả dụng"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Máy chủ này yêu cầu kết nối bảo mật."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Không có tác vụ tự động nào phù hợp."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Điện thoại có thể truy cập Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"设备端语音识别不可用。"</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"附件过大,无法加入单条消息的队列;请移除部分附件后重试。"</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"已配置 1 个模型。刷新以重新检查可用性。"</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"小组件不可用"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"此主机需要安全连接。"</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"没有匹配的自动化。"</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"手机可以连接到 Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"無法使用裝置端語音辨識。"</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"附件過大,無法加入單則訊息的佇列;請移除部分附件後再試一次。"</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"已設定 1 個模型。請重新整理以再次檢查可用性。"</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"小工具無法使用"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"此主機需要安全連線。"</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"沒有相符的自動化。"</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"手機可以連線到 Gateway"</string>
@@ -375,6 +375,7 @@
<string name="native_40d04f6fb9437ff3" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"On-device speech recognition is unavailable."</string>
<string name="native_4122a5c9825224af" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Attachments are too large to queue for one message; remove some and try again."</string>
<string name="native_4135d64f3bde1a92" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"1 configured model. Refresh to recheck availability."</string>
<string name="native_41659b48f2dc4c36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Widget unavailable"</string>
<string name="native_4186107d2194d09a" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Secure connection is required for this host."</string>
<string name="native_41cb8a2ca2cf1e36" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"No matching automations."</string>
<string name="native_41df9f586511e084" formatted="false" tools:ignore="Typos,TypographyDashes,TypographyEllipsis">"Phone can reach the Gateway"</string>
@@ -1,12 +1,29 @@
package ai.openclaw.app.chat
import ai.openclaw.app.ui.chat.readBoundedWidgetDocument
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import okio.Buffer
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
class ChatMessageContentParsingTest {
@Test
fun boundedWidgetDocumentReadAcceptsAtMostLimitAndRejectsOverflow() {
assertArrayEquals(
byteArrayOf(1, 2),
readBoundedWidgetDocument(Buffer().write(byteArrayOf(1, 2)), maxBytes = 3),
)
assertArrayEquals(
byteArrayOf(1, 2, 3),
readBoundedWidgetDocument(Buffer().write(byteArrayOf(1, 2, 3)), maxBytes = 3),
)
assertNull(readBoundedWidgetDocument(Buffer().write(byteArrayOf(1, 2, 3, 4)), maxBytes = 3))
}
@Test
fun dropsInternalToolBlocksFromDisplayHistory() {
val content =
@@ -27,6 +44,237 @@ class ChatMessageContentParsingTest {
assertEquals(ChatMessageContent(type = "text", text = "Done."), parseChatMessageContent(content))
}
@Test
fun parsesCapabilityGatedCanvasWidgets() {
val content =
Json.parseToJsonElement(
"""{"type":"canvas","preview":{"kind":"canvas","surface":"assistant_message","render":"url","title":"Status","preferredHeight":240,"url":"/__openclaw__/canvas/documents/widget-1/index.html","sandbox":"scripts"}}""",
)
assertEquals(
ChatMessageContent(
type = "canvas",
widget =
ChatWidgetPreview(
title = "Status",
path = "/__openclaw__/canvas/documents/widget-1/index.html",
preferredHeight = 240,
sandbox = "scripts",
),
),
parseChatMessageContent(content),
)
}
@Test
fun dropsCanvasBlocksWithoutWidgetSandbox() {
val content =
Json.parseToJsonElement(
"""{"type":"canvas","preview":{"kind":"canvas","surface":"assistant_message","render":"url","url":"/__openclaw__/canvas/documents/widget-1/index.html"}}""",
)
assertNull(parseChatMessageContent(content))
}
@Test
fun dropsCanvasBlocksWithUntrustedWidgetTargets() {
val content =
Json.parseToJsonElement(
"""{"type":"canvas","preview":{"kind":"canvas","surface":"assistant_message","render":"url","url":"https://attacker.example/widget.html","sandbox":"scripts"}}""",
)
assertNull(parseChatMessageContent(content))
}
@Test
fun resolvesOnlyCapabilityScopedWidgetDocuments() {
val surface = "https://gateway.example/__openclaw__/cap/token"
assertEquals(
"https://gateway.example/__openclaw__/cap/token/__openclaw__/canvas/documents/widget-1/index.html",
ChatWidgetUrlResolver.resolve(surface, "/__openclaw__/canvas/documents/widget-1/index.html"),
)
assertEquals(
"https://gateway.example/__openclaw__/cap/token/__openclaw__/canvas/documents/widget-1/index.html",
ChatWidgetUrlResolver.resolve(
"HTTPS://gateway.example/__openclaw__/cap/token",
"/__openclaw__/canvas/documents/widget-1/index.html",
),
)
assertNull(ChatWidgetUrlResolver.resolve("https://gateway.example", "/__openclaw__/canvas/documents/widget-1/index.html"))
assertNull(ChatWidgetUrlResolver.resolve(surface, "https://attacker.example/widget.html"))
assertNull(ChatWidgetUrlResolver.resolve(surface, "/__openclaw__/a2ui/index.html"))
assertNull(ChatWidgetUrlResolver.resolve(surface, "/__openclaw__/canvas/documents/%252e%252e/index.html"))
}
@Test
fun initialResolutionUsesOperatorFallbackWhenNodeUnavailable() {
val target = "/__openclaw__/canvas/documents/widget-1/index.html"
val fallbackSurface = "https://operator.example/__openclaw__/cap/fallback"
val surfaces =
ChatWidgetSurfaceUrls(
node = null,
operator = ChatWidgetSurface(url = fallbackSurface, tlsFingerprintSha256 = null),
)
val resolved = ChatWidgetUrlResolver.resolvePreferred(surfaces, target, excluding = null)
assertEquals(ChatWidgetUrlResolver.resolve(fallbackSurface, target), resolved?.url)
}
@Test
fun usesReplacementRouteAfterCapabilityRefreshLosesItsLease() =
runTest {
val target = "/__openclaw__/canvas/documents/widget-1/index.html"
val oldSurface = "https://gateway.example/__openclaw__/cap/old"
val newSurface = "https://gateway.example/__openclaw__/cap/new"
val oldPin = "aa".repeat(32)
val newPin = "bb".repeat(32)
val failedUrl = ChatWidgetUrlResolver.resolve(oldSurface, target)
val failedResource = ChatWidgetResource(url = requireNotNull(failedUrl), tlsFingerprintSha256 = oldPin)
var current =
ChatWidgetSurfaceUrls(
node = ChatWidgetSurface(url = oldSurface, tlsFingerprintSha256 = oldPin),
operator = null,
)
val resolved =
ChatWidgetUrlResolver.resolveAfterFailure(
target = target,
failedResource = failedResource,
currentSurfaceUrls = { current },
refreshNodeSurface = {
current =
ChatWidgetSurfaceUrls(
node = ChatWidgetSurface(url = newSurface, tlsFingerprintSha256 = newPin),
operator = null,
)
null
},
refreshOperatorSurface = { null },
)
assertEquals(ChatWidgetUrlResolver.resolve(newSurface, target), resolved?.url)
assertEquals(newPin, resolved?.tlsFingerprintSha256)
}
@Test
fun acceptsSameUrlReplacementWhenTlsPinChanged() =
runTest {
val target = "/__openclaw__/canvas/documents/widget-1/index.html"
val surface = "https://gateway.example/__openclaw__/cap/token"
val oldPin = "aa".repeat(32)
val newPin = "bb".repeat(32)
val url = requireNotNull(ChatWidgetUrlResolver.resolve(surface, target))
val failedResource = ChatWidgetResource(url = url, tlsFingerprintSha256 = oldPin)
var current =
ChatWidgetSurfaceUrls(
node = ChatWidgetSurface(url = surface, tlsFingerprintSha256 = oldPin),
operator = null,
)
val resolved =
ChatWidgetUrlResolver.resolveAfterFailure(
target = target,
failedResource = failedResource,
currentSurfaceUrls = { current },
refreshNodeSurface = {
current =
ChatWidgetSurfaceUrls(
node = ChatWidgetSurface(url = surface, tlsFingerprintSha256 = newPin),
operator = null,
)
null
},
refreshOperatorSurface = { null },
)
assertEquals(url, resolved?.url)
assertEquals(newPin, resolved?.tlsFingerprintSha256)
}
@Test
fun refreshesNodeOnceBeforeTryingOperatorFallback() =
runTest {
val target = "/__openclaw__/canvas/documents/widget-1/index.html"
val oldSurface = "https://gateway.example/__openclaw__/cap/old"
val newSurface = "https://gateway.example/__openclaw__/cap/new"
val fallbackSurface = "https://operator.example/__openclaw__/cap/fallback"
var refreshCount = 0
var current =
ChatWidgetSurfaceUrls(
node = ChatWidgetSurface(url = oldSurface, tlsFingerprintSha256 = null),
operator = ChatWidgetSurface(url = fallbackSurface, tlsFingerprintSha256 = null),
)
val initialNode = ChatWidgetUrlResolver.resolvePreferred(current, target, excluding = null)
val refreshedNode =
ChatWidgetUrlResolver.resolveAfterFailure(
target = target,
failedResource = requireNotNull(initialNode),
currentSurfaceUrls = { current },
refreshNodeSurface = {
refreshCount += 1
current = current.copy(node = ChatWidgetSurface(url = newSurface, tlsFingerprintSha256 = null))
null
},
refreshOperatorSurface = { null },
)
assertEquals(ChatWidgetUrlResolver.resolve(newSurface, target), refreshedNode?.url)
val fallback =
ChatWidgetUrlResolver.resolveAfterFailure(
target = target,
failedResource = requireNotNull(refreshedNode),
currentSurfaceUrls = { current },
refreshNodeSurface = {
refreshCount += 1
null
},
refreshOperatorSurface = { null },
)
assertEquals(ChatWidgetUrlResolver.resolve(fallbackSurface, target), fallback?.url)
assertEquals(1, refreshCount)
}
@Test
fun refreshesOperatorCapabilityWhenNodeUnavailable() =
runTest {
val target = "/__openclaw__/canvas/documents/widget-1/index.html"
val oldSurface = "https://operator.example/__openclaw__/cap/old"
val newSurface = "https://operator.example/__openclaw__/cap/new"
val failedResource =
ChatWidgetResource(
url = requireNotNull(ChatWidgetUrlResolver.resolve(oldSurface, target)),
tlsFingerprintSha256 = null,
)
var operatorRefreshCount = 0
var current =
ChatWidgetSurfaceUrls(
node = null,
operator = ChatWidgetSurface(url = oldSurface, tlsFingerprintSha256 = null),
)
val resolved =
ChatWidgetUrlResolver.resolveAfterFailure(
target = target,
failedResource = failedResource,
currentSurfaceUrls = { current },
refreshNodeSurface = { null },
refreshOperatorSurface = {
operatorRefreshCount += 1
ChatWidgetSurface(url = newSurface, tlsFingerprintSha256 = null).also {
current = current.copy(operator = it)
}
},
)
assertEquals(ChatWidgetUrlResolver.resolve(newSurface, target), resolved?.url)
assertEquals(1, operatorRefreshCount)
}
@Test
fun parsesImageBlocksOnlyWhenInlineContentExists() {
val image =
@@ -92,6 +92,121 @@ private data class InvokeScenarioResult(
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
class GatewaySessionInvokeTest {
@Test
fun canvasRoutePinsOnlyTheConnectedTlsEndpoint() {
val fingerprint = "ab".repeat(32)
val endpoint = GatewayEndpoint.manual(host = "gateway.example", port = 7443)
assertEquals(
fingerprint,
gatewayTlsFingerprintForCanvasSurface(
fingerprint = fingerprint,
surfaceUrl = "https://gateway.example:7443/__openclaw__/cap/token",
endpoint = endpoint,
isTlsConnection = true,
),
)
assertNull(
gatewayTlsFingerprintForCanvasSurface(
fingerprint = fingerprint,
surfaceUrl = "https://canvas.example:7443/__openclaw__/cap/token",
endpoint = endpoint,
isTlsConnection = true,
),
)
assertNull(
gatewayTlsFingerprintForCanvasSurface(
fingerprint = fingerprint,
surfaceUrl = "https://gateway.example:9443/__openclaw__/cap/token",
endpoint = endpoint,
isTlsConnection = true,
),
)
}
@Test
fun refreshCanvasHostUrl_usesNodeRefreshMethod() =
runBlocking {
assertCanvasHostRefreshMethod(role = "node", expectedMethod = "node.pluginSurface.refresh")
}
@Test
fun refreshCanvasHostUrl_usesOperatorRefreshMethod() =
runBlocking {
assertCanvasHostRefreshMethod(role = "operator", expectedMethod = "plugin.surface.refresh")
}
private suspend fun assertCanvasHostRefreshMethod(
role: String,
expectedMethod: String,
) {
val json = testJson()
val connected = CompletableDeferred<Unit>()
val lastDisconnect = AtomicReference("")
val refreshRequests = AtomicInteger()
val server =
startGatewayServer(json) { webSocket, id, method, frame ->
when (method) {
"connect" ->
webSocket.send(
connectResponseFrame(
id,
pluginSurfaceUrls =
mapOf("canvas" to "http://127.0.0.1:18789/__openclaw__/cap/old-token"),
),
)
expectedMethod -> {
refreshRequests.incrementAndGet()
assertEquals(
"canvas",
frame["params"]
?.jsonObject
?.get("surface")
?.jsonPrimitive
?.content,
)
assertTrue(
frame["params"]
?.jsonObject
?.get("observedUrl")
?.jsonPrimitive
?.content
?.endsWith("/old-token") == true,
)
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"surface":"canvas","pluginSurfaceUrls":{"canvas":"http://127.0.0.1:18789/__openclaw__/cap/new-token"}}}""",
)
}
}
}
val harness =
createNodeHarness(connected = connected, lastDisconnect = lastDisconnect) {
GatewaySession.InvokeResult.ok("""{"handled":true}""")
}
try {
connectNodeSession(
session = harness.session,
port = server.port,
role = role,
scopes = if (role == "operator") listOf("operator.read") else listOf("node:invoke"),
)
awaitConnectedOrThrow(connected, lastDisconnect, server)
val oldUrl = requireNotNull(harness.session.currentCanvasHostUrl())
assertTrue(oldUrl.endsWith("/old-token"))
val refreshed = harness.session.refreshCanvasHostUrlIfCurrent(oldUrl)
val lagging = harness.session.refreshCanvasHostUrlIfCurrent(oldUrl)
assertTrue(refreshed?.endsWith("/new-token") == true)
assertEquals(refreshed, harness.session.currentCanvasHostUrl())
assertEquals(refreshed, lagging)
assertEquals(1, refreshRequests.get())
} finally {
shutdownHarness(harness, server)
}
}
@Test
fun connect_advertisesCompatibleProtocolRange() =
runBlocking {
@@ -189,8 +189,13 @@ class GatewaySessionReconnectTest {
requireNotNull(
harness.session.captureRequestLease("manual|127.0.0.1|${server.port}"),
)
assertTrue(lease.isCurrent())
harness.session.reconnect()
withTimeout(LIFECYCLE_TEST_TIMEOUT_MS) { reconnected.await() }
assertFalse(lease.isCurrent())
var committed = false
assertFalse(lease.commitIfCurrent { committed = true })
assertFalse(committed)
val result =
runCatching {
lease.request(
@@ -14,6 +14,24 @@ import javax.net.ssl.X509ExtendedTrustManager
import kotlin.concurrent.thread
class GatewayTlsTest {
@Test
fun buildGatewayTlsConfig_exposesNormalizedExpectedRouteFingerprint() {
val expected = "ab".repeat(32)
val config: GatewayTlsConfig =
buildGatewayTlsConfig(
params =
GatewayTlsParams(
required = true,
expectedFingerprint = "SHA-256: $expected",
allowTOFU = false,
stableId = "gateway-1",
),
defaultTrust = RecordingExtendedTrustManager(),
)
assertEquals(expected, config.effectiveFingerprintSha256)
}
@Test
fun buildGatewayTlsConfig_forwardsPlatformTrustWithSocketAndEngineContext() {
val defaultTrust = RecordingExtendedTrustManager()
@@ -439,6 +439,14 @@ class ConnectionManagerTest {
),
options.scopes,
)
assertEquals(listOf(ConnectionManager.INLINE_WIDGETS_CLIENT_CAPABILITY), options.caps)
}
@Test
fun buildOperatorConnectOptions_omitsInlineWidgetsWithoutIsolatedWebViews() {
val options = newManager(inlineWidgetsAvailable = false).buildOperatorConnectOptions()
assertTrue(options.caps.isEmpty())
}
@Test
@@ -619,6 +627,7 @@ class ConnectionManagerTest {
installedAppsSharingEnabled: Boolean = false,
voiceWakeEnabled: Boolean = false,
voiceWakeAvailable: Boolean = true,
inlineWidgetsAvailable: Boolean = true,
): ConnectionManager {
val context = RuntimeEnvironment.getApplication()
context
@@ -646,6 +655,7 @@ class ConnectionManagerTest {
photosAvailable = { photosAvailable },
installedAppsSharingEnabled = { installedAppsSharingEnabled },
voiceWakeAvailable = { voiceWakeAvailable },
inlineWidgetsAvailable = { inlineWidgetsAvailable },
manualTls = { false },
)
}
+136
View File
@@ -179657,6 +179657,142 @@
}
}
},
"Widget unavailable": {
"localizations": {
"en": {
"stringUnit": {
"state": "translated",
"value": "Widget unavailable"
}
},
"zh-CN": {
"stringUnit": {
"state": "translated",
"value": "小组件不可用"
}
},
"zh-TW": {
"stringUnit": {
"state": "translated",
"value": "小工具無法使用"
}
},
"pt-BR": {
"stringUnit": {
"state": "translated",
"value": "Widget indisponível"
}
},
"de": {
"stringUnit": {
"state": "translated",
"value": "Widget nicht verfügbar"
}
},
"es": {
"stringUnit": {
"state": "translated",
"value": "Widget no disponible"
}
},
"ja-JP": {
"stringUnit": {
"state": "translated",
"value": "ウィジェットを利用できません"
}
},
"ko": {
"stringUnit": {
"state": "translated",
"value": "위젯을 사용할 수 없음"
}
},
"fr": {
"stringUnit": {
"state": "translated",
"value": "Widget indisponible"
}
},
"hi": {
"stringUnit": {
"state": "translated",
"value": "विजेट उपलब्ध नहीं है"
}
},
"ar": {
"stringUnit": {
"state": "translated",
"value": "الأداة غير متاحة"
}
},
"it": {
"stringUnit": {
"state": "translated",
"value": "Widget non disponibile"
}
},
"tr": {
"stringUnit": {
"state": "translated",
"value": "Widget kullanılamıyor"
}
},
"uk": {
"stringUnit": {
"state": "translated",
"value": "Віджет недоступний"
}
},
"id": {
"stringUnit": {
"state": "translated",
"value": "Widget tidak tersedia"
}
},
"pl": {
"stringUnit": {
"state": "translated",
"value": "Widżet niedostępny"
}
},
"th": {
"stringUnit": {
"state": "translated",
"value": "วิดเจ็ตไม่พร้อมใช้งาน"
}
},
"vi": {
"stringUnit": {
"state": "translated",
"value": "Tiện ích không khả dụng"
}
},
"nl": {
"stringUnit": {
"state": "translated",
"value": "Widget niet beschikbaar"
}
},
"fa": {
"stringUnit": {
"state": "translated",
"value": "ویجت در دسترس نیست"
}
},
"ru": {
"stringUnit": {
"state": "translated",
"value": "Виджет недоступен"
}
},
"sv": {
"stringUnit": {
"state": "translated",
"value": "Widgeten är inte tillgänglig"
}
}
}
},
"Workboard": {
"localizations": {
"en": {
@@ -7,6 +7,7 @@ import OSLog
struct IOSGatewayChatTransport: OpenClawChatTransport {
static let logger = Logger(subsystem: "ai.openclawfoundation.app", category: "ios.chat.transport")
private let gateway: GatewayNodeSession
private let widgetGateway: GatewayNodeSession?
private let globalAgentId: String?
private let outboxGatewayID: String?
private let sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)?
@@ -17,11 +18,13 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
init(
gateway: GatewayNodeSession,
widgetGateway: GatewayNodeSession? = nil,
globalAgentId: String? = nil,
outboxGatewayID: String? = nil,
sessionMutationRequest: (@Sendable (OpenClawChatGatewayRequest) async throws -> Data)? = nil)
{
self.gateway = gateway
self.widgetGateway = widgetGateway
let normalized = globalAgentId?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
self.globalAgentId = normalized?.isEmpty == false ? normalized : nil
let normalizedGatewayID = outboxGatewayID?.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -296,6 +299,36 @@ struct IOSGatewayChatTransport: OpenClawChatTransport {
try await self.requestHistory(sessionKey: sessionKey, agentID: nil, ifCurrentRoute: nil)
}
func resolveInlineWidgetResource(
path: String,
replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
{
let gateway = self.gateway
let widgetGateway = self.widgetGateway
return await OpenClawChatWidgetURLResolver.resolveResource(
target: path,
replacing: failedResource,
currentSurfaceRoutes: {
let node = await widgetGateway?.currentCanvasHostRoute()
let operatorSurface = await gateway.currentCanvasHostRoute()
return (node: node, operatorSurface: operatorSurface)
},
// Prefer the device's node route; operator rotation covers clients
// whose node role is unavailable or intentionally disabled.
refreshNodeSurfaceRoute: { observed in
await widgetGateway?.refreshCanvasHostRoute(replacing: observed?.url)
},
refreshOperatorSurfaceRoute: { observed in
await gateway.refreshCanvasHostRoute(replacing: observed?.url)
})
}
func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL? {
await self.resolveInlineWidgetResource(
path: path,
replacing: failedURL.map { OpenClawChatWidgetResource(url: $0) })?.url
}
func requestHistory(
sessionKey: String,
agentID: String? = nil,
+2 -1
View File
@@ -641,6 +641,7 @@ final class NodeAppModel {
}
return IOSGatewayChatTransport(
gateway: self.operatorSession,
widgetGateway: self.nodeGateway,
globalAgentId: self.chatDeliveryAgentId,
outboxGatewayID: outboxGatewayID)
}
@@ -4877,7 +4878,7 @@ extension NodeAppModel {
role: "operator",
scopes: scopes,
scopesAreExplicit: forceExplicitScopes,
caps: [],
caps: [OpenClawGatewayClientCapability.inlineWidgets],
commands: [],
permissions: [:],
clientId: clientId,
@@ -231,6 +231,7 @@ private func waitForActiveGateway(stableID: String, appModel: NodeAppModel) asyn
#expect(caps.contains(OpenClawCapability.canvas.rawValue))
#expect(caps.contains(OpenClawCapability.screen.rawValue))
#expect(!caps.contains(OpenClawGatewayClientCapability.inlineWidgets))
#expect(caps.contains(OpenClawCapability.camera.rawValue))
#expect(caps.contains(OpenClawCapability.location.rawValue))
#expect(caps.contains(OpenClawCapability.voiceWake.rawValue))
@@ -322,6 +323,7 @@ private func waitForActiveGateway(stableID: String, appModel: NodeAppModel) asyn
#expect(!withoutApprovalScope.scopes.contains("operator.approvals"))
#expect(withoutApprovalScope.scopes.contains("operator.talk.secrets"))
#expect(!withoutApprovalScope.scopesAreExplicit)
#expect(withoutApprovalScope.caps == [OpenClawGatewayClientCapability.inlineWidgets])
#expect(withApprovalScope.scopes.contains("operator.approvals"))
#expect(withAdminScope.scopes.contains("operator.admin"))
+3 -1
View File
@@ -251,7 +251,9 @@ struct SwiftUIRenderSmokeTests {
assistantAvatarTint: nil,
showsAssistantAvatar: true,
isClean: false,
contextWindowTokens: 1_000_000)
contextWindowTokens: 1_000_000,
inlineWidgetResolverReady: true,
inlineWidgetResourceResolver: { _, _ in nil })
.environment(\.dynamicTypeSize, typeSize)
_ = Self.host(root, size: CGSize(width: 320, height: 280))
@@ -0,0 +1,27 @@
import Foundation
import OpenClawChatUI
import OpenClawProtocol
extension GatewayConnection {
func configuredInferenceModel(
ifCurrentRoute route: Route,
timeoutMs: Double = 15000) async throws -> String?
{
let data = try await request(
OpenClawChatGatewayRequests.agentsList(timeoutMs: timeoutMs),
ifCurrentRoute: route)
guard await self.isCurrentRoute(route) else {
throw CancellationError()
}
return try Self.decodeConfiguredInferenceModel(data)
}
static func decodeConfiguredInferenceModel(_ data: Data) throws -> String? {
let result = try JSONDecoder().decode(AgentsListResult.self, from: data)
let primary = result.agents
.first(where: { $0.id == result.defaultid })?
.model?["primary"]?.value as? String
let trimmed = primary?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return trimmed.isEmpty ? nil : trimmed
}
}
@@ -0,0 +1,108 @@
import Foundation
import OpenClawKit
import OpenClawProtocol
import OSLog
private let gatewayWidgetSurfaceLogger = Logger(subsystem: "ai.openclaw", category: "gateway.widget-surface")
private struct PluginSurfaceRefreshResponse: Decodable {
let pluginSurfaceUrls: [String: AnyCodable]?
}
extension GatewayConnection {
func canvasPluginSurfaceUrl() async -> String? {
self.canvasPluginSurfaceURL
}
func canvasPluginSurfaceRoute() async -> GatewayCanvasHostRoute? {
self.currentCanvasPluginSurfaceRoute()
}
func refreshCanvasPluginSurfaceRoute(replacing observedURL: String?) async -> GatewayCanvasHostRoute? {
if self.canvasPluginSurfaceURL != observedURL {
return self.currentCanvasPluginSurfaceRoute()
}
if let activeRefresh = self.canvasPluginSurfaceRefresh {
return await activeRefresh.task.value
}
guard let lease = await self.captureServerLease(), !Task.isCancelled else { return nil }
// Capturing the physical connection before spawning prevents a canceled
// refresh from adopting a replacement connection when its task starts.
if self.canvasPluginSurfaceURL != observedURL {
return self.currentCanvasPluginSurfaceRoute()
}
if let activeRefresh = self.canvasPluginSurfaceRefresh {
return await activeRefresh.task.value
}
let id = UUID()
let task = Task<GatewayCanvasHostRoute?, Never> { [weak self] in
guard let self, !Task.isCancelled else { return nil }
return await self.requestCanvasPluginSurfaceRefresh(
replacing: observedURL,
ifCurrentServerLease: lease)
}
// Install before the task's first suspension so sibling widgets share
// one rotation instead of invalidating each other's new capability.
self.canvasPluginSurfaceRefresh = CanvasPluginSurfaceRefresh(id: id, task: task)
let route = await task.value
if self.canvasPluginSurfaceRefresh?.id == id {
self.canvasPluginSurfaceRefresh = nil
}
return route
}
private func requestCanvasPluginSurfaceRefresh(
replacing observedURL: String?,
ifCurrentServerLease lease: ServerLease) async -> GatewayCanvasHostRoute?
{
guard !Task.isCancelled else { return nil }
var params = ["surface": AnyCodable("canvas")]
if let observedURL {
params["observedUrl"] = AnyCodable(observedURL)
}
do {
let data = try await self.request(
method: "plugin.surface.refresh",
params: params,
timeoutMs: 8000,
ifCurrentServerLease: lease)
let response = try JSONDecoder().decode(PluginSurfaceRefreshResponse.self, from: data)
guard !Task.isCancelled, await self.isCurrentServerLease(lease) else { return nil }
if self.canvasPluginSurfaceURL != observedURL {
return self.currentCanvasPluginSurfaceRoute()
}
let raw = response.pluginSurfaceUrls?["canvas"]?.value as? String
guard let refreshed = GatewayPluginSurfaceURL.canonicalize(
raw: raw,
against: self.configuredGatewayURL())
else { return nil }
self.canvasPluginSurfaceURL = refreshed
return self.currentCanvasPluginSurfaceRoute()
} catch {
gatewayWidgetSurfaceLogger.debug(
"plugin.surface.refresh failed: \(error.localizedDescription, privacy: .public)")
return nil
}
}
private func currentCanvasPluginSurfaceRoute() -> GatewayCanvasHostRoute? {
guard let url = self.canvasPluginSurfaceURL else { return nil }
// The operator channel uses platform trust. Pinned remote routes belong
// to MacNodeModeCoordinator and arrive through its node session.
return GatewayCanvasHostRoute(url: url, tlsFingerprintSHA256: nil)
}
func installCanvasPluginSurfaceURL(from snapshot: HelloOk) {
let raw = snapshot.pluginsurfaceurls?["canvas"]?.value as? String
self.resetCanvasPluginSurfaceState()
self.canvasPluginSurfaceURL = GatewayPluginSurfaceURL.canonicalize(
raw: raw,
against: self.configuredGatewayURL())
}
func resetCanvasPluginSurfaceState() {
self.canvasPluginSurfaceRefresh?.task.cancel()
self.canvasPluginSurfaceRefresh = nil
self.canvasPluginSurfaceURL = nil
}
}
@@ -77,6 +77,7 @@ private enum GatewayActivationBindingKeyStore {
actor GatewayConnection {
static let shared = GatewayConnection(
endpointProvider: GatewayConnection.defaultEndpointProvider)
nonisolated static let operatorClientCaps = [OpenClawGatewayClientCapability.inlineWidgets]
typealias Config = (url: URL, token: String?, password: String?)
@@ -206,6 +207,14 @@ actor GatewayConnection {
private var subscribers: [UUID: AsyncStream<GatewayPush>.Continuation] = [:]
private var lastSnapshot: HelloOk?
var canvasPluginSurfaceURL: String?
struct CanvasPluginSurfaceRefresh {
let id: UUID
let task: Task<GatewayCanvasHostRoute?, Never>
}
var canvasPluginSurfaceRefresh: CanvasPluginSurfaceRefresh?
private struct LossyDecodable<Value: Decodable>: Decodable {
let value: Value?
@@ -828,26 +837,8 @@ extension GatewayConnection {
return try OpenClawChatGatewayPayloadCodec.decodeSessionRoutingIdentity(data)
}
func configuredInferenceModel(
ifCurrentRoute route: Route,
timeoutMs: Double = 15000) async throws -> String?
{
let data = try await request(
OpenClawChatGatewayRequests.agentsList(timeoutMs: timeoutMs),
ifCurrentRoute: route)
guard await self.isCurrentRoute(route) else {
throw CancellationError()
}
return try Self.decodeConfiguredInferenceModel(data)
}
static func decodeConfiguredInferenceModel(_ data: Data) throws -> String? {
let result = try JSONDecoder().decode(AgentsListResult.self, from: data)
let primary = result.agents
.first(where: { $0.id == result.defaultid })?
.model?["primary"]?.value as? String
let trimmed = primary?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return trimmed.isEmpty ? nil : trimmed
func configuredGatewayURL() -> URL? {
self.configuredURL
}
func authSource() async -> GatewayAuthSource? {
@@ -860,6 +851,7 @@ extension GatewayConnection {
self.routeGeneration &+= 1
resetSocketGeneration()
self.lastSnapshot = nil
self.resetCanvasPluginSurfaceState()
let client = client
self.client = nil
self.configuredURL = nil
@@ -898,6 +890,7 @@ extension GatewayConnection {
self.routeGeneration &+= 1
resetSocketGeneration()
self.lastSnapshot = nil
self.resetCanvasPluginSurfaceState()
let configuredRouteGeneration = self.routeGeneration
let previousClient = client
client = nil
@@ -947,7 +940,7 @@ extension GatewayConnection {
connectOptions: GatewayConnectOptions(
role: "operator",
scopes: GatewayChannelActor.defaultOperatorConnectScopes,
caps: [],
caps: Self.operatorClientCaps,
commands: [],
permissions: [:],
clientId: "openclaw-macos",
@@ -1017,6 +1010,7 @@ extension GatewayConnection {
admitSocketGeneration(socketGeneration)
else { return }
self.lastSnapshot = snapshot
self.installCanvasPluginSurfaceURL(from: snapshot)
}
private func handleDisconnect(routeGeneration: UInt64, socketGeneration: UInt64) {
@@ -1024,6 +1018,7 @@ extension GatewayConnection {
retireSocketGeneration(socketGeneration)
else { return }
self.lastSnapshot = nil
self.resetCanvasPluginSurfaceState()
}
}
@@ -1138,13 +1133,6 @@ extension GatewayConnection {
// MARK: - Snapshot cache and subscriptions
extension GatewayConnection {
func canvasPluginSurfaceUrl() async -> String? {
guard let snapshot = lastSnapshot else { return nil }
let raw = snapshot.pluginsurfaceurls?["canvas"]?.value as? String
let trimmed = raw?.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) ?? ""
return trimmed.isEmpty ? nil : trimmed
}
func controlUiAutoAuthToken(config: Config) async -> String? {
guard let endpoint = try? await currentEndpoint(),
endpoint.config.url == config.url,
@@ -1270,6 +1258,9 @@ extension GatewayConnection {
private func broadcast(_ push: GatewayPush) {
if case let .snapshot(snapshot) = push {
self.lastSnapshot = snapshot
if self.canvasPluginSurfaceURL == nil {
self.installCanvasPluginSurfaceURL(from: snapshot)
}
if let mainSessionKey = cachedMainSessionKey() {
Task { @MainActor in
WorkActivityStore.shared.setMainSessionKey(mainSessionKey)
@@ -2,37 +2,39 @@ import Foundation
struct MacNodeCanvasHostedSurfaceResolver: Sendable {
private let currentSurfaceURL: @Sendable () async -> String?
private let refreshSurfaceURL: @Sendable () async -> String?
private let refreshSurfaceURL: @Sendable (String?) async -> String?
init(
currentSurfaceURL: @escaping @Sendable () async -> String?,
refreshSurfaceURL: @escaping @Sendable () async -> String?)
refreshSurfaceURL: @escaping @Sendable (String?) async -> String?)
{
self.currentSurfaceURL = currentSurfaceURL
self.refreshSurfaceURL = refreshSurfaceURL
}
func resolveA2UIURL(forceRefresh: Bool = false) async -> String? {
let observedSurface = await currentSurfaceURL()
if !forceRefresh,
let currentSurface = await currentSurfaceURL(),
let current = CanvasHostedURLResolver.resolveA2UIURL(surfaceURL: currentSurface)
let current = CanvasHostedURLResolver.resolveA2UIURL(surfaceURL: observedSurface)
{
return current
}
let refreshedSurface = await refreshSurfaceURL()
let refreshedSurface = await refreshSurfaceURL(observedSurface)
return CanvasHostedURLResolver.resolveA2UIURL(surfaceURL: refreshedSurface)
}
func resolveTarget(_ target: String?) async throws -> CanvasHostedTarget? {
guard let target, CanvasHostedURLResolver.isHostedTarget(target) else { return nil }
if let refreshedSurface = await refreshSurfaceURL(),
let observedSurface = await currentSurfaceURL()
if let refreshedSurface = await refreshSurfaceURL(observedSurface),
let resolved = CanvasHostedURLResolver.resolve(surfaceURL: refreshedSurface, target: target)
{
return resolved
}
if let currentSurface = await currentSurfaceURL(),
let resolved = CanvasHostedURLResolver.resolve(surfaceURL: currentSurface, target: target)
{
// A failed refresh can mean the route changed while it was in flight.
// Re-read so the replacement gateway's capability wins over the stale observation.
let currentSurface = await currentSurfaceURL()
if let resolved = CanvasHostedURLResolver.resolve(surfaceURL: currentSurface, target: target) {
return resolved
}
throw NSError(domain: "Canvas", code: 32, userInfo: [
@@ -123,7 +123,9 @@ final class MacNodeModeCoordinator: NSObject {
runtime: MacNodeRuntime(
nodeHostWorker: nodeHostWorker,
canvasSurfaceUrl: { await session.currentCanvasHostUrl() },
refreshCanvasSurfaceUrl: { await session.refreshCanvasHostUrl() }),
refreshCanvasSurfaceUrl: { observedURL in
await session.refreshCanvasHostUrl(replacing: observedURL)
}),
nodeHostWorker: nodeHostWorker,
presenceReporter: MacNodePresenceReporter(),
observeNotifications: true,
@@ -263,6 +265,14 @@ final class MacNodeModeCoordinator: NSObject {
forKey: computerControlEnabledKey) as? Bool ?? false)
}
func currentCanvasPluginSurfaceRoute() async -> GatewayCanvasHostRoute? {
await self.session.currentCanvasHostRoute()
}
func refreshCanvasPluginSurfaceRoute(replacing observedURL: String?) async -> GatewayCanvasHostRoute? {
await self.session.refreshCanvasHostRoute(replacing: observedURL)
}
private func refresh(isPaused: Bool, computerControlEnabled: Bool) {
let shouldRevoke = Self.controlTransitionRequiresRouteInvalidation(
previousPaused: self.lastObservedPaused,
@@ -38,7 +38,7 @@ actor MacNodeRuntime {
canvasSurfaceUrl: @escaping @Sendable () async -> String? = {
await GatewayConnection.shared.canvasPluginSurfaceUrl()
},
refreshCanvasSurfaceUrl: @escaping @Sendable () async -> String? = { nil },
refreshCanvasSurfaceUrl: @escaping @Sendable (String?) async -> String? = { _ in nil },
codexThreadCatalogEnabled: @escaping @Sendable () -> Bool = {
MacNodeCodexThreadCatalog.shouldAdvertise()
},
@@ -89,6 +89,34 @@ struct MacGatewayChatTransport: OpenClawChatTransport {
agentID: target.agentID)
}
func resolveInlineWidgetResource(
path: String,
replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
{
await OpenClawChatWidgetURLResolver.resolveResource(
target: path,
replacing: failedResource,
currentSurfaceRoutes: {
let node = await MacNodeModeCoordinator.shared.currentCanvasPluginSurfaceRoute()
let operatorSurface = await GatewayConnection.shared.canvasPluginSurfaceRoute()
return (node: node, operatorSurface: operatorSurface)
},
// Prefer the local node route; operator rotation keeps chat usable
// while macOS node mode is disabled or reconnecting.
refreshNodeSurfaceRoute: { observed in
await MacNodeModeCoordinator.shared.refreshCanvasPluginSurfaceRoute(replacing: observed?.url)
},
refreshOperatorSurfaceRoute: { observed in
await GatewayConnection.shared.refreshCanvasPluginSurfaceRoute(replacing: observed?.url)
})
}
func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL? {
await self.resolveInlineWidgetResource(
path: path,
replacing: failedURL.map { OpenClawChatWidgetResource(url: $0) })?.url
}
func listModels() async throws -> [OpenClawChatModelChoice] {
do {
let data = try await GatewayConnection.shared.request(OpenClawChatGatewayRequests.modelsList())
@@ -167,12 +167,13 @@ struct AsyncTimeoutTests {
}
}
@Test func `caller cancellation returns before operation finishes`() async {
@Test(arguments: [0.0, 60.0])
func `caller cancellation returns before operation finishes`(seconds: Double) async {
let operation = CancellationIgnoringOperation()
let cancellation = CancellationProbe()
let task = Task {
try await AsyncTimeout.withTimeout(
seconds: 60,
seconds: seconds,
onTimeout: { ExpectedTimeout() },
operation: {
await withTaskCancellationHandler {
@@ -235,6 +235,78 @@ private func makeTestGatewayConnection() -> (GatewayConnection, FakeWebSocketSes
}
@Suite(.serialized) struct GatewayConnectionControlTests {
@Test func `operator widget capability refresh is shared and retained`() async throws {
let rawOldSurface = "http://127.0.0.1:18789/__openclaw__/cap/old-token"
let rawNewSurface = "http://127.0.0.1:18789/__openclaw__/cap/new-token"
let oldSurface = "https://gateway.example.invalid:9443/__openclaw__/cap/old-token"
let newSurface = "https://gateway.example.invalid:9443/__openclaw__/cap/new-token"
let recorder = WebSocketMessageRecorder()
let session = GatewayTestWebSocketSession(taskFactory: {
GatewayTestWebSocketTask(
sendHook: { task, message, sendIndex in
recorder.append(message)
guard sendIndex > 0,
let data = Self.messageData(message),
let frame = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let method = frame["method"] as? String,
let id = frame["id"] as? String
else { return }
if method == "plugin.surface.refresh" {
let response = """
{
"type": "res",
"id": "\(id)",
"ok": true,
"payload": {
"surface": "canvas",
"pluginSurfaceUrls": { "canvas": "\(rawNewSurface)" }
}
}
"""
task.emitReceiveSuccess(.data(Data(response.utf8)))
} else {
task.emitReceiveSuccess(.data(GatewayWebSocketTestSupport.okResponseData(id: id)))
}
},
receiveHook: { task, receiveIndex in
if receiveIndex == 0 {
return .data(GatewayWebSocketTestSupport.connectChallengeData())
}
let id = task.snapshotConnectRequestID() ?? "connect"
return .data(GatewayWebSocketTestSupport.connectOkData(
id: id,
canvasPluginSurfaceURL: rawOldSurface))
})
})
let connection = GatewayConnection(
configProvider: {
(
url: URL(string: "wss://gateway.example.invalid:9443")!,
token: "test-token-placeholder",
password: nil)
},
sessionBox: WebSocketSessionBox(session: session))
try await connection.refresh()
_ = try await connection.acquireServerLease()
#expect(await connection.canvasPluginSurfaceUrl() == oldSurface)
async let first = connection.refreshCanvasPluginSurfaceRoute(replacing: oldSurface)
async let second = connection.refreshCanvasPluginSurfaceRoute(replacing: oldSurface)
let routes = await (first, second)
#expect(routes.0?.url == newSurface)
#expect(routes.1?.url == newSurface)
#expect(await connection.canvasPluginSurfaceUrl() == newSurface)
let refreshCount = recorder.snapshot().count { message in
guard let data = Self.messageData(message),
let frame = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
else { return false }
return frame["method"] as? String == "plugin.surface.refresh"
}
#expect(refreshCount == 1)
await connection.shutdown()
}
@Test func `wizard not found means cancellation already reached a terminal session`() {
let notFound = GatewayResponseError(
method: "wizard.cancel",
@@ -48,9 +48,13 @@ enum GatewayWebSocketTestSupport {
static func connectOkData(
id: String,
tickIntervalMs: Int = 30000,
deviceToken: String? = nil) -> Data
deviceToken: String? = nil,
canvasPluginSurfaceURL: String? = nil) -> Data
{
let deviceTokenField = deviceToken.map { #", "deviceToken": "\#($0)""# } ?? ""
let pluginSurfaceField = canvasPluginSurfaceURL.map {
#", "pluginSurfaceUrls": { "canvas": "\#($0)" }"#
} ?? ""
let json = """
{
"type": "res",
@@ -60,7 +64,7 @@ enum GatewayWebSocketTestSupport {
"type": "hello-ok",
"protocol": 2,
"server": { "version": "test", "connId": "test" },
"features": { "methods": [], "events": [] },
"features": { "methods": [], "events": [] }\(pluginSurfaceField),
"snapshot": {
"presence": [ { "ts": 1 } ],
"health": {},
@@ -150,7 +154,8 @@ enum GatewayWebSocketTestSupport {
extension NSLock {
@inline(__always)
fileprivate func withLock<T>(_ body: () throws -> T) rethrows -> T {
self.lock(); defer { self.unlock() }
self.lock()
defer { self.unlock() }
return try body()
}
}
@@ -1,9 +1,14 @@
import OpenClawChatUI
import OpenClawKit
import OpenClawProtocol
import Testing
@testable import OpenClaw
struct MacGatewayChatTransportMappingTests {
@Test func `mac chat advertises inline widgets`() {
#expect(GatewayConnection.operatorClientCaps == [OpenClawGatewayClientCapability.inlineWidgets])
}
@Test func `bare global session target carries normalized selected agent`() {
let transport = MacGatewayChatTransport(defaultGlobalAgentID: " Agent-A ")
@@ -61,6 +61,19 @@ struct MacNodeRuntimeTests {
}
}
actor CanvasReconnectProbe {
private var surfaceURL = "http://127.0.0.1:18789/__openclaw__/cap/old-token"
func current() -> String? {
self.surfaceURL
}
func reconnectDuringRefresh() -> String? {
self.surfaceURL = "http://127.0.0.1:18789/__openclaw__/cap/new-token"
return nil
}
}
@MainActor
final class ScreenSnapshotProbeServices: MacNodeRuntimeMainActorServices, @unchecked Sendable {
var snapshotCallCount = 0
@@ -251,7 +264,7 @@ struct MacNodeRuntimeTests {
let probe = CanvasRefreshProbe()
let resolver = MacNodeCanvasHostedSurfaceResolver(
currentSurfaceURL: { "http://127.0.0.1:18789/__openclaw__/cap/current-token" },
refreshSurfaceURL: { await probe.refresh() })
refreshSurfaceURL: { _ in await probe.refresh() })
let current = await resolver.resolveA2UIURL()
#expect(current ==
@@ -268,7 +281,7 @@ struct MacNodeRuntimeTests {
let probe = CanvasRefreshProbe()
let resolver = MacNodeCanvasHostedSurfaceResolver(
currentSurfaceURL: { "http://127.0.0.1:18789/__openclaw__/cap/current-token" },
refreshSurfaceURL: { await probe.refresh() })
refreshSurfaceURL: { _ in await probe.refresh() })
let resolved = try await resolver.resolveTarget(
"/__openclaw__/canvas/demo%20page.html?mode=proof#result")
@@ -282,6 +295,18 @@ struct MacNodeRuntimeTests {
#expect(await probe.calls == 1)
}
@Test func `hosted Canvas commands use replacement route after refresh fails`() async throws {
let probe = CanvasReconnectProbe()
let resolver = MacNodeCanvasHostedSurfaceResolver(
currentSurfaceURL: { await probe.current() },
refreshSurfaceURL: { _ in await probe.reconnectDuringRefresh() })
let resolved = try await resolver.resolveTarget("/__openclaw__/canvas/demo.html")
#expect(resolved?.url.absoluteString ==
"http://127.0.0.1:18789/__openclaw__/cap/new-token/__openclaw__/canvas/demo.html")
}
@Test func `handle invoke rejects empty notification`() async throws {
let runtime = MacNodeRuntime()
let params = OpenClawSystemNotifyParams(title: "", body: "")
@@ -979,5 +1004,4 @@ struct MacNodeRuntimeTests {
nodeId: controlHeavyNodeId)
#expect(controlHeavyProjection > 25 * 1024 * 1024)
}
}
@@ -0,0 +1,635 @@
import CryptoKit
import Foundation
import OpenClawKit
import SwiftUI
#if canImport(WebKit) && (os(iOS) || os(macOS))
import Security
import WebKit
#endif
enum OpenClawChatWidgetSurfaceRole: Sendable, Hashable {
case node
case operatorSurface
case legacy
}
public struct OpenClawChatWidgetResource: Sendable, Equatable {
public let url: URL
public let tlsFingerprintSHA256: String?
let surfaceRole: OpenClawChatWidgetSurfaceRole
let attemptedSurfaceRoles: Set<OpenClawChatWidgetSurfaceRole>
public init(url: URL, tlsFingerprintSHA256: String? = nil) {
self.url = url
self.tlsFingerprintSHA256 = tlsFingerprintSHA256
self.surfaceRole = .legacy
self.attemptedSurfaceRoles = []
}
init(
url: URL,
tlsFingerprintSHA256: String?,
surfaceRole: OpenClawChatWidgetSurfaceRole,
attemptedSurfaceRoles: Set<OpenClawChatWidgetSurfaceRole>)
{
self.url = url
self.tlsFingerprintSHA256 = tlsFingerprintSHA256
self.surfaceRole = surfaceRole
self.attemptedSurfaceRoles = attemptedSurfaceRoles
}
public static func == (lhs: Self, rhs: Self) -> Bool {
lhs.url == rhs.url && lhs.tlsFingerprintSHA256 == rhs.tlsFingerprintSHA256
}
}
public enum OpenClawChatWidgetURLResolver {
private static let documentsPath = "/__openclaw__/canvas/documents"
public static func resolve(surfaceURL rawSurfaceURL: String?, target rawTarget: String) -> URL? {
guard let target = self.relativeWidgetTarget(rawTarget),
var surface = self.capabilitySurface(rawSurfaceURL)
else { return nil }
var surfacePath = surface.percentEncodedPath
while surfacePath.hasSuffix("/") {
surfacePath.removeLast()
}
surface.percentEncodedPath = surfacePath + target.percentEncodedPath
surface.percentEncodedQuery = target.percentEncodedQuery
surface.fragment = target.fragment
return surface.url
}
public static func supportsTarget(_ rawTarget: String) -> Bool {
self.relativeWidgetTarget(rawTarget) != nil
}
public static func resolveResource(
target: String,
replacing failedResource: OpenClawChatWidgetResource?,
currentSurfaceRoutes: @Sendable () async -> (
node: GatewayCanvasHostRoute?,
operatorSurface: GatewayCanvasHostRoute?),
refreshNodeSurfaceRoute: @Sendable (GatewayCanvasHostRoute?) async -> GatewayCanvasHostRoute?,
refreshOperatorSurfaceRoute: @Sendable (GatewayCanvasHostRoute?) async -> GatewayCanvasHostRoute?) async
-> OpenClawChatWidgetResource?
{
let observed = await currentSurfaceRoutes()
guard let failedResource else {
return self.resolvePreferred(
surfaces: observed,
target: target,
excluding: nil,
blockedRoles: [],
attemptedRoles: [])
}
let blockedRoles = failedResource.attemptedSurfaceRoles
if failedResource.surfaceRole == .legacy,
blockedRoles.contains(.legacy)
{
return nil
}
let attemptedRoles = blockedRoles.union([failedResource.surfaceRole])
if !blockedRoles.contains(.node),
let nodeSurface = observed.node,
let currentNode = self.resolve(
surface: nodeSurface,
role: .node,
target: target,
attemptedRoles: attemptedRoles),
self.isReplacement(currentNode, for: failedResource)
{
return currentNode
}
if !blockedRoles.contains(.node),
let refreshedSurface = await refreshNodeSurfaceRoute(observed.node),
let refreshed = resolve(
surface: refreshedSurface,
role: .node,
target: target,
attemptedRoles: attemptedRoles),
self.isReplacement(refreshed, for: failedResource)
{
return refreshed
}
// A nil refresh can mean its route lease lost a reconnect race. Re-read
// both roles so a replacement connection and its TLS pin win together.
let afterNodeRefresh = await currentSurfaceRoutes()
if let replacement = self.resolvePreferred(
surfaces: afterNodeRefresh,
target: target,
excluding: failedResource,
blockedRoles: blockedRoles,
attemptedRoles: attemptedRoles)
{
return replacement
}
if !blockedRoles.contains(.operatorSurface),
let refreshedSurface = await refreshOperatorSurfaceRoute(afterNodeRefresh.operatorSurface),
let refreshed = resolve(
surface: refreshedSurface,
role: .operatorSurface,
target: target,
attemptedRoles: attemptedRoles),
self.isReplacement(refreshed, for: failedResource)
{
return refreshed
}
return await self.resolvePreferred(
surfaces: currentSurfaceRoutes(),
target: target,
excluding: failedResource,
blockedRoles: blockedRoles,
attemptedRoles: attemptedRoles)
}
private static func relativeWidgetTarget(_ rawTarget: String) -> URLComponents? {
let target = rawTarget.trimmingCharacters(in: .whitespacesAndNewlines)
guard target.hasPrefix("/"),
let components = URLComponents(string: target),
components.scheme == nil,
components.host == nil,
components.user == nil,
components.password == nil,
self.isCanonicalPath(components.percentEncodedPath),
components.percentEncodedPath.hasPrefix("\(self.documentsPath)/")
else { return nil }
return components
}
private static func capabilitySurface(_ rawSurfaceURL: String?) -> URLComponents? {
let raw = rawSurfaceURL?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !raw.isEmpty,
let components = URLComponents(string: raw),
self.isWebURL(components),
components.user == nil,
components.password == nil,
components.percentEncodedQuery == nil,
components.fragment == nil
else { return nil }
let segments = components.percentEncodedPath.split(separator: "/", omittingEmptySubsequences: true)
guard segments.count >= 3,
segments[segments.count - 3] == "__openclaw__",
segments[segments.count - 2] == "cap",
let capability = String(segments[segments.count - 1]).removingPercentEncoding,
!capability.isEmpty
else { return nil }
return components
}
private static func resolve(
surface: GatewayCanvasHostRoute,
role: OpenClawChatWidgetSurfaceRole,
target: String,
attemptedRoles: Set<OpenClawChatWidgetSurfaceRole>) -> OpenClawChatWidgetResource?
{
guard let url = resolve(surfaceURL: surface.url, target: target) else { return nil }
let resource = OpenClawChatWidgetResource(
url: url,
tlsFingerprintSHA256: surface.tlsFingerprintSHA256,
surfaceRole: role,
attemptedSurfaceRoles: attemptedRoles)
return resource.hasValidTLSBinding ? resource : nil
}
private static func resolvePreferred(
surfaces: (node: GatewayCanvasHostRoute?, operatorSurface: GatewayCanvasHostRoute?),
target: String,
excluding failedResource: OpenClawChatWidgetResource?,
blockedRoles: Set<OpenClawChatWidgetSurfaceRole>,
attemptedRoles: Set<OpenClawChatWidgetSurfaceRole>) -> OpenClawChatWidgetResource?
{
[
(role: OpenClawChatWidgetSurfaceRole.node, surface: surfaces.node),
(role: OpenClawChatWidgetSurfaceRole.operatorSurface, surface: surfaces.operatorSurface),
]
.lazy
.filter { !blockedRoles.contains($0.role) }
.compactMap { candidate in
candidate.surface.flatMap {
self.resolve(
surface: $0,
role: candidate.role,
target: target,
attemptedRoles: attemptedRoles)
}
}
.first { self.isReplacement($0, for: failedResource) }
}
private static func isReplacement(
_ candidate: OpenClawChatWidgetResource,
for failedResource: OpenClawChatWidgetResource?) -> Bool
{
guard let failedResource else { return true }
// Legacy URL-only callers cannot express trust identity, so retain
// their URL-only exclusion while resource-aware callers compare both.
if failedResource.surfaceRole == .legacy,
failedResource.tlsFingerprintSHA256 == nil
{
return candidate.url != failedResource.url
}
return candidate.url != failedResource.url ||
candidate.tlsFingerprintSHA256 != failedResource.tlsFingerprintSHA256
}
private static func isWebURL(_ components: URLComponents) -> Bool {
let scheme = components.scheme?.lowercased()
return (scheme == "http" || scheme == "https") && components.host?.isEmpty == false
}
private static func isCanonicalPath(_ path: String) -> Bool {
let segments = path.split(separator: "/", omittingEmptySubsequences: false)
guard segments.first?.isEmpty == true else { return false }
for (index, encodedSegment) in segments.enumerated() {
if index == 0 || (index == segments.count - 1 && encodedSegment.isEmpty) {
continue
}
guard !encodedSegment.isEmpty else { return false }
guard let segment = self.decodeRepeatedly(String(encodedSegment)) else { return false }
if segment == "." || segment == ".." || segment.contains("/") || segment.contains("\\") {
return false
}
}
return true
}
private static func decodeRepeatedly(_ encoded: String) -> String? {
var value = encoded
for _ in 0..<8 {
guard let decoded = value.removingPercentEncoding else { return nil }
if decoded == value { return decoded }
value = decoded
}
return nil
}
}
@MainActor
struct ChatInlineWidgetView: View {
let preview: OpenClawChatCanvasPreview
let resolverReady: Bool
let resolveResource: @MainActor @Sendable (
String,
OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
@State private var resolvedResource: OpenClawChatWidgetResource?
@State private var recoveryAttempts = 0
@State private var refreshInFlight = false
@State private var unavailable = false
@State private var activePath: String?
/// A reset can keep the same path while installing a new connection route.
/// Its generation prevents older resolver completions from restoring stale trust state.
@State private var loadGeneration = UUID()
var body: some View {
VStack(alignment: .leading, spacing: 6) {
if let title = self.preview.title?.trimmingCharacters(in: .whitespacesAndNewlines), !title.isEmpty {
Text(title)
.font(OpenClawChatTypography.footnote)
.fontWeight(.semibold)
.foregroundStyle(OpenClawChatTheme.muted)
}
#if canImport(WebKit) && (os(iOS) || os(macOS))
if let resolvedResource {
ChatInlineWidgetWebView(
resource: resolvedResource,
allowsScripts: self.preview.sandbox == "scripts",
onFailure: { self.handleLoadFailure(resource: resolvedResource) })
.id([
resolvedResource.url.absoluteString,
resolvedResource.tlsFingerprintSHA256 ?? "",
self.preview.sandbox ?? "",
].joined(separator: "\u{0}"))
.frame(height: self.preview.inlineWidgetHeight)
.clipShape(RoundedRectangle(cornerRadius: 10, style: .continuous))
.overlay {
RoundedRectangle(cornerRadius: 10, style: .continuous)
.stroke(OpenClawChatTheme.muted.opacity(0.24), lineWidth: 1)
}
} else if self.unavailable {
Text("Widget unavailable")
.font(OpenClawChatTypography.footnote)
.foregroundStyle(OpenClawChatTheme.muted)
} else {
ProgressView()
.controlSize(.small)
.frame(maxWidth: .infinity, minHeight: 44)
}
#else
Text("Widget unavailable")
.font(OpenClawChatTypography.footnote)
.foregroundStyle(OpenClawChatTheme.muted)
#endif
}
.task(id: LoadID(path: self.preview.inlineWidgetPath, resolverReady: self.resolverReady)) {
let path = self.preview.inlineWidgetPath
if self.activePath != path {
self.reset(path: path)
}
guard self.resolverReady else { return }
self.reset(path: path)
guard let path else {
self.unavailable = true
return
}
await self.load(path: path, replacing: nil, generation: self.loadGeneration)
}
}
private struct LoadID: Hashable {
let path: String?
let resolverReady: Bool
}
private func reset(path: String?) {
self.loadGeneration = UUID()
self.activePath = path
self.resolvedResource = nil
self.recoveryAttempts = 0
self.refreshInFlight = false
self.unavailable = false
}
private func load(
path: String,
replacing failedResource: OpenClawChatWidgetResource?,
generation: UUID) async
{
let candidate = await self.resolveResource(path, failedResource)
guard !Task.isCancelled,
self.activePath == path,
self.loadGeneration == generation
else { return }
let resource = candidate?.hasValidTLSBinding == true ? candidate : nil
self.resolvedResource = resource
self.unavailable = resource == nil
}
private func handleLoadFailure(resource: OpenClawChatWidgetResource) {
guard self.resolvedResource == resource,
let path = self.activePath,
!self.refreshInFlight
else { return }
guard self.recoveryAttempts < 3 else {
self.resolvedResource = nil
self.unavailable = true
return
}
self.recoveryAttempts += 1
self.refreshInFlight = true
let generation = self.loadGeneration
Task { @MainActor in
await self.load(path: path, replacing: resource, generation: generation)
guard self.activePath == path, self.loadGeneration == generation else { return }
self.refreshInFlight = false
}
}
}
extension OpenClawChatWidgetResource {
fileprivate var hasValidTLSBinding: Bool {
self.tlsFingerprintSHA256 == nil || self.url.scheme?.lowercased() == "https"
}
}
#if canImport(WebKit) && (os(iOS) || os(macOS))
enum ChatInlineWidgetTLSPin {
static func normalize(_ raw: String) -> String? {
let stripped = raw.replacingOccurrences(
of: #"(?i)^sha-?256\s*:?\s*"#,
with: "",
options: .regularExpression)
let normalized = stripped.lowercased().filter(\.isHexDigit)
return normalized.count == 64 ? normalized : nil
}
static func fingerprint(certificateData: Data) -> String {
SHA256.hash(data: certificateData).map { String(format: "%02x", $0) }.joined()
}
static func matches(_ expected: String, trust: SecTrust) -> Bool {
guard let expected = normalize(expected),
let chain = SecTrustCopyCertificateChain(trust) as? [SecCertificate],
let certificate = chain.first
else { return false }
return self.fingerprint(certificateData: SecCertificateCopyData(certificate) as Data) == expected
}
}
struct ChatInlineWidgetContentProcessRecovery {
enum Action: Equatable {
case reload
case fail
}
private var didReload = false
mutating func nextAction() -> Action {
guard !self.didReload else { return .fail }
self.didReload = true
return .reload
}
mutating func reset() {
self.didReload = false
}
}
@MainActor
private final class ChatInlineWidgetNavigationDelegate: NSObject, WKNavigationDelegate {
var resource: OpenClawChatWidgetResource {
didSet {
if self.resource != oldValue {
self.contentProcessRecovery.reset()
}
}
}
let onFailure: @MainActor @Sendable () -> Void
private var contentProcessRecovery = ChatInlineWidgetContentProcessRecovery()
init(resource: OpenClawChatWidgetResource, onFailure: @escaping @MainActor @Sendable () -> Void) {
self.resource = resource
self.onFailure = onFailure
}
func webView(
_: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
if navigationAction.targetFrame?.isMainFrame == false {
decisionHandler(.cancel)
return
}
guard navigationAction.request.httpMethod?.caseInsensitiveCompare("GET") == .orderedSame,
let url = navigationAction.request.url,
self.matchesExpectedDocument(url)
else {
decisionHandler(.cancel)
return
}
decisionHandler(.allow)
}
func webView(
_: WKWebView,
decidePolicyFor navigationResponse: WKNavigationResponse,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationResponsePolicy) -> Void)
{
if navigationResponse.isForMainFrame,
let response = navigationResponse.response as? HTTPURLResponse,
response.statusCode >= 400
{
self.onFailure()
decisionHandler(.cancel)
return
}
decisionHandler(.allow)
}
func webView(_: WKWebView, didFailProvisionalNavigation _: WKNavigation?, withError _: any Error) {
self.onFailure()
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError _: any Error) {
self.onFailure()
}
func webView(
_: WKWebView,
didReceive challenge: URLAuthenticationChallenge,
completionHandler: @escaping @MainActor @Sendable (
URLSession.AuthChallengeDisposition,
URLCredential?) -> Void)
{
guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust,
let expectedFingerprint = resource.tlsFingerprintSHA256
else {
completionHandler(.performDefaultHandling, nil)
return
}
guard self.matchesExpectedProtectionSpace(challenge.protectionSpace),
let trust = challenge.protectionSpace.serverTrust,
ChatInlineWidgetTLSPin.matches(expectedFingerprint, trust: trust)
else {
completionHandler(.cancelAuthenticationChallenge, nil)
self.onFailure()
return
}
completionHandler(.useCredential, URLCredential(trust: trust))
}
func webViewWebContentProcessDidTerminate(_ webView: WKWebView) {
// One same-document recovery handles incidental process loss. A second
// termination enters the view's bounded capability-refresh/failure path.
switch self.contentProcessRecovery.nextAction() {
case .reload:
webView.load(URLRequest(url: self.resource.url, cachePolicy: .reloadIgnoringLocalCacheData))
case .fail:
self.onFailure()
}
}
private func matchesExpectedDocument(_ candidate: URL) -> Bool {
guard var expected = URLComponents(url: self.resource.url, resolvingAgainstBaseURL: false),
var candidate = URLComponents(url: candidate, resolvingAgainstBaseURL: false)
else { return false }
expected.fragment = nil
candidate.fragment = nil
return expected == candidate
}
private func matchesExpectedProtectionSpace(_ protectionSpace: URLProtectionSpace) -> Bool {
guard let expectedHost = self.resource.url.host,
protectionSpace.host.caseInsensitiveCompare(expectedHost) == .orderedSame
else { return false }
let expectedPort = self.resource.url.port ?? (self.resource.url.scheme?.lowercased() == "https" ? 443 : 80)
return protectionSpace.port == expectedPort
}
}
@MainActor
private func makeChatInlineWidgetWebView(
resource: OpenClawChatWidgetResource,
allowsScripts: Bool,
coordinator: ChatInlineWidgetNavigationDelegate) -> WKWebView
{
let configuration = WKWebViewConfiguration()
configuration.websiteDataStore = .nonPersistent()
configuration.defaultWebpagePreferences.allowsContentJavaScript = allowsScripts
configuration.preferences.javaScriptCanOpenWindowsAutomatically = false
let webView = WKWebView(frame: .zero, configuration: configuration)
webView.navigationDelegate = coordinator
webView.allowsLinkPreview = false
webView.load(URLRequest(url: resource.url, cachePolicy: .reloadIgnoringLocalCacheData))
return webView
}
#if os(iOS)
private struct ChatInlineWidgetWebView: UIViewRepresentable {
let resource: OpenClawChatWidgetResource
let allowsScripts: Bool
let onFailure: @MainActor @Sendable () -> Void
func makeCoordinator() -> ChatInlineWidgetNavigationDelegate {
ChatInlineWidgetNavigationDelegate(resource: self.resource, onFailure: self.onFailure)
}
func makeUIView(context: Context) -> WKWebView {
makeChatInlineWidgetWebView(
resource: self.resource,
allowsScripts: self.allowsScripts,
coordinator: context.coordinator)
}
func updateUIView(_ webView: WKWebView, context: Context) {
guard context.coordinator.resource != self.resource else { return }
context.coordinator.resource = self.resource
webView.load(URLRequest(url: self.resource.url, cachePolicy: .reloadIgnoringLocalCacheData))
}
static func dismantleUIView(_ webView: WKWebView, coordinator: ChatInlineWidgetNavigationDelegate) {
webView.stopLoading()
webView.navigationDelegate = nil
}
}
#elseif os(macOS)
private struct ChatInlineWidgetWebView: NSViewRepresentable {
let resource: OpenClawChatWidgetResource
let allowsScripts: Bool
let onFailure: @MainActor @Sendable () -> Void
func makeCoordinator() -> ChatInlineWidgetNavigationDelegate {
ChatInlineWidgetNavigationDelegate(resource: self.resource, onFailure: self.onFailure)
}
func makeNSView(context: Context) -> WKWebView {
makeChatInlineWidgetWebView(
resource: self.resource,
allowsScripts: self.allowsScripts,
coordinator: context.coordinator)
}
func updateNSView(_ webView: WKWebView, context: Context) {
guard context.coordinator.resource != self.resource else { return }
context.coordinator.resource = self.resource
webView.load(URLRequest(url: self.resource.url, cachePolicy: .reloadIgnoringLocalCacheData))
}
static func dismantleNSView(_ webView: WKWebView, coordinator: ChatInlineWidgetNavigationDelegate) {
webView.stopLoading()
webView.navigationDelegate = nil
}
}
#endif
#endif
@@ -213,6 +213,10 @@ struct ChatMessageBubble: View {
let showsAssistantAvatar: Bool
let isClean: Bool
let contextWindowTokens: Int?
let inlineWidgetResolverReady: Bool
let inlineWidgetResourceResolver: @MainActor @Sendable (
String,
OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
var body: some View {
if self.isUser {
@@ -251,7 +255,9 @@ struct ChatMessageBubble: View {
userAccent: self.userAccent,
showsAssistantTrace: self.showsAssistantTrace,
isClean: self.isClean,
contextWindowTokens: self.contextWindowTokens)
contextWindowTokens: self.contextWindowTokens,
inlineWidgetResolverReady: self.inlineWidgetResolverReady,
inlineWidgetResourceResolver: self.inlineWidgetResourceResolver)
}
}
@@ -266,6 +272,10 @@ private struct ChatMessageBody: View {
let showsAssistantTrace: Bool
let isClean: Bool
let contextWindowTokens: Int?
let inlineWidgetResolverReady: Bool
let inlineWidgetResourceResolver: @MainActor @Sendable (
String,
OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
var body: some View {
let text = self.primaryText
@@ -325,6 +335,13 @@ private struct ChatMessageBody: View {
}
}
ForEach(self.inlineWidgets.indices, id: \.self) { idx in
ChatInlineWidgetView(
preview: self.inlineWidgets[idx],
resolverReady: self.inlineWidgetResolverReady,
resolveResource: self.inlineWidgetResourceResolver)
}
if self.showsAssistantTrace, !self.toolCalls.isEmpty {
ForEach(self.toolCalls.indices, id: \.self) { idx in
ToolCallCard(
@@ -394,6 +411,17 @@ private struct ChatMessageBody: View {
}
}
private var inlineWidgets: [OpenClawChatCanvasPreview] {
guard self.message.role.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() == "assistant"
else { return [] }
return self.message.content.compactMap { content in
guard (content.type ?? "").lowercased() == "canvas",
content.preview?.inlineWidgetPath != nil
else { return nil }
return content.preview
}
}
private var toolCalls: [OpenClawChatMessageContent] {
self.message.content.filter { content in
let kind = (content.type ?? "").lowercased()
@@ -118,6 +118,7 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
public let fileName: String?
public let durationSeconds: Double?
public let content: AnyCodable?
public let preview: OpenClawChatCanvasPreview?
// Tool-call fields (when `type == "toolCall"` or similar)
public let id: String?
@@ -133,6 +134,7 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
fileName: String?,
durationSeconds: Double? = nil,
content: AnyCodable?,
preview: OpenClawChatCanvasPreview? = nil,
id: String? = nil,
name: String? = nil,
arguments: AnyCodable? = nil)
@@ -145,6 +147,7 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
self.fileName = fileName
self.durationSeconds = durationSeconds
self.content = content
self.preview = preview
self.id = id
self.name = name
self.arguments = arguments
@@ -159,6 +162,7 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
case fileName
case durationSeconds
case content
case preview
case id
case name
case arguments
@@ -176,6 +180,7 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
self.id = try container.decodeIfPresent(String.self, forKey: .id)
self.name = try container.decodeIfPresent(String.self, forKey: .name)
self.arguments = try container.decodeIfPresent(AnyCodable.self, forKey: .arguments)
self.preview = try container.decodeIfPresent(OpenClawChatCanvasPreview.self, forKey: .preview)
if let any = try container.decodeIfPresent(AnyCodable.self, forKey: .content) {
self.content = any
@@ -187,6 +192,32 @@ public struct OpenClawChatMessageContent: Codable, Hashable, Sendable {
}
}
public struct OpenClawChatCanvasPreview: Codable, Hashable, Sendable {
public let kind: String?
public let surface: String?
public let render: String?
public let title: String?
public let preferredHeight: Double?
public let url: String?
public let viewId: String?
public let sandbox: String?
public var inlineWidgetPath: String? {
guard self.kind == "canvas",
self.surface == "assistant_message",
self.render == "url",
self.sandbox == "scripts" || self.sandbox == "strict",
let url = self.url?.trimmingCharacters(in: .whitespacesAndNewlines),
OpenClawChatWidgetURLResolver.supportsTarget(url)
else { return nil }
return url
}
public var inlineWidgetHeight: Double {
min(max(self.preferredHeight ?? 320, 160), 1200)
}
}
public struct OpenClawChatMessage: Codable, Hashable, Identifiable, Sendable {
private struct OpenClawMetadata: Codable {
let idempotencyKey: String?
@@ -290,6 +290,10 @@ public protocol OpenClawChatTransport: Sendable {
func requestHealth(timeoutMs: Int) async throws -> Bool
func waitForRunCompletion(runId: String, timeoutMs: Int) async -> OpenClawChatRunObservation
func events() -> AsyncStream<OpenClawChatTransportEvent>
func resolveInlineWidgetResource(
path: String,
replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
func resolveInlineWidgetURL(path: String, replacing failedURL: URL?) async -> URL?
func setActiveSessionKey(_ sessionKey: String) async throws
func resetSession(sessionKey: String) async throws
@@ -297,6 +301,18 @@ public protocol OpenClawChatTransport: Sendable {
}
extension OpenClawChatTransport {
public func resolveInlineWidgetResource(
path: String,
replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
{
guard let url = await resolveInlineWidgetURL(path: path, replacing: failedResource?.url) else { return nil }
return OpenClawChatWidgetResource(url: url)
}
public func resolveInlineWidgetURL(path _: String, replacing _: URL?) async -> URL? {
nil
}
public var outboxRequiresSessionRoutingContract: Bool {
false
}
@@ -435,7 +435,11 @@ public struct OpenClawChatView: View {
assistantAvatarTint: self.assistantAvatarTint,
showsAssistantAvatar: self.showsAssistantAvatars,
isClean: self.composerChrome == .clean,
contextWindowTokens: contextWindowTokens)
contextWindowTokens: contextWindowTokens,
inlineWidgetResolverReady: self.viewModel.healthOK,
inlineWidgetResourceResolver: { [weak viewModel] path, failedResource in
await viewModel?.resolveInlineWidgetResource(path: path, replacing: failedResource)
})
.frame(
maxWidth: .infinity,
alignment: msg.role.lowercased() == "user" ? .trailing : .leading)
@@ -14,6 +14,13 @@ private final class PendingRunOwnerReference {
}
extension OpenClawChatViewModel {
func resolveInlineWidgetResource(
path: String,
replacing failedResource: OpenClawChatWidgetResource?) async -> OpenClawChatWidgetResource?
{
await self.transport.resolveInlineWidgetResource(path: path, replacing: failedResource)
}
func handleTransportEvent(_ evt: OpenClawChatTransportEvent) {
switch evt {
case let .health(ok):
@@ -66,12 +66,8 @@ public enum AsyncTimeout {
operation: @escaping @Sendable () async throws -> T) async throws -> T
{
let clamped = max(0, seconds)
if clamped == 0 {
return try await operation()
}
// Unstructured racers let the caller return without awaiting a cancellation-ignoring loser.
// The actor resumes once and cancels both tasks; noncooperative work may still finish later,
// The actor resumes once and cancels every racer; noncooperative work may still finish later,
// so callers must own resource cleanup and stale-result safety.
let race = AsyncTimeoutRace<T>()
return try await withTaskCancellationHandler {
@@ -85,22 +81,25 @@ public enum AsyncTimeout {
await race.resolveFailure(error)
}
}
let timeoutTask = Task {
do {
try await Task.sleep(nanoseconds: UInt64(clamped * 1_000_000_000))
await race.resolveFailure(onTimeout())
} catch is CancellationError {
// The operation or caller resolved the race first.
} catch {
await race.resolveFailure(error)
var tasks = [operationTask]
if clamped > 0 {
let timeoutTask = Task {
do {
try await Task.sleep(nanoseconds: UInt64(clamped * 1_000_000_000))
await race.resolveFailure(onTimeout())
} catch is CancellationError {
// The operation or caller resolved the race first.
} catch {
await race.resolveFailure(error)
}
}
tasks.append(timeoutTask)
}
if Task.isCancelled {
operationTask.cancel()
timeoutTask.cancel()
tasks.forEach { $0.cancel() }
throw CancellationError()
}
return try await race.wait(for: [operationTask, timeoutTask])
return try await race.wait(for: tasks)
} onCancel: {
Task { await race.resolveFailure(CancellationError()) }
}
@@ -1,3 +1,7 @@
public enum OpenClawGatewayClientCapability {
public static let inlineWidgets = "inline-widgets"
}
public struct GatewayConnectOptions: Sendable {
public var role: String
public var scopes: [String]
@@ -16,37 +16,6 @@ private struct NodeInvokeCancelPayload: Codable {
var invokeId: String
}
func canonicalizeCanvasHostUrl(raw: String?, activeURL: URL?) -> String? {
let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !trimmed.isEmpty else { return nil }
guard var parsed = URLComponents(string: trimmed) else { return trimmed }
let parsedHost = parsed.host?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let parsedIsLoopback = !parsedHost.isEmpty && LoopbackHost.isLoopback(parsedHost)
if !parsedHost.isEmpty, !parsedIsLoopback {
guard let activeURL else { return trimmed }
let isTLS = activeURL.scheme?.lowercased() == "wss"
guard isTLS else { return trimmed }
parsed.scheme = "https"
if parsed.port == nil {
let tlsPort = activeURL.port ?? 443
parsed.port = (tlsPort == 443) ? nil : tlsPort
}
return parsed.string ?? trimmed
}
guard let activeURL, let fallbackHost = activeURL.host, !LoopbackHost.isLoopback(fallbackHost) else {
return trimmed
}
let isTLS = activeURL.scheme?.lowercased() == "wss"
parsed.scheme = isTLS ? "https" : "http"
parsed.host = fallbackHost
let fallbackPort = activeURL.port ?? (isTLS ? 443 : 80)
parsed.port = ((isTLS && fallbackPort == 443) || (!isTLS && fallbackPort == 80)) ? nil : fallbackPort
return parsed.string ?? trimmed
}
/// Binds suspended work to one installed gateway channel generation.
/// Callers use this lease so an actor hop cannot retarget a payload to a replacement gateway.
public struct GatewayNodeSessionRoute: Sendable, Equatable {
@@ -55,6 +24,17 @@ public struct GatewayNodeSessionRoute: Sendable, Equatable {
fileprivate let socketGeneration: UInt64
}
/// One capability-scoped Canvas URL and the transport trust bound to its route.
public struct GatewayCanvasHostRoute: Sendable, Equatable {
public let url: String
public let tlsFingerprintSHA256: String?
public init(url: String, tlsFingerprintSHA256: String?) {
self.url = url
self.tlsFingerprintSHA256 = tlsFingerprintSHA256
}
}
/// A route lease became stale before its request touched the channel. Unlike
/// a socket cancellation, this proves the payload was never dispatched.
public enum GatewayNodeSessionRequestError: Error, Sendable {
@@ -98,6 +78,7 @@ public struct GatewayNodeSessionCredentials: Sendable, Equatable {
public actor GatewayNodeSession {
@TaskLocal private static var executingLifecycleCallbackID: UUID?
private static let pluginSurfaceRefreshTimeoutMs = 8000.0
private static let staleRouteInvokeMessage = "UNAVAILABLE: node route changed before dispatch"
private enum ComputerInvokeReceiptState {
@@ -157,6 +138,7 @@ public actor GatewayNodeSession {
private var activeSessionIdentity: ObjectIdentifier?
private var channelGeneration: UInt64 = 0
private var admissionGeneration: UInt64 = 0
private var activeTLSRouteMetadataProvider: GatewayTLSRouteMetadataProviding?
// A delayed push keeps its physical socket epoch. Once disconnect cleanup
// retires that epoch, it cannot adopt the replacement route's admission.
private var activeSocketGeneration: UInt64?
@@ -281,6 +263,18 @@ public actor GatewayNodeSession {
private var serverEventSubscribers: [UUID: ServerEventSubscriber] = [:]
private var pluginSurfaceUrls: [String: String] = [:]
private struct PluginSurfaceRefresh {
let id: UUID
let channelGeneration: UInt64
let admissionGeneration: UInt64
let task: Task<String?, Never>
var waiterIDs: Set<UUID>
}
/// Surface tokens belong to the shared session. A second rotation can invalidate
/// the URL returned to another chat before its web view has loaded it.
private var pluginSurfaceRefreshes: [String: PluginSurfaceRefresh] = [:]
private struct PluginSurfaceRefreshResponse: Decodable {
let pluginSurfaceUrls: [String: AnyCodable]?
}
@@ -347,6 +341,7 @@ public actor GatewayNodeSession {
{
let nextOptionsKey = self.connectOptionsKey(connectOptions)
let nextSessionIdentity = sessionBox.map { ObjectIdentifier($0.session) }
let nextTLSRouteMetadataProvider = sessionBox?.session as? GatewayTLSRouteMetadataProviding
let shouldReconnect = self.activeURL != url ||
self.activeCredentials != credentials ||
self.activeConnectOptionsKey != nextOptionsKey ||
@@ -419,6 +414,7 @@ public actor GatewayNodeSession {
self.activeCredentials = credentials
self.activeConnectOptionsKey = nextOptionsKey
self.activeSessionIdentity = nextSessionIdentity
self.activeTLSRouteMetadataProvider = nextTLSRouteMetadataProvider
} else {
channelGeneration = self.channelGeneration
self.connectOptions = connectOptions
@@ -519,6 +515,7 @@ public actor GatewayNodeSession {
self.activeCredentials = nil
self.activeConnectOptionsKey = nil
self.activeSessionIdentity = nil
self.activeTLSRouteMetadataProvider = nil
self.connectOptions = nil
self.onConnected = nil
self.onDisconnected = nil
@@ -613,20 +610,148 @@ public actor GatewayNodeSession {
self.pluginSurfaceUrls["canvas"]
}
@discardableResult
public func refreshPluginSurfaceUrl(surface: String, timeoutSeconds: Int = 8) async -> String? {
guard let channel else { return nil }
let trimmedSurface = surface.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmedSurface.isEmpty else { return nil }
return await self.requestPluginSurfaceRefresh(
channel: channel,
method: "node.pluginSurface.refresh",
params: ["surface": AnyCodable(trimmedSurface)],
surface: trimmedSurface,
timeoutSeconds: timeoutSeconds)
public func currentCanvasHostRoute() -> GatewayCanvasHostRoute? {
guard let url = currentCanvasHostUrl() else { return nil }
return GatewayCanvasHostRoute(
url: url,
tlsFingerprintSHA256: GatewayPluginSurfaceURL.tlsFingerprintForSurface(
self.activeTLSRouteMetadataProvider?.effectiveTLSFingerprintSHA256,
surfaceURL: url,
gatewayURL: self.activeURL))
}
@discardableResult
public func refreshCanvasHostRoute(replacing observedURL: String?) async -> GatewayCanvasHostRoute? {
_ = await self.refreshCanvasHostUrl(replacing: observedURL)
// Re-read after the refresh suspension. A reconnect may have installed
// both a replacement capability and a different certificate pin.
return self.currentCanvasHostRoute()
}
@discardableResult
public func refreshPluginSurfaceUrl(
surface: String,
replacing observedURL: String?) async -> String?
{
await self.refreshPluginSurfaceUrl(
surface: surface,
observedURL: observedURL,
timeoutMs: Self.pluginSurfaceRefreshTimeoutMs)
}
// periphery:ignore - Shipped public refresh API; keep until a breaking OpenClawKit window.
@discardableResult
public func refreshPluginSurfaceUrl(surface: String, timeoutSeconds: Int = 8) async -> String? {
let trimmedSurface = surface.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmedSurface.isEmpty else { return nil }
return await self.refreshPluginSurfaceUrl(
surface: trimmedSurface,
observedURL: self.pluginSurfaceUrls[trimmedSurface],
timeoutMs: Double(timeoutSeconds) * 1000)
}
private func refreshPluginSurfaceUrl(
surface: String,
observedURL: String?,
timeoutMs: Double) async -> String?
{
guard let channel else { return nil }
guard let method = self.pluginSurfaceRefreshMethod() else { return nil }
let trimmedSurface = surface.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmedSurface.isEmpty else { return nil }
if self.pluginSurfaceUrls[trimmedSurface] != observedURL {
return self.pluginSurfaceUrls[trimmedSurface]
}
let channelGeneration = self.channelGeneration
let admissionGeneration = self.admissionGeneration
let waiterID = UUID()
let refresh: PluginSurfaceRefresh
if var activeRefresh = self.pluginSurfaceRefreshes[trimmedSurface],
activeRefresh.channelGeneration == channelGeneration,
activeRefresh.admissionGeneration == admissionGeneration
{
activeRefresh.waiterIDs.insert(waiterID)
self.pluginSurfaceRefreshes[trimmedSurface] = activeRefresh
refresh = activeRefresh
} else {
// A reconnect retires the old channel owner. Do not leave its
// deadline-free request alive after installing the new generation.
self.pluginSurfaceRefreshes.removeValue(forKey: trimmedSurface)?.task.cancel()
let id = UUID()
let task = Task<String?, Never> { [weak self] in
guard let self else { return nil }
return await self.requestPluginSurfaceRefresh(
channel: channel,
channelGeneration: channelGeneration,
admissionGeneration: admissionGeneration,
method: method,
surface: trimmedSurface,
observedURL: observedURL)
}
refresh = PluginSurfaceRefresh(
id: id,
channelGeneration: channelGeneration,
admissionGeneration: admissionGeneration,
task: task,
waiterIDs: [waiterID])
self.pluginSurfaceRefreshes[trimmedSurface] = refresh
}
let value = await withTaskCancellationHandler {
await self.awaitPluginSurfaceRefresh(refresh.task, timeoutMs: timeoutMs)
} onCancel: {
Task {
await self.releasePluginSurfaceRefreshWaiter(
surface: trimmedSurface,
refreshID: refresh.id,
waiterID: waiterID)
}
}
self.releasePluginSurfaceRefreshWaiter(
surface: trimmedSurface,
refreshID: refresh.id,
waiterID: waiterID)
return value
}
private func awaitPluginSurfaceRefresh(_ task: Task<String?, Never>, timeoutMs: Double) async -> String? {
do {
return try await AsyncTimeout.withTimeout(
seconds: max(0, timeoutMs) / 1000,
onTimeout: {
NSError(
domain: "Gateway",
code: 8,
userInfo: [NSLocalizedDescriptionKey: "plugin surface refresh timed out"])
},
operation: { await task.value })
} catch {
return nil
}
}
private func releasePluginSurfaceRefreshWaiter(surface: String, refreshID: UUID, waiterID: UUID) {
guard var refresh = self.pluginSurfaceRefreshes[surface], refresh.id == refreshID else { return }
guard refresh.waiterIDs.remove(waiterID) != nil else { return }
if refresh.waiterIDs.isEmpty {
// The request itself has no deadline because callers may select different
// timeouts. The last waiter cancels locally; observedUrl makes any retry
// reuse an on-wire predecessor's rotation instead of invalidating it.
self.pluginSurfaceRefreshes[surface] = nil
refresh.task.cancel()
} else {
self.pluginSurfaceRefreshes[surface] = refresh
}
}
@discardableResult
public func refreshCanvasHostUrl(
replacing observedURL: String?) async -> String?
{
await self.refreshPluginSurfaceUrl(surface: "canvas", replacing: observedURL)
}
// periphery:ignore - Shipped public refresh API; keep until a breaking OpenClawKit window.
@discardableResult
public func refreshCanvasHostUrl(timeoutSeconds: Int = 8) async -> String? {
await self.refreshPluginSurfaceUrl(surface: "canvas", timeoutSeconds: timeoutSeconds)
@@ -959,7 +1084,7 @@ extension GatewayNodeSession {
}
private func normalizeCanvasHostUrl(_ raw: String?) -> String? {
canonicalizeCanvasHostUrl(raw: raw, activeURL: self.activeURL)
GatewayPluginSurfaceURL.canonicalize(raw: raw, against: self.activeURL)
}
private func normalizePluginSurfaceUrls(_ raw: [String: AnyCodable]?) -> [String: String] {
@@ -972,21 +1097,45 @@ extension GatewayNodeSession {
return normalized
}
private func pluginSurfaceRefreshMethod() -> String? {
switch self.connectOptions?.role.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
case "node": "node.pluginSurface.refresh"
case "operator": "plugin.surface.refresh"
default: nil
}
}
private func requestPluginSurfaceRefresh(
channel: GatewayChannelActor,
channelGeneration: UInt64,
admissionGeneration: UInt64,
method: String,
params: [String: AnyCodable]?,
surface: String,
timeoutSeconds: Int) async -> String?
observedURL: String?) async -> String?
{
do {
// Waiters own independent deadlines around this shared request. Zero
// leaves its lifetime unbounded; the last waiter cancels channel work.
var params = ["surface": AnyCodable(surface)]
if let observedURL {
// The gateway compares capability tokens, not hosts, because
// native clients rewrite loopback surface URLs for remote access.
params["observedUrl"] = AnyCodable(observedURL)
}
let data = try await channel.request(
method: method,
params: params,
timeoutMs: Double(timeoutSeconds * 1000))
timeoutMs: 0)
let decoded = try decoder.decode(PluginSurfaceRefreshResponse.self, from: data)
let urls = self.normalizePluginSurfaceUrls(decoded.pluginSurfaceUrls)
guard let refreshed = urls[surface] else { return nil }
guard self.channel === channel,
self.channelGeneration == channelGeneration,
self.admissionGeneration == admissionGeneration
else { return nil }
if self.pluginSurfaceUrls[surface] != observedURL {
return self.pluginSurfaceUrls[surface]
}
self.pluginSurfaceUrls[surface] = refreshed
return refreshed
} catch {
@@ -1214,6 +1363,7 @@ extension GatewayNodeSession {
func _test_broadcastServerEvent(_ event: EventFrame) {
self.broadcastServerEvent(event)
}
#endif
private func cancelActiveInvokes(
@@ -0,0 +1,57 @@
import Foundation
public enum GatewayPluginSurfaceURL {
public static func canonicalize(raw: String?, against activeGatewayURL: URL?) -> String? {
let trimmed = raw?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !trimmed.isEmpty else { return nil }
guard var parsed = URLComponents(string: trimmed) else { return trimmed }
let parsedHost = parsed.host?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let parsedIsLoopback = !parsedHost.isEmpty && LoopbackHost.isLoopback(parsedHost)
if !parsedHost.isEmpty, !parsedIsLoopback {
guard let activeGatewayURL else { return trimmed }
let isTLS = activeGatewayURL.scheme?.lowercased() == "wss"
guard isTLS else { return trimmed }
parsed.scheme = "https"
if parsed.port == nil {
let tlsPort = activeGatewayURL.port ?? 443
parsed.port = (tlsPort == 443) ? nil : tlsPort
}
return parsed.string ?? trimmed
}
guard let activeGatewayURL,
let fallbackHost = activeGatewayURL.host,
!LoopbackHost.isLoopback(fallbackHost)
else { return trimmed }
let isTLS = activeGatewayURL.scheme?.lowercased() == "wss"
parsed.scheme = isTLS ? "https" : "http"
parsed.host = fallbackHost
let fallbackPort = activeGatewayURL.port ?? (isTLS ? 443 : 80)
parsed.port = ((isTLS && fallbackPort == 443) || (!isTLS && fallbackPort == 80)) ? nil : fallbackPort
return parsed.string ?? trimmed
}
static func tlsFingerprintForSurface(
_ fingerprint: String?,
surfaceURL: String,
gatewayURL: URL?) -> String?
{
guard let fingerprint,
let gatewayURL,
gatewayURL.scheme?.lowercased() == "wss",
let surface = URLComponents(string: surfaceURL),
surface.scheme?.lowercased() == "https",
self.normalizedEndpointHost(surface.host) == self.normalizedEndpointHost(gatewayURL.host),
(surface.port ?? 443) == (gatewayURL.port ?? 443)
else { return nil }
return fingerprint
}
private static func normalizedEndpointHost(_ raw: String?) -> String? {
let host = raw?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() ?? ""
let normalized = host.hasSuffix(".") ? String(host.dropLast()) : host
return normalized.isEmpty ? nil : normalized
}
}
@@ -214,11 +214,18 @@ public enum GatewayTLSStore {
}
}
protocol GatewayTLSRouteMetadataProviding: AnyObject {
var effectiveTLSFingerprintSHA256: String? { get }
}
public final class GatewayTLSPinningSession: NSObject, WebSocketSessioning, URLSessionDelegate,
GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, @unchecked Sendable {
GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, GatewayTLSRouteMetadataProviding,
@unchecked Sendable
{
private let params: GatewayTLSParams
private let failureLock = NSLock()
private var lastTLSFailure: GatewayTLSValidationFailure?
private var acceptedTLSFingerprintSHA256: String?
private lazy var session: URLSession = {
let config = URLSessionConfiguration.default
config.waitsForConnectivity = true
@@ -227,6 +234,9 @@ GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, @unchecked Se
public init(params: GatewayTLSParams) {
self.params = params
self.acceptedTLSFingerprintSHA256 = params.expectedFingerprint
.map(normalizeFingerprint)
.flatMap { $0.count == 64 ? $0 : nil }
super.init()
}
@@ -234,6 +244,12 @@ GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, @unchecked Se
self.params.expectedFingerprint?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false
}
var effectiveTLSFingerprintSHA256: String? {
self.failureLock.lock()
defer { self.failureLock.unlock() }
return self.acceptedTLSFingerprintSHA256
}
public func consumeLastTLSFailure() -> GatewayTLSValidationFailure? {
self.failureLock.lock()
defer { self.failureLock.unlock() }
@@ -248,9 +264,12 @@ GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, @unchecked Se
self.failureLock.unlock()
}
private func clearTLSFailure() {
private func recordTLSAcceptance(_ fingerprint: String?) {
self.failureLock.lock()
self.lastTLSFailure = nil
if let fingerprint {
self.acceptedTLSFingerprintSHA256 = fingerprint
}
self.failureLock.unlock()
}
@@ -283,7 +302,7 @@ GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, @unchecked Se
if let fingerprint {
if let expected {
if fingerprint == expected {
self.clearTLSFailure()
self.recordTLSAcceptance(fingerprint)
completionHandler(.useCredential, URLCredential(trust: trust))
} else {
self.recordTLSFailure(GatewayTLSValidationFailure(
@@ -302,7 +321,7 @@ GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, @unchecked Se
if let storeKey = params.storeKey {
GatewayTLSStore.saveFingerprint(fingerprint, stableID: storeKey)
}
self.clearTLSFailure()
self.recordTLSAcceptance(fingerprint)
completionHandler(.useCredential, URLCredential(trust: trust))
return
}
@@ -310,7 +329,7 @@ GatewayTLSFailureProviding, GatewayDeviceTokenRetryTrustProviding, @unchecked Se
}
if systemTrustOk || !self.params.required {
self.clearTLSFailure()
self.recordTLSAcceptance(fingerprint)
completionHandler(.useCredential, URLCredential(trust: trust))
} else {
self.recordTLSFailure(GatewayTLSValidationFailure(
@@ -0,0 +1,261 @@
import Foundation
import OpenClawKit
import Testing
@testable import OpenClawChatUI
struct ChatInlineWidgetTests {
@Test func `decodes projected canvas widget block`() throws {
let data = Data(
#"{"role":"assistant","content":[{"type":"text","text":"Done"},{"type":"canvas","preview":{"kind":"canvas","surface":"assistant_message","render":"url","title":"Status","preferredHeight":240,"url":"/__openclaw__/canvas/documents/widget-1/index.html","sandbox":"scripts"}}]}"#
.utf8)
let message = try JSONDecoder().decode(OpenClawChatMessage.self, from: data)
let preview = try #require(message.content.last?.preview)
#expect(preview.inlineWidgetPath == "/__openclaw__/canvas/documents/widget-1/index.html")
#expect(preview.inlineWidgetHeight == 240)
let unsafe = OpenClawChatCanvasPreview(
kind: "canvas",
surface: "assistant_message",
render: "url",
title: nil,
preferredHeight: nil,
url: "https://attacker.example/widget.html",
viewId: nil,
sandbox: "scripts")
#expect(unsafe.inlineWidgetPath == nil)
}
@Test func `resolves only capability scoped widget documents`() {
let surface = "https://gateway.example/__openclaw__/cap/token"
let target = "/__openclaw__/canvas/documents/widget-1/index.html"
#expect(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: surface,
target: target)?.absoluteString ==
"https://gateway.example/__openclaw__/cap/token/__openclaw__/canvas/documents/widget-1/index.html")
#expect(OpenClawChatWidgetURLResolver.resolve(surfaceURL: "https://gateway.example", target: target) == nil)
#expect(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: surface,
target: "https://attacker.example/widget.html") == nil)
#expect(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: surface,
target: "/__openclaw__/a2ui/index.html") == nil)
#expect(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: surface,
target: "/__openclaw__/canvas/documents/%252e%252e/index.html") == nil)
#expect(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: surface,
target: "/__openclaw__/canvas/documents/%2525252525252525252525252525252525/index.html") == nil)
}
@Test func `resource equality ignores internal recovery metadata`() throws {
let url = try #require(URL(string: "https://gateway.example/widget"))
let publicResource = OpenClawChatWidgetResource(url: url)
let trackedResource = OpenClawChatWidgetResource(
url: url,
tlsFingerprintSHA256: nil,
surfaceRole: .node,
attemptedSurfaceRoles: [.node])
#expect(publicResource == trackedResource)
}
@Test func `uses replacement route after capability refresh loses its lease`() async throws {
let target = "/__openclaw__/canvas/documents/widget-1/index.html"
let oldSurface = "https://gateway.example/__openclaw__/cap/old"
let newSurface = "https://gateway.example/__openclaw__/cap/new"
let failedURL = try #require(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: oldSurface,
target: target))
let failedResource = OpenClawChatWidgetResource(url: failedURL)
let probe = ChatWidgetRouteReconnectProbe(
route: GatewayCanvasHostRoute(url: oldSurface, tlsFingerprintSHA256: nil),
replacement: GatewayCanvasHostRoute(url: newSurface, tlsFingerprintSHA256: nil))
let resolved = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: failedResource,
currentSurfaceRoutes: { await probe.current() },
refreshNodeSurfaceRoute: { observed in await probe.reconnect(observed: observed) },
refreshOperatorSurfaceRoute: { _ in nil })
#expect(resolved?.url == OpenClawChatWidgetURLResolver.resolve(surfaceURL: newSurface, target: target))
#expect(await probe.refreshCount == 1)
}
@Test func `accepts a same URL widget route when it acquires a TLS pin`() async throws {
let target = "/__openclaw__/canvas/documents/widget-1/index.html"
let surface = "https://gateway.example/__openclaw__/cap/token"
let newPin = String(repeating: "bb", count: 32)
let failedURL = try #require(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: surface,
target: target))
let probe = ChatWidgetRouteReconnectProbe(
route: GatewayCanvasHostRoute(url: surface, tlsFingerprintSHA256: nil),
replacement: GatewayCanvasHostRoute(url: surface, tlsFingerprintSHA256: newPin))
let initialResource = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: nil,
currentSurfaceRoutes: { await probe.current() },
refreshNodeSurfaceRoute: { _ in nil },
refreshOperatorSurfaceRoute: { _ in nil })
let resolved = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: initialResource,
currentSurfaceRoutes: { await probe.current() },
refreshNodeSurfaceRoute: { observed in await probe.reconnect(observed: observed) },
refreshOperatorSurfaceRoute: { _ in nil })
#expect(resolved?.url == failedURL)
#expect(resolved?.tlsFingerprintSHA256 == newPin)
#expect(await probe.refreshCount == 1)
}
@Test func `refreshes node once before trying the operator fallback`() async {
let target = "/__openclaw__/canvas/documents/widget-1/index.html"
let oldSurface = "https://gateway.example/__openclaw__/cap/old"
let newSurface = "https://gateway.example/__openclaw__/cap/new"
let fallbackSurface = "https://operator.example/__openclaw__/cap/fallback"
let probe = ChatWidgetRouteReconnectProbe(
route: GatewayCanvasHostRoute(url: oldSurface, tlsFingerprintSHA256: nil),
replacement: GatewayCanvasHostRoute(url: newSurface, tlsFingerprintSHA256: nil),
operatorRoute: GatewayCanvasHostRoute(url: fallbackSurface, tlsFingerprintSHA256: nil))
let initialNode = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: nil,
currentSurfaceRoutes: { await probe.current() },
refreshNodeSurfaceRoute: { _ in nil },
refreshOperatorSurfaceRoute: { _ in nil })
let refreshedNode = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: initialNode,
currentSurfaceRoutes: { await probe.current() },
refreshNodeSurfaceRoute: { observed in await probe.reconnect(observed: observed) },
refreshOperatorSurfaceRoute: { _ in nil })
#expect(refreshedNode?.url == OpenClawChatWidgetURLResolver.resolve(surfaceURL: newSurface, target: target))
let fallback = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: refreshedNode,
currentSurfaceRoutes: { await probe.current() },
refreshNodeSurfaceRoute: { observed in await probe.reconnect(observed: observed) },
refreshOperatorSurfaceRoute: { _ in nil })
#expect(fallback?.url == OpenClawChatWidgetURLResolver.resolve(
surfaceURL: fallbackSurface,
target: target))
#expect(await probe.refreshCount == 1)
}
@Test func `refreshes the operator capability when the node route is unavailable`() async throws {
let target = "/__openclaw__/canvas/documents/widget-1/index.html"
let oldSurface = "https://operator.example/__openclaw__/cap/old"
let newSurface = "https://operator.example/__openclaw__/cap/new"
let failedURL = try #require(OpenClawChatWidgetURLResolver.resolve(
surfaceURL: oldSurface,
target: target))
let failedResource = OpenClawChatWidgetResource(url: failedURL)
let probe = ChatWidgetOperatorRouteRefreshProbe(
route: GatewayCanvasHostRoute(url: oldSurface, tlsFingerprintSHA256: nil),
replacement: GatewayCanvasHostRoute(url: newSurface, tlsFingerprintSHA256: nil))
let resolved = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: failedResource,
currentSurfaceRoutes: { await probe.current() },
refreshNodeSurfaceRoute: { _ in nil },
refreshOperatorSurfaceRoute: { observed in await probe.refresh(observed: observed) })
#expect(resolved?.url == OpenClawChatWidgetURLResolver.resolve(surfaceURL: newSurface, target: target))
#expect(await probe.refreshCount == 1)
}
@Test func `rejects a TLS pin on a cleartext widget route`() async {
let target = "/__openclaw__/canvas/documents/widget-1/index.html"
let route = GatewayCanvasHostRoute(
url: "http://gateway.example/__openclaw__/cap/token",
tlsFingerprintSHA256: String(repeating: "aa", count: 32))
let resolved = await OpenClawChatWidgetURLResolver.resolveResource(
target: target,
replacing: nil,
currentSurfaceRoutes: { (node: route, operatorSurface: nil) },
refreshNodeSurfaceRoute: { _ in nil },
refreshOperatorSurfaceRoute: { _ in nil })
#expect(resolved == nil)
}
#if canImport(WebKit) && (os(iOS) || os(macOS))
@Test func `bounds WebKit content process recovery per document`() {
var recovery = ChatInlineWidgetContentProcessRecovery()
#expect(recovery.nextAction() == .reload)
#expect(recovery.nextAction() == .fail)
#expect(recovery.nextAction() == .fail)
recovery.reset()
#expect(recovery.nextAction() == .reload)
}
@Test func `normalizes exact SHA-256 widget certificate pins`() {
let fingerprint = String(repeating: "aB", count: 32)
#expect(ChatInlineWidgetTLSPin.normalize("SHA-256: \(fingerprint)") == fingerprint.lowercased())
#expect(ChatInlineWidgetTLSPin.normalize("abc") == nil)
#expect(ChatInlineWidgetTLSPin.fingerprint(certificateData: Data("abc".utf8)) ==
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
}
#endif
}
private actor ChatWidgetOperatorRouteRefreshProbe {
private var route: GatewayCanvasHostRoute
private let replacement: GatewayCanvasHostRoute
private(set) var refreshCount = 0
init(route: GatewayCanvasHostRoute, replacement: GatewayCanvasHostRoute) {
self.route = route
self.replacement = replacement
}
func current() -> (node: GatewayCanvasHostRoute?, operatorSurface: GatewayCanvasHostRoute?) {
(node: nil, operatorSurface: self.route)
}
func refresh(observed _: GatewayCanvasHostRoute?) -> GatewayCanvasHostRoute? {
self.refreshCount += 1
self.route = self.replacement
return self.replacement
}
}
private actor ChatWidgetRouteReconnectProbe {
private var route: GatewayCanvasHostRoute?
private let replacement: GatewayCanvasHostRoute
private let operatorRoute: GatewayCanvasHostRoute?
private(set) var refreshCount = 0
init(
route: GatewayCanvasHostRoute,
replacement: GatewayCanvasHostRoute,
operatorRoute: GatewayCanvasHostRoute? = nil)
{
self.route = route
self.replacement = replacement
self.operatorRoute = operatorRoute
}
func current() -> (node: GatewayCanvasHostRoute?, operatorSurface: GatewayCanvasHostRoute?) {
(node: self.route, operatorSurface: self.operatorRoute)
}
func reconnect(observed _: GatewayCanvasHostRoute?) -> GatewayCanvasHostRoute? {
self.refreshCount += 1
self.route = self.replacement
return nil
}
}
@@ -301,6 +301,24 @@ private final class FakeGatewayWebSocketTask: WebSocketTasking, @unchecked Senda
idempotencyKey: idempotencyKey))))
}
func emitResponse(id: String, payload: [String: Any]) {
let handler = self.lock.withLock { () -> (@Sendable (Result<
URLSessionWebSocketTask.Message,
Error,
>) -> Void)? in
defer { self.pendingReceiveHandler = nil }
return self.pendingReceiveHandler
}
let frame: [String: Any] = [
"type": "res",
"id": id,
"ok": true,
"payload": payload,
]
let data = (try? JSONSerialization.data(withJSONObject: frame)) ?? Data()
handler?(.success(.data(data)))
}
private static func connectChallengeData(nonce: String) -> Data {
let frame: [String: Any] = [
"type": "event",
@@ -401,7 +419,9 @@ private final class FakeGatewayWebSocketTask: WebSocketTasking, @unchecked Senda
}
}
private final class FakeGatewayWebSocketSession: WebSocketSessioning, @unchecked Sendable {
private final class FakeGatewayWebSocketSession: WebSocketSessioning, GatewayTLSRouteMetadataProviding,
@unchecked Sendable
{
private let lock = NSLock()
private let helloAuth: [String: Any]?
private let helloMethods: [String]
@@ -409,6 +429,7 @@ private final class FakeGatewayWebSocketSession: WebSocketSessioning, @unchecked
private let helloDelayNanoseconds: UInt64
private let connectError: [String: Any]?
private let cancelGate: FirstCancelGate?
let effectiveTLSFingerprintSHA256: String?
private var tasks: [FakeGatewayWebSocketTask] = []
private var requests: [URLRequest] = []
private var makeCount = 0
@@ -419,7 +440,8 @@ private final class FakeGatewayWebSocketSession: WebSocketSessioning, @unchecked
helloSessionDefaults: [String: Any]? = nil,
helloDelayNanoseconds: UInt64 = 0,
connectError: [String: Any]? = nil,
cancelGate: FirstCancelGate? = nil)
cancelGate: FirstCancelGate? = nil,
effectiveTLSFingerprintSHA256: String? = nil)
{
self.helloAuth = helloAuth
self.helloMethods = helloMethods
@@ -427,6 +449,7 @@ private final class FakeGatewayWebSocketSession: WebSocketSessioning, @unchecked
self.helloDelayNanoseconds = helloDelayNanoseconds
self.connectError = connectError
self.cancelGate = cancelGate
self.effectiveTLSFingerprintSHA256 = effectiveTLSFingerprintSHA256
}
func snapshotMakeCount() -> Int {
@@ -604,6 +627,306 @@ private func nodeInvokePush(id: String, command: String) -> GatewayPush {
@Suite(.serialized)
struct GatewayNodeSessionTests {
@Test func `operator canvas refresh uses the operator surface method`() async throws {
let session = FakeGatewayWebSocketSession()
let gateway = GatewayNodeSession()
let options = GatewayConnectOptions(
role: "operator",
scopes: ["operator.read"],
caps: [OpenClawGatewayClientCapability.inlineWidgets],
commands: [],
permissions: [:],
clientId: "openclaw-ios",
clientMode: "ui",
clientDisplayName: "iOS Test",
includeDeviceIdentity: false)
try await gateway.connect(
url: #require(URL(string: "ws://gateway.example.invalid")),
connectOptions: options,
sessionBox: WebSocketSessionBox(session: session),
onConnected: {},
onDisconnected: { _ in },
onInvoke: { request in BridgeInvokeResponse(id: request.id, ok: true) })
async let refreshed = gateway.refreshCanvasHostUrl(replacing: nil)
try await waitUntil("operator surface refresh sent") {
session.latestTask()?.sentRequestCount(method: "plugin.surface.refresh") == 1
}
let task = try #require(session.latestTask())
let request = try #require(task.sentRequests(method: "plugin.surface.refresh").first)
try task.emitResponse(
id: #require(request["id"] as? String),
payload: [
"surface": "canvas",
"pluginSurfaceUrls": [
"canvas": "http://gateway.example.invalid/__openclaw__/cap/operator-token",
],
])
#expect(await refreshed?.hasSuffix("/operator-token") == true)
#expect(task.sentRequestCount(method: "node.pluginSurface.refresh") == 0)
await gateway.disconnect()
}
@Test func `canvas surface refresh is shared across callers with different timeouts`() async throws {
let expectedFingerprint = String(repeating: "ab", count: 32)
let session = FakeGatewayWebSocketSession(effectiveTLSFingerprintSHA256: expectedFingerprint)
let gateway = GatewayNodeSession()
let options = GatewayConnectOptions(
role: "node",
scopes: [],
caps: ["canvas"],
commands: [],
permissions: [:],
clientId: "openclaw-macos",
clientMode: "node",
clientDisplayName: "macOS Test",
includeDeviceIdentity: false)
try await gateway.connect(
url: #require(URL(string: "wss://gateway.example.invalid")),
connectOptions: options,
sessionBox: WebSocketSessionBox(session: session),
onConnected: {},
onDisconnected: { _ in },
onInvoke: { request in BridgeInvokeResponse(id: request.id, ok: true) })
async let first = gateway.refreshCanvasHostUrl(replacing: nil)
async let second = gateway.refreshCanvasHostUrl(timeoutSeconds: 1)
async let third = gateway.refreshPluginSurfaceUrl(surface: "canvas", timeoutSeconds: 2)
try await waitUntil("single surface refresh sent") {
session.latestTask()?.sentRequestCount(method: "node.pluginSurface.refresh") == 1
}
let task = try #require(session.latestTask())
let request = try #require(task.sentRequests(method: "node.pluginSurface.refresh").first)
let requestID = try #require(request["id"] as? String)
task.emitResponse(
id: requestID,
payload: [
"surface": "canvas",
"pluginSurfaceUrls": [
"canvas": "http://gateway.example.invalid/__openclaw__/cap/new-token",
],
])
let values = await (first, second, third)
#expect(values.0 == values.1)
#expect(values.0 == values.2)
#expect(values.0?.hasSuffix("/new-token") == true)
#expect(task.sentRequestCount(method: "node.pluginSurface.refresh") == 1)
let route = try #require(await gateway.currentCanvasHostRoute())
#expect(route.url == values.0)
#expect(route.tlsFingerprintSHA256 == expectedFingerprint)
await gateway.disconnect()
}
@Test func `timed out surface caller does not cancel rotation with longer waiter`() async throws {
let session = FakeGatewayWebSocketSession()
let gateway = GatewayNodeSession()
let options = GatewayConnectOptions(
role: "node",
scopes: [],
caps: ["canvas"],
commands: [],
permissions: [:],
clientId: "openclaw-macos",
clientMode: "node",
clientDisplayName: "macOS Test",
includeDeviceIdentity: false)
try await gateway.connect(
url: #require(URL(string: "ws://gateway.example.invalid")),
connectOptions: options,
sessionBox: WebSocketSessionBox(session: session),
onConnected: {},
onDisconnected: { _ in },
onInvoke: { request in BridgeInvokeResponse(id: request.id, ok: true) })
async let shortWait = gateway.refreshCanvasHostUrl(timeoutSeconds: 1)
try await waitUntil("single surface refresh sent") {
session.latestTask()?.sentRequestCount(method: "node.pluginSurface.refresh") == 1
}
async let longWait = gateway.refreshCanvasHostUrl(replacing: nil)
let shortValue = await shortWait
#expect(shortValue == nil)
async let joinedWait = gateway.refreshPluginSurfaceUrl(surface: "canvas", timeoutSeconds: 8)
let task = try #require(session.latestTask())
#expect(task.sentRequestCount(method: "node.pluginSurface.refresh") == 1)
let request = try #require(task.sentRequests(method: "node.pluginSurface.refresh").first)
try task.emitResponse(
id: #require(request["id"] as? String),
payload: [
"surface": "canvas",
"pluginSurfaceUrls": [
"canvas": "http://gateway.example.invalid/__openclaw__/cap/new-token",
],
])
let values = await (longWait, joinedWait)
#expect(values.0 == values.1)
#expect(values.0?.hasSuffix("/new-token") == true)
#expect(task.sentRequestCount(method: "node.pluginSurface.refresh") == 1)
await gateway.disconnect()
}
@Test func `last timed out surface waiter releases stalled rotation for retry`() async throws {
let session = FakeGatewayWebSocketSession()
let gateway = GatewayNodeSession()
let options = GatewayConnectOptions(
role: "node",
scopes: [],
caps: ["canvas"],
commands: [],
permissions: [:],
clientId: "openclaw-macos",
clientMode: "node",
clientDisplayName: "macOS Test",
includeDeviceIdentity: false)
try await gateway.connect(
url: #require(URL(string: "ws://gateway.example.invalid")),
connectOptions: options,
sessionBox: WebSocketSessionBox(session: session),
onConnected: {},
onDisconnected: { _ in },
onInvoke: { request in BridgeInvokeResponse(id: request.id, ok: true) })
let first = await gateway.refreshCanvasHostUrl(timeoutSeconds: 1)
#expect(first == nil)
async let retry = gateway.refreshCanvasHostUrl(timeoutSeconds: 1)
let task = try #require(session.latestTask())
try await waitUntil("second surface refresh sent") {
task.sentRequestCount(method: "node.pluginSurface.refresh") == 2
}
let request = try #require(task.sentRequests(method: "node.pluginSurface.refresh").last)
try task.emitResponse(
id: #require(request["id"] as? String),
payload: [
"surface": "canvas",
"pluginSurfaceUrls": [
"canvas": "http://gateway.example.invalid/__openclaw__/cap/retry-token",
],
])
let retryValue = await retry
#expect(retryValue?.hasSuffix("/retry-token") == true)
await gateway.disconnect()
}
@Test func `cancelled unbounded surface waiter releases stalled rotation for retry`() async throws {
let session = FakeGatewayWebSocketSession()
let gateway = GatewayNodeSession()
let options = GatewayConnectOptions(
role: "node",
scopes: [],
caps: ["canvas"],
commands: [],
permissions: [:],
clientId: "openclaw-macos",
clientMode: "node",
clientDisplayName: "macOS Test",
includeDeviceIdentity: false)
try await gateway.connect(
url: #require(URL(string: "ws://gateway.example.invalid")),
connectOptions: options,
sessionBox: WebSocketSessionBox(session: session),
onConnected: {},
onDisconnected: { _ in },
onInvoke: { request in BridgeInvokeResponse(id: request.id, ok: true) })
let stalled = Task { await gateway.refreshCanvasHostUrl(timeoutSeconds: 0) }
try await waitUntil("first surface refresh sent") {
session.latestTask()?.sentRequestCount(method: "node.pluginSurface.refresh") == 1
}
stalled.cancel()
#expect(await stalled.value == nil)
async let retry = gateway.refreshCanvasHostUrl(timeoutSeconds: 1)
let task = try #require(session.latestTask())
try await waitUntil("second surface refresh sent") {
task.sentRequestCount(method: "node.pluginSurface.refresh") == 2
}
let request = try #require(task.sentRequests(method: "node.pluginSurface.refresh").last)
try task.emitResponse(
id: #require(request["id"] as? String),
payload: [
"surface": "canvas",
"pluginSurfaceUrls": [
"canvas": "http://gateway.example.invalid/__openclaw__/cap/retry-token",
],
])
let retryValue = await retry
#expect(retryValue?.hasSuffix("/retry-token") == true)
await gateway.disconnect()
}
@Test func `lagging canvas refresh reuses the rotated capability`() async throws {
let session = FakeGatewayWebSocketSession()
let gateway = GatewayNodeSession()
let options = GatewayConnectOptions(
role: "node",
scopes: [],
caps: ["canvas"],
commands: [],
permissions: [:],
clientId: "openclaw-macos",
clientMode: "node",
clientDisplayName: "macOS Test",
includeDeviceIdentity: false)
try await gateway.connect(
url: #require(URL(string: "ws://gateway.example.invalid")),
connectOptions: options,
sessionBox: WebSocketSessionBox(session: session),
onConnected: {},
onDisconnected: { _ in },
onInvoke: { request in BridgeInvokeResponse(id: request.id, ok: true) })
async let seeded = gateway.refreshCanvasHostUrl(replacing: nil)
try await waitUntil("seed surface refresh sent") {
session.latestTask()?.sentRequestCount(method: "node.pluginSurface.refresh") == 1
}
let task = try #require(session.latestTask())
let seedRequest = try #require(task.sentRequests(method: "node.pluginSurface.refresh").first)
try task.emitResponse(
id: #require(seedRequest["id"] as? String),
payload: [
"surface": "canvas",
"pluginSurfaceUrls": [
"canvas": "http://gateway.example.invalid/__openclaw__/cap/old-token",
],
])
let oldURL = try #require(await seeded)
async let rotated = gateway.refreshCanvasHostUrl(replacing: oldURL)
try await waitUntil("rotating surface refresh sent") {
task.sentRequestCount(method: "node.pluginSurface.refresh") == 2
}
let rotateRequest = try #require(task.sentRequests(method: "node.pluginSurface.refresh").last)
let rotateParams = try #require(rotateRequest["params"] as? [String: Any])
#expect(rotateParams["observedUrl"] as? String == oldURL)
try task.emitResponse(
id: #require(rotateRequest["id"] as? String),
payload: [
"surface": "canvas",
"pluginSurfaceUrls": [
"canvas": "http://gateway.example.invalid/__openclaw__/cap/new-token",
],
])
let newURL = try #require(await rotated)
let laggingURL = await gateway.refreshCanvasHostUrl(replacing: oldURL)
#expect(laggingURL == newURL)
#expect(task.sentRequestCount(method: "node.pluginSurface.refresh") == 2)
await gateway.disconnect()
}
@Test func `node requests preserve numeric JSON params`() async throws {
let session = FakeGatewayWebSocketSession()
let gateway = GatewayNodeSession()
@@ -3036,22 +3359,41 @@ struct GatewayNodeSessionTests {
@Test
func `normalize canvas host url preserves explicit secure canvas port`() {
let normalized = canonicalizeCanvasHostUrl(
let normalized = GatewayPluginSurfaceURL.canonicalize(
raw: "https://canvas.example.com:9443/__openclaw__/cap/token",
activeURL: URL(string: "wss://gateway.example.com"))
against: URL(string: "wss://gateway.example.com"))
#expect(normalized == "https://canvas.example.com:9443/__openclaw__/cap/token")
}
@Test
func `normalize canvas host url backfills gateway host for loopback canvas`() {
let normalized = canonicalizeCanvasHostUrl(
let normalized = GatewayPluginSurfaceURL.canonicalize(
raw: "http://127.0.0.1:18789/__openclaw__/cap/token",
activeURL: URL(string: "wss://gateway.example.com:7443"))
against: URL(string: "wss://gateway.example.com:7443"))
#expect(normalized == "https://gateway.example.com:7443/__openclaw__/cap/token")
}
@Test
func `gateway TLS pin applies only to the same canvas endpoint`() {
let fingerprint = String(repeating: "ab", count: 32)
let gateway = URL(string: "wss://gateway.example.com:7443")
#expect(GatewayPluginSurfaceURL.tlsFingerprintForSurface(
fingerprint,
surfaceURL: "https://gateway.example.com:7443/__openclaw__/cap/token",
gatewayURL: gateway) == fingerprint)
#expect(GatewayPluginSurfaceURL.tlsFingerprintForSurface(
fingerprint,
surfaceURL: "https://canvas.example.com:7443/__openclaw__/cap/token",
gatewayURL: gateway) == nil)
#expect(GatewayPluginSurfaceURL.tlsFingerprintForSurface(
fingerprint,
surfaceURL: "https://gateway.example.com:9443/__openclaw__/cap/token",
gatewayURL: gateway) == nil)
}
@Test
func `watch invoke payload decodes without bridge frame type`() throws {
let data = Data(
+1 -1
View File
@@ -109,7 +109,7 @@ for contract details.
Common plugin-provided tools include:
- [Diffs](/tools/diffs) for rendering file and markdown diffs
- [Show widget](/tools/show-widget) for self-contained inline SVG and HTML in web chat
- [Show widget](/tools/show-widget) for self-contained inline SVG and HTML in supported chat clients
- [LLM Task](/tools/llm-task) for JSON-only workflow steps
- [Lobster](/tools/lobster) for typed workflows with resumable approvals
- [Tokenjuice](/tools/tokenjuice) for compacting noisy `exec` and `bash` tool
+10 -10
View File
@@ -1,27 +1,27 @@
---
summary: "Show self-contained HTML widgets on the current chat surface"
summary: "Show self-contained HTML widgets on supported chat surfaces"
title: "Show widget"
sidebarTitle: "Show widget"
read_when:
- You want an agent to render an interactive result in web chat or Discord
- You want an agent to render an interactive result in web chat, a native app, or Discord
- You want widget buttons to send follow-up prompts into the chat
- You need the show_widget input, security, or retention contract
---
`show_widget` shows a self-contained HTML widget on the user's current surface. In the Control UI, the Canvas plugin renders it inline in the chat transcript. In a Discord session with [Activities](/channels/discord-activities) enabled, the Discord plugin posts an **Open widget** button that launches it as an Activity.
`show_widget` shows a self-contained HTML widget on the user's current surface. The Canvas plugin renders it inline in the Control UI, iOS, Android, and macOS chat transcripts; Linux uses the browser Control UI. In a Discord session with [Activities](/channels/discord-activities) enabled, the Discord plugin posts an **Open widget** button that launches it as an Activity.
## How widgets work
When the agent calls `show_widget`, the Canvas plugin wraps `widget_code` in a minimal HTML document, stores it as a Canvas document, and returns a preview handle. Web chat renders that handle as a sandboxed iframe directly under the tool call and restores it after history reload.
When the agent calls `show_widget`, the Canvas plugin wraps `widget_code` in a minimal HTML document, stores it as a Canvas document, and returns a preview handle. The Control UI renders that handle as a sandboxed iframe directly under the tool call, while native apps use an isolated web view. Both restore the widget after history reload.
The wrapper document injects two small host bridges around the widget code:
For browser embedding, the wrapper document injects two small host bridges around the widget code:
- A size reporter posts the rendered content height to the embedding chat, which clamps it and fits the iframe (160 to 1200 pixels).
- A prompt bridge defines a global `sendPrompt(text)` function that widget scripts can call to submit a follow-up message into the chat. The bridge creates a private message channel and offers one endpoint to the chat before any widget code runs; the chat adopts only that first offer. See [Interactive widgets](#interactive-widgets).
Everything else stays inside the frame: the document runs in an opaque origin with a strict Content Security Policy, so widget scripts cannot reach the Control UI, the Gateway, or the network.
The Canvas implementation is available only when the originating Gateway client declares the `inline-widgets` capability. The Control UI declares this capability automatically. The Discord implementation is available only in Discord sessions with Activities configured. Other channel runs do not receive `show_widget`.
The Canvas implementation is available only when the originating Gateway client declares the `inline-widgets` capability. The Control UI and supported native apps declare this capability automatically. The Discord implementation is available only in Discord sessions with Activities configured. Other channel runs do not receive `show_widget`.
Capability transport covers embedded, Codex app-server, and CLI-backed model backends. Grant-authenticated MCP callers and direct HTTP tool-invoke callers remain fail closed because they do not declare client capabilities.
@@ -34,18 +34,18 @@ Both implementations use the same required fields:
</ParamField>
<ParamField path="widget_code" type="string" required>
Self-contained HTML or SVG. In the Control UI, input beginning with `<svg` after trimming is rendered in SVG mode; maximum length is 262,144 characters. Discord accepts a complete HTML document or body fragment up to 48 KiB.
Self-contained HTML or SVG. For Canvas-backed clients, input beginning with `<svg` after trimming is rendered in SVG mode; maximum length is 262,144 characters. Discord accepts a complete HTML document or body fragment up to 48 KiB.
</ParamField>
Discord also accepts optional `button_label` text for the Activity launch button. The Canvas schema intentionally omits this Discord-only field.
The Control UI result includes a Canvas preview handle, so web chat renders the widget directly from the tool call and restores it after history reload. Discord returns the stored widget and posted-message identifiers.
The Canvas result includes a preview handle, so the Control UI and supported native apps render the widget directly from the tool call and restore it after history reload. Discord returns the stored widget and posted-message identifiers.
`discord_widget` remains registered as a deprecated alias for one release. New agent calls should use `show_widget`.
## Interactive widgets
Widget scripts can drive the conversation. The wrapper document defines a global `sendPrompt(text)` function; calling it submits `text` to the chat as if the user had typed and sent the message. Wire it to buttons or other controls to build interactive flows such as pickers, quizzes, or drill-down dashboards:
In the Control UI, widget scripts can drive the conversation. The wrapper document defines a global `sendPrompt(text)` function; calling it submits `text` to the chat as if the user had typed and sent the message. Wire it to buttons or other controls to build interactive flows such as pickers, quizzes, or drill-down dashboards. Native apps render interactive widget code but do not expose this chat prompt bridge.
```html
<button onclick="sendPrompt('Show the failing tests in detail')">Failing tests</button>
@@ -66,7 +66,7 @@ Accepted prompts appear in the transcript as regular user messages and start a n
Widget documents use restrictive Content Security Policies. Inline style and script are allowed, while external fetches and resource loads are blocked. Keep all markup, styles, scripts, and image data inside `widget_code`.
The iframe always omits `allow-same-origin`, even when the Control UI's global embed mode is `trusted`, so widget scripts cannot read the parent application origin. The Canvas host also serves widget documents with a `Content-Security-Policy: sandbox allow-scripts` response header, so opening the hosted URL directly still runs the widget in an opaque origin instead of the Control UI origin. Browser sandboxing does not prevent a script from navigating its own iframe; only render widget code you are willing to execute in that isolated frame.
The Control UI iframe always omits `allow-same-origin`, even when the global embed mode is `trusted`, so widget scripts cannot read the parent application origin. Native clients use isolated, nonpersistent web views and block navigation away from the hosted widget. The Canvas host also serves widget documents with a `Content-Security-Policy: sandbox allow-scripts` response header, so direct rendering still runs the widget in an opaque origin instead of an application origin. Only render widget code you are willing to execute in that isolated frame.
The iframe also follows [`gateway.controlUi.embedSandbox`](/web/control-ui#hosted-embeds). The default `scripts` tier supports interactive widgets while preserving origin isolation.
+1 -1
View File
@@ -463,7 +463,7 @@ Web Push is independent of the iOS APNS relay path (see [Configuration](/gateway
Assistant messages can render hosted web content inline with the `[embed ...]` shortcode. The iframe sandbox policy is controlled by `gateway.controlUi.embedSandbox`:
The bundled Canvas plugin provides [`show_widget`](/tools/show-widget) to render self-contained SVG or HTML directly from a tool call. The browser advertises the `inline-widgets` Gateway capability, and the resulting Canvas document remains available when chat history reloads. Discord Activities provide the same tool name on Discord; other channel-originated runs do not receive it.
The bundled Canvas plugin provides [`show_widget`](/tools/show-widget) to render self-contained SVG or HTML directly from a tool call. The browser and supported native chat clients advertise the `inline-widgets` Gateway capability, and the resulting Canvas document remains available when chat history reloads. Discord Activities provide the same tool name on Discord; other channel-originated runs do not receive it.
<Tabs>
<Tab title="strict">
+1 -1
View File
@@ -65,7 +65,7 @@ function createLazyShowWidgetTool(params: {
label: "Show Widget",
name: "show_widget",
description:
"Show an interactive, self-contained HTML or SVG widget to the user on their current surface. Inline all required code and data. A global sendPrompt(text) function submits text to the chat as if the user typed it — wire it to buttons or controls to build interactive widgets. It only works after the user clicks inside the widget (plain conversational text only; slash commands are rejected), so never call it automatically.",
"Show an interactive, self-contained HTML or SVG widget to the user on their current surface. Inline all required code and data. In web chat, a global sendPrompt(text) function submits text to the chat as if the user typed it — wire it to buttons or controls to build interactive widgets. It only works after the user clicks inside the widget (plain conversational text only; slash commands are rejected), so never call it automatically.",
parameters: ShowWidgetToolSchema,
requiredClientCaps: SHOW_WIDGET_REQUIRED_CLIENT_CAPS,
execute: async (...args: Parameters<AnyAgentTool["execute"]>) =>
+1 -1
View File
@@ -26,7 +26,7 @@ const CANVAS_SNAPSHOT_FORMATS = ["png", "jpg", "jpeg"] as const;
/** Gateway capability required for inline transcript widgets. */
export const SHOW_WIDGET_REQUIRED_CLIENT_CAPS = ["inline-widgets"];
/** TypeBox schema for inline web chat widgets. */
/** TypeBox schema for inline chat widgets. */
export const ShowWidgetToolSchema = Type.Object({
title: Type.String(),
widget_code: Type.String(),
+2 -2
View File
@@ -1,4 +1,4 @@
/** Agent-facing inline web chat widget tool. */
/** Agent-facing inline chat widget tool. */
import { createHash } from "node:crypto";
import { jsonResult, readStringParam } from "openclaw/plugin-sdk/channel-actions";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
@@ -76,7 +76,7 @@ export function createShowWidgetTool(options: ShowWidgetToolOptions = {}): AnyAg
label: "Show Widget",
name: "show_widget",
description:
"Show an interactive, self-contained HTML or SVG widget to the user on their current surface. Inline all required code and data. A global sendPrompt(text) function submits text to the chat as if the user typed it — wire it to buttons or controls to build interactive widgets. It only works after the user clicks inside the widget (plain conversational text only; slash commands are rejected), so never call it automatically.",
"Show an interactive, self-contained HTML or SVG widget to the user on their current surface. Inline all required code and data. In web chat, a global sendPrompt(text) function submits text to the chat as if the user typed it — wire it to buttons or controls to build interactive widgets. It only works after the user clicks inside the widget (plain conversational text only; slash commands are rejected), so never call it automatically.",
parameters: ShowWidgetToolSchema,
requiredClientCaps: SHOW_WIDGET_REQUIRED_CLIENT_CAPS,
execute: async (_toolCallId, args) => {
+1
View File
@@ -367,6 +367,7 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [
{ name: "migrations.memory.apply", scope: "operator.admin", controlPlaneWrite: true },
{ name: "ui.command", scope: "operator.write" },
{ name: "approval.history", scope: "operator.approvals" },
{ name: "plugin.surface.refresh", scope: "operator.read" },
] as const;
const CORE_GATEWAY_METHOD_SPEC_BY_NAME: ReadonlyMap<string, CoreGatewayMethodSpec> = new Map(
@@ -3,9 +3,11 @@
import { describe, expect, test } from "vitest";
import {
buildPluginNodeCapabilityScopedHostUrl,
hasAuthorizedClientPluginNodeCapabilityUrl,
hasAuthorizedPluginNodeCapability,
indexPluginNodeCapabilitySurfaces,
normalizePluginNodeCapabilityScopedUrl,
pluginNodeCapabilityScopedHostUrlsConflict,
refreshClientPluginNodeCapability,
setClientPluginNodeCapability,
} from "./plugin-node-capability.js";
@@ -55,6 +57,58 @@ describe("plugin node capability helpers", () => {
});
});
test("detects conflicting scoped host capabilities across rewritten hosts", () => {
expect(
pluginNodeCapabilityScopedHostUrlsConflict(
"http://127.0.0.1:18789/__openclaw__/cap/token%20value",
"https://gateway.example:7443/__openclaw__/cap/token%20value",
),
).toBe(false);
expect(
pluginNodeCapabilityScopedHostUrlsConflict(
"https://gateway.example/__openclaw__/cap/old-token",
"https://gateway.example/__openclaw__/cap/new-token",
),
).toBe(true);
expect(pluginNodeCapabilityScopedHostUrlsConflict("not-a-url", "also-not-a-url")).toBe(false);
});
test("validates a current scoped URL without extending its authorization", () => {
const client = makeClient({
pluginNodeCapabilities: {
canvas: { capability: "current-token", expiresAtMs: 1_500 },
},
});
const params = {
client,
surface: { surface: "canvas" },
url: "https://gateway.example/__openclaw__/cap/current-token",
nowMs: 1_000,
};
expect(hasAuthorizedClientPluginNodeCapabilityUrl(params)).toBe(true);
expect(client.pluginNodeCapabilities?.canvas?.expiresAtMs).toBe(1_500);
expect(
hasAuthorizedClientPluginNodeCapabilityUrl({
...params,
url: "https://gateway.example/__openclaw__/cap/other-token",
}),
).toBe(false);
expect(hasAuthorizedClientPluginNodeCapabilityUrl({ ...params, nowMs: 1_500 })).toBe(false);
expect(
hasAuthorizedClientPluginNodeCapabilityUrl({
...params,
client: makeClient(),
}),
).toBe(false);
expect(
hasAuthorizedClientPluginNodeCapabilityUrl({
...params,
surface: { surface: "canvas", scopeKey: "other-plugin:canvas" },
}),
).toBe(false);
});
test("treats the scoped path capability as authoritative over a stale query", () => {
const normalized = normalizePluginNodeCapabilityScopedUrl(
"/__openclaw__/cap/current-token/__openclaw__/canvas/?oc_cap=stale-token",
+50
View File
@@ -152,6 +152,56 @@ function replacePluginNodeCapabilityInScopedHostUrl(
}
}
function pluginNodeCapabilityFromScopedHostUrl(rawUrl: string): string | undefined {
try {
const pathname = new URL(rawUrl).pathname;
const prefix = `${PLUGIN_NODE_CAPABILITY_PATH_PREFIX}/`;
const markerStart = pathname.indexOf(prefix);
if (markerStart < 0) {
return undefined;
}
const capabilityStart = markerStart + prefix.length;
const nextSlashIndex = pathname.indexOf("/", capabilityStart);
const capabilityEnd = nextSlashIndex >= 0 ? nextSlashIndex : pathname.length;
if (capabilityEnd <= capabilityStart) {
return undefined;
}
return normalizeCapability(decodeURIComponent(pathname.slice(capabilityStart, capabilityEnd)));
} catch {
return undefined;
}
}
/** Detect conflicting scoped capabilities while allowing transport host rewriting. */
export function pluginNodeCapabilityScopedHostUrlsConflict(first: string, second: string): boolean {
const firstCapability = pluginNodeCapabilityFromScopedHostUrl(first);
const secondCapability = pluginNodeCapabilityFromScopedHostUrl(second);
return Boolean(
firstCapability && secondCapability && !safeEqualSecret(firstCapability, secondCapability),
);
}
/** Check whether a client's current scoped surface URL still has live authorization. */
export function hasAuthorizedClientPluginNodeCapabilityUrl(params: {
client: PluginNodeCapabilityClient;
surface: PluginNodeCapabilitySurface;
url: string;
nowMs?: number;
}): boolean {
const storageKey = resolvePluginNodeCapabilityStorageKey(params.surface);
const capability = pluginNodeCapabilityFromScopedHostUrl(params.url);
if (!storageKey || !capability) {
return false;
}
const entry = params.client.pluginNodeCapabilities?.[storageKey];
const nowMs = params.nowMs ?? Date.now();
return Boolean(
entry &&
isFutureDateTimestampMs(entry.expiresAtMs, { nowMs }) &&
safeEqualSecret(entry.capability, capability),
);
}
/** Parse and rewrite scoped capability URLs into canonical paths plus query tokens. */
export function normalizePluginNodeCapabilityScopedUrl(
rawUrl: string,

Some files were not shown because too many files have changed in this diff Show More