mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
feat(android): inline audio/video players with Media3 (#115916)
* feat(android): add inline media players * fix(android): release inactive media players * fix(android): preserve paused media playback * test(android): include Media3 license notice * chore(android): refresh native i18n inventory
This commit is contained in:
committed by
GitHub
parent
26b236dae3
commit
80176dab8d
+319
-255
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,18 @@
|
||||
AndroidX Media3
|
||||
Artifacts:
|
||||
- androidx.media3:media3-datasource-okhttp:1.10.1
|
||||
- androidx.media3:media3-exoplayer:1.10.1
|
||||
- androidx.media3:media3-ui:1.10.1
|
||||
|
||||
Copyright 2016 The Android Open Source Project
|
||||
|
||||
Licensed under the Apache License, Version 2.0.
|
||||
You may obtain a copy of the License at:
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -336,6 +336,9 @@ dependencies {
|
||||
ksp(libs.androidx.room.compiler)
|
||||
implementation(libs.androidx.exifinterface)
|
||||
implementation(libs.okhttp)
|
||||
implementation(libs.media3.datasource.okhttp)
|
||||
implementation(libs.media3.exoplayer)
|
||||
implementation(libs.media3.ui)
|
||||
implementation(libs.bcprov)
|
||||
implementation(libs.coil.compose)
|
||||
implementation(libs.coil.svg)
|
||||
|
||||
@@ -23,6 +23,7 @@ import ai.openclaw.app.chat.SessionRewindResult
|
||||
import ai.openclaw.app.chat.defaultChatThinkingLevelSelection
|
||||
import ai.openclaw.app.chat.resolveChatComposerOwner
|
||||
import ai.openclaw.app.gateway.GatewayEndpoint
|
||||
import ai.openclaw.app.gateway.GatewayMediaKind
|
||||
import ai.openclaw.app.gateway.GatewayRegistryEntry
|
||||
import ai.openclaw.app.gateway.GatewayRegistryEntryKind
|
||||
import ai.openclaw.app.gateway.GatewayUpdateAvailableSummary
|
||||
@@ -1310,6 +1311,11 @@ class MainViewModel private constructor(
|
||||
|
||||
internal suspend fun loadChatImageArtifact(artifactId: String) = ensureRuntime().loadChatImageArtifact(artifactId)
|
||||
|
||||
internal suspend fun loadChatMediaArtifact(
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
) = ensureRuntime().loadChatMediaArtifact(artifactId, kind)
|
||||
|
||||
fun requestCanvasRehydrate(source: String = "screen_tab") {
|
||||
ensureRuntime().requestCanvasRehydrate(source = source, force = true)
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ import ai.openclaw.app.gateway.DeviceIdentityStore
|
||||
import ai.openclaw.app.gateway.GatewayDiscovery
|
||||
import ai.openclaw.app.gateway.GatewayEndpoint
|
||||
import ai.openclaw.app.gateway.GatewayEvent
|
||||
import ai.openclaw.app.gateway.GatewayMediaKind
|
||||
import ai.openclaw.app.gateway.GatewayMethod
|
||||
import ai.openclaw.app.gateway.GatewayRegistryEntry
|
||||
import ai.openclaw.app.gateway.GatewayRegistryEntryKind
|
||||
@@ -4915,6 +4916,11 @@ class NodeRuntime private constructor(
|
||||
|
||||
internal suspend fun loadChatImageArtifact(artifactId: String) = chat.loadImageArtifact(artifactId)
|
||||
|
||||
internal suspend fun loadChatMediaArtifact(
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
) = chat.loadMediaArtifact(artifactId, kind)
|
||||
|
||||
fun loadChat(
|
||||
sessionKey: String,
|
||||
ownerAgentId: String? = null,
|
||||
|
||||
@@ -2,6 +2,8 @@ package ai.openclaw.app.chat
|
||||
|
||||
import ai.openclaw.app.GatewayModelSummary
|
||||
import ai.openclaw.app.gateway.GatewayLoadedImage
|
||||
import ai.openclaw.app.gateway.GatewayLoadedMedia
|
||||
import ai.openclaw.app.gateway.GatewayMediaKind
|
||||
import ai.openclaw.app.gateway.GatewayRequestDefinitiveFailure
|
||||
import ai.openclaw.app.gateway.GatewayRequestNotEnqueued
|
||||
import ai.openclaw.app.gateway.GatewayRequestOutcomeUnknown
|
||||
@@ -57,7 +59,7 @@ internal const val SESSION_LIST_FETCH_LIMIT = 200
|
||||
private val QUESTION_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
|
||||
private val SWARM_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
|
||||
private const val SESSION_EDITOR_MAX_BASE64_CHARS = ((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 2) / 3) * 4
|
||||
private val MANAGED_IMAGE_PATH_REGEX =
|
||||
private val MANAGED_MEDIA_PATH_REGEX =
|
||||
Regex("^/api/chat/media/outgoing/[^/]+/([0-9a-fA-F-]{36})/full(?:\\?.*)?$")
|
||||
|
||||
internal fun chatOutboxQueueFailureText(): NativeText = ChatController.queueFailureText()
|
||||
@@ -120,6 +122,13 @@ class ChatController internal constructor(
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
) -> GatewayLoadedImage? = { _, _, _, _ -> null },
|
||||
private val loadGatewayMediaArtifact: suspend (
|
||||
gatewayId: String?,
|
||||
sessionKey: String,
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
) -> GatewayLoadedMedia? = { _, _, _, _, _ -> null },
|
||||
private val commandOutbox: ChatCommandOutbox? = null,
|
||||
private val recordModelRecent: (String) -> Unit = {},
|
||||
private val onSessionDeleted: (ChatSessionDeletion) -> Unit = {},
|
||||
@@ -154,6 +163,9 @@ class ChatController internal constructor(
|
||||
loadGatewayImageArtifact = { gatewayId, sessionKey, agentId, artifactId ->
|
||||
session.loadImageArtifact(gatewayId, sessionKey, agentId, artifactId)
|
||||
},
|
||||
loadGatewayMediaArtifact = { gatewayId, sessionKey, agentId, artifactId, kind ->
|
||||
session.loadMediaArtifact(gatewayId, sessionKey, agentId, artifactId, kind)
|
||||
},
|
||||
commandOutbox = commandOutbox,
|
||||
recordModelRecent = recordModelRecent,
|
||||
onSessionDeleted = onSessionDeleted,
|
||||
@@ -171,6 +183,22 @@ class ChatController internal constructor(
|
||||
)
|
||||
}
|
||||
|
||||
suspend fun loadMediaArtifact(
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
): GatewayLoadedMedia? {
|
||||
val normalizedArtifactId = artifactId.trim().takeIf(String::isNotEmpty) ?: return null
|
||||
if (kind == GatewayMediaKind.Image) return null
|
||||
val sessionKey = normalizeRequestedSessionKey(_sessionKey.value)
|
||||
return loadGatewayMediaArtifact(
|
||||
currentCacheScope()?.gatewayId,
|
||||
sessionKey,
|
||||
resolveAgentIdForSessionKey(sessionKey),
|
||||
normalizedArtifactId,
|
||||
kind,
|
||||
)
|
||||
}
|
||||
|
||||
private var appliedMainSessionKey = "main"
|
||||
private val cacheMutationMutex = Mutex()
|
||||
private val defaultAgentPersistenceMutex = Mutex()
|
||||
@@ -6671,7 +6699,7 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
text = obj["text"].asStringOrNull() ?: obj["content"].asStringOrNull(),
|
||||
)
|
||||
|
||||
"image", "audio" -> {
|
||||
"image", "audio", "video" -> {
|
||||
val type = obj["type"].asStringOrNull() ?: "image"
|
||||
val inlineContent = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() }
|
||||
val url = obj["url"].asStringOrNull()
|
||||
@@ -6679,25 +6707,42 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
type = type,
|
||||
mimeType = obj["mimeType"].asStringOrNull(),
|
||||
fileName = obj["fileName"].asStringOrNull(),
|
||||
artifactId = obj["artifactId"].asStringOrNull() ?: managedImageArtifactId(url),
|
||||
artifactId =
|
||||
obj["artifactId"].asStringOrNull()
|
||||
?: if (type == "image") managedImageArtifactId(url) else managedMediaArtifactId(url),
|
||||
url = url,
|
||||
openUrl = obj["openUrl"].asStringOrNull(),
|
||||
alt = obj["alt"].asStringOrNull(),
|
||||
width = obj["width"].asLongOrNull()?.toInt(),
|
||||
height = obj["height"].asLongOrNull()?.toInt(),
|
||||
sizeBytes = obj["sizeBytes"].asLongOrNull(),
|
||||
base64 = inlineContent?.takeIf { type != "image" || it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
|
||||
base64 = inlineContent?.takeIf { type == "image" && it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
|
||||
durationMs = obj["durationMs"].asLongOrNull(),
|
||||
)
|
||||
}
|
||||
|
||||
"attachment" -> {
|
||||
val attachment = obj["attachment"].asObjectOrNull() ?: return null
|
||||
val mimeType = attachment["mimeType"].asStringOrNull()
|
||||
if (attachment["kind"].asStringOrNull() != "audio" && mimeType?.startsWith("audio/") != true) return null
|
||||
val type =
|
||||
when {
|
||||
attachment["kind"].asStringOrNull() == "audio" || mimeType?.startsWith("audio/") == true -> "audio"
|
||||
attachment["kind"].asStringOrNull() == "video" || mimeType?.startsWith("video/") == true -> "video"
|
||||
else -> return null
|
||||
}
|
||||
val url = attachment["url"].asStringOrNull()
|
||||
ChatMessageContent(
|
||||
type = "audio",
|
||||
type = type,
|
||||
mimeType = mimeType,
|
||||
fileName = attachment["label"].asStringOrNull(),
|
||||
fileName = attachment["fileName"].asStringOrNull() ?: attachment["label"].asStringOrNull(),
|
||||
artifactId = attachment["artifactId"].asStringOrNull() ?: managedMediaArtifactId(url),
|
||||
url = url,
|
||||
openUrl = attachment["openUrl"].asStringOrNull(),
|
||||
alt = attachment["alt"].asStringOrNull(),
|
||||
width = attachment["width"].asLongOrNull()?.toInt(),
|
||||
height = attachment["height"].asLongOrNull()?.toInt(),
|
||||
sizeBytes = attachment["sizeBytes"].asLongOrNull(),
|
||||
durationMs = attachment["durationMs"].asLongOrNull(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -6730,13 +6775,22 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
}
|
||||
|
||||
internal fun managedImageArtifactId(rawUrl: String?): String? {
|
||||
val attachmentId = managedMediaAttachmentId(rawUrl) ?: return null
|
||||
return "artifact_managed_image_$attachmentId"
|
||||
}
|
||||
|
||||
internal fun managedMediaArtifactId(rawUrl: String?): String? {
|
||||
val attachmentId = managedMediaAttachmentId(rawUrl) ?: return null
|
||||
return "artifact_managed_media_$attachmentId"
|
||||
}
|
||||
|
||||
private fun managedMediaAttachmentId(rawUrl: String?): String? {
|
||||
val match =
|
||||
rawUrl
|
||||
?.trim()
|
||||
?.let(MANAGED_IMAGE_PATH_REGEX::matchEntire)
|
||||
?.let(MANAGED_MEDIA_PATH_REGEX::matchEntire)
|
||||
?: return null
|
||||
val attachmentId = runCatching { UUID.fromString(match.groupValues[1]).toString() }.getOrNull() ?: return null
|
||||
return "artifact_managed_image_$attachmentId"
|
||||
return runCatching { UUID.fromString(match.groupValues[1]).toString() }.getOrNull()
|
||||
}
|
||||
|
||||
internal fun parseChatMessageContents(obj: JsonObject): List<ChatMessageContent> {
|
||||
|
||||
@@ -63,7 +63,7 @@ data class ChatTranscriptAnchorState(
|
||||
)
|
||||
|
||||
/**
|
||||
* One content part in a chat message; images carry either bounded base64 or a managed artifact reference.
|
||||
* One content part in a chat message; media carries either bounded base64 or a managed artifact reference.
|
||||
*/
|
||||
data class ChatMessageContent(
|
||||
val type: String = "text",
|
||||
|
||||
@@ -31,6 +31,7 @@ private data class CachedMessageContent(
|
||||
val width: Int? = null,
|
||||
val height: Int? = null,
|
||||
val sizeBytes: Long? = null,
|
||||
val durationMs: Long? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -108,7 +109,7 @@ internal data class CachedMessageEntity(
|
||||
val sessionKey: String,
|
||||
val rowOrder: Int,
|
||||
val role: String,
|
||||
// JSON array of text and managed-image references; attachment bytes are never persisted.
|
||||
// JSON array of text and managed-media references; attachment bytes are never persisted.
|
||||
val textPartsJson: String,
|
||||
val timestampMs: Long?,
|
||||
// Kept so live history reconciliation can match cached rows by identity key.
|
||||
@@ -318,6 +319,7 @@ class RoomChatTranscriptCache internal constructor(
|
||||
width = part.width,
|
||||
height = part.height,
|
||||
sizeBytes = part.sizeBytes,
|
||||
durationMs = part.durationMs,
|
||||
)
|
||||
},
|
||||
timestampMs = row.timestampMs,
|
||||
@@ -399,8 +401,8 @@ class RoomChatTranscriptCache internal constructor(
|
||||
val gateway = scopedGatewayId(gatewayId) ?: return
|
||||
val agent = scopedAgentId(agentId) ?: return
|
||||
val key = sessionKey.trim().takeIf { it.isNotEmpty() } ?: return
|
||||
// Persist small managed-image references, never attachment bytes. This keeps generated images
|
||||
// visible offline without turning the disposable transcript cache into a binary store.
|
||||
// Persist small managed-media references, never attachment bytes. Cards remain visible offline
|
||||
// even though their short-lived download capability must be reacquired after reconnecting.
|
||||
val rows =
|
||||
messages
|
||||
.mapNotNull { message ->
|
||||
@@ -423,6 +425,20 @@ class RoomChatTranscriptCache internal constructor(
|
||||
height = part.height,
|
||||
sizeBytes = part.sizeBytes,
|
||||
)
|
||||
part.type == "audio" || part.type == "video" ->
|
||||
CachedMessageContent(
|
||||
type = part.type,
|
||||
mimeType = part.mimeType,
|
||||
fileName = part.fileName,
|
||||
artifactId = part.artifactId,
|
||||
url = part.url,
|
||||
openUrl = part.openUrl,
|
||||
alt = part.alt,
|
||||
width = part.width,
|
||||
height = part.height,
|
||||
sizeBytes = part.sizeBytes,
|
||||
durationMs = part.durationMs,
|
||||
)
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ import okhttp3.WebSocket
|
||||
import okhttp3.WebSocketListener
|
||||
import okio.Buffer
|
||||
import java.net.URI
|
||||
import java.util.Base64
|
||||
import java.util.Locale
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -60,6 +61,32 @@ data class GatewayLoadedImage(
|
||||
val mimeType: String,
|
||||
)
|
||||
|
||||
enum class GatewayMediaKind(
|
||||
val wireValue: String,
|
||||
val maximumBufferedBytes: Long,
|
||||
) {
|
||||
Image("image", 12L * 1024L * 1024L),
|
||||
Audio("audio", 16L * 1024L * 1024L),
|
||||
Video("video", 0L),
|
||||
}
|
||||
|
||||
sealed interface GatewayLoadedMedia {
|
||||
data class Buffered(
|
||||
val bytes: ByteArray,
|
||||
val mimeType: String,
|
||||
val headers: Map<String, String>,
|
||||
val client: OkHttpClient,
|
||||
) : GatewayLoadedMedia
|
||||
|
||||
/** A short-lived gateway capability resolved with the exact socket's HTTP transport policy. */
|
||||
data class Streaming(
|
||||
val url: String,
|
||||
val headers: Map<String, String>,
|
||||
val client: OkHttpClient,
|
||||
val mimeType: String?,
|
||||
) : GatewayLoadedMedia
|
||||
}
|
||||
|
||||
/**
|
||||
* Role, scopes, commands, and permission snapshot sent with the connect frame.
|
||||
*/
|
||||
@@ -629,6 +656,24 @@ class GatewaySession(
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
): GatewayLoadedImage? {
|
||||
val loaded =
|
||||
loadMediaArtifact(
|
||||
expectedEndpointStableId = expectedEndpointStableId,
|
||||
sessionKey = sessionKey,
|
||||
agentId = agentId,
|
||||
artifactId = artifactId,
|
||||
kind = GatewayMediaKind.Image,
|
||||
) as? GatewayLoadedMedia.Buffered ?: return null
|
||||
return GatewayLoadedImage(bytes = loaded.bytes, mimeType = loaded.mimeType)
|
||||
}
|
||||
|
||||
suspend fun loadMediaArtifact(
|
||||
expectedEndpointStableId: String?,
|
||||
sessionKey: String,
|
||||
agentId: String?,
|
||||
artifactId: String,
|
||||
kind: GatewayMediaKind,
|
||||
): GatewayLoadedMedia? {
|
||||
val conn = readyConnection(expectedEndpointStableId) ?: return null
|
||||
val params =
|
||||
buildJsonObject {
|
||||
@@ -640,16 +685,35 @@ class GatewaySession(
|
||||
if (!response.ok) {
|
||||
throw GatewayRequestRejected(response.error ?: ErrorShape("UNAVAILABLE", "artifact download failed"))
|
||||
}
|
||||
val ticketedPath =
|
||||
response.payloadJson
|
||||
?.let(::parseJsonOrNull)
|
||||
.asObjectOrNull()
|
||||
?.get("url")
|
||||
val payload = response.payloadJson?.let(::parseJsonOrNull).asObjectOrNull() ?: return null
|
||||
val artifact = payload["artifact"].asObjectOrNull()
|
||||
// Older gateways returned only the managed-image URL. Audio/video shipped with the typed
|
||||
// artifact envelope, so keep the narrow image fallback without weakening their type gate.
|
||||
if (artifact == null && kind != GatewayMediaKind.Image) return null
|
||||
if (artifact != null && artifact["type"].asStringOrNull()?.trim()?.lowercase(Locale.ROOT) != kind.wireValue) return null
|
||||
val mimeType =
|
||||
artifact
|
||||
?.get("mimeType")
|
||||
.asStringOrNull()
|
||||
?.trim()
|
||||
?.takeIf(String::isNotEmpty)
|
||||
?: return null
|
||||
val loaded = conn.loadTicketedImage(ticketedPath) ?: return null
|
||||
?.lowercase(Locale.ROOT)
|
||||
if (mimeType != null && !mimeType.startsWith("${kind.wireValue}/")) return null
|
||||
val loaded =
|
||||
payload["data"].asStringOrNull()?.let { encoded ->
|
||||
if (kind == GatewayMediaKind.Video) return@let null
|
||||
val maximumEncodedLength = ((kind.maximumBufferedBytes + 2L) / 3L) * 4L
|
||||
if (encoded.length.toLong() > maximumEncodedLength) return@let null
|
||||
val bytes = runCatching { Base64.getDecoder().decode(encoded) }.getOrNull() ?: return@let null
|
||||
if (bytes.size.toLong() > kind.maximumBufferedBytes) return@let null
|
||||
val resolvedMimeType = mimeType ?: return@let null
|
||||
conn.bufferedMedia(bytes = bytes, mimeType = resolvedMimeType)
|
||||
} ?: payload["url"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty)?.let { ticketedPath ->
|
||||
if (kind == GatewayMediaKind.Video) {
|
||||
conn.resolveTicketedMediaStream(ticketedPath, mimeType)
|
||||
} else {
|
||||
conn.loadTicketedMedia(ticketedPath, kind)
|
||||
}
|
||||
} ?: return null
|
||||
return synchronized(lifecycleLock) {
|
||||
loaded.takeIf { currentConnection === conn && conn.isReady() }
|
||||
}
|
||||
@@ -773,6 +837,11 @@ class GatewaySession(
|
||||
val error: ErrorShape?,
|
||||
)
|
||||
|
||||
private data class TicketedMediaRequest(
|
||||
val url: String,
|
||||
val headers: Map<String, String>,
|
||||
)
|
||||
|
||||
private data class ConnectedGateway(
|
||||
val pluginSurfaceUrls: Map<String, String>,
|
||||
val mainSessionKey: String?,
|
||||
@@ -869,22 +938,40 @@ class GatewaySession(
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun loadTicketedImage(ticketedPath: String): GatewayLoadedImage? =
|
||||
fun resolveTicketedMediaStream(
|
||||
ticketedPath: String,
|
||||
mimeType: String?,
|
||||
): GatewayLoadedMedia.Streaming? {
|
||||
val request = resolveTicketedMediaRequest(ticketedPath) ?: return null
|
||||
return GatewayLoadedMedia.Streaming(
|
||||
url = request.url,
|
||||
headers = request.headers + ("Accept" to "video/*"),
|
||||
client = client,
|
||||
mimeType = mimeType,
|
||||
)
|
||||
}
|
||||
|
||||
fun bufferedMedia(
|
||||
bytes: ByteArray,
|
||||
mimeType: String,
|
||||
): GatewayLoadedMedia.Buffered =
|
||||
GatewayLoadedMedia.Buffered(
|
||||
bytes = bytes,
|
||||
mimeType = mimeType,
|
||||
headers = mediaTransportHeaders(),
|
||||
client = client,
|
||||
)
|
||||
|
||||
suspend fun loadTicketedMedia(
|
||||
ticketedPath: String,
|
||||
kind: GatewayMediaKind,
|
||||
): GatewayLoadedMedia.Buffered? =
|
||||
withContext(Dispatchers.IO) {
|
||||
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return@withContext null
|
||||
val rawPath = uri.rawPath ?: return@withContext null
|
||||
val rawQuery = uri.rawQuery ?: return@withContext null
|
||||
if (uri.isAbsolute || uri.rawAuthority != null || uri.rawFragment != null) return@withContext null
|
||||
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !rawQuery.contains("mediaTicket=")) {
|
||||
return@withContext null
|
||||
}
|
||||
val scheme = if (tlsConfig != null) "https" else "http"
|
||||
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
|
||||
val request = Request.Builder().url(url).header("Accept", "image/*")
|
||||
if (tlsConfig != null) {
|
||||
for ((name, value) in GatewayCustomHeaders.sanitized(customHeadersProvider?.invoke(endpoint.stableId).orEmpty())) {
|
||||
request.header(name, value)
|
||||
}
|
||||
if (kind == GatewayMediaKind.Video) return@withContext null
|
||||
val resolved = resolveTicketedMediaRequest(ticketedPath) ?: return@withContext null
|
||||
val request = Request.Builder().url(resolved.url).header("Accept", "${kind.wireValue}/*")
|
||||
for ((name, value) in resolved.headers) {
|
||||
request.header(name, value)
|
||||
}
|
||||
val call = client.newCall(request.build())
|
||||
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
|
||||
@@ -892,8 +979,8 @@ class GatewaySession(
|
||||
if (!response.isSuccessful) return@withContext null
|
||||
val body = response.body
|
||||
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
|
||||
if (!mimeType.startsWith("image/")) return@withContext null
|
||||
val maximumBytes = 12L * 1024L * 1024L
|
||||
if (!mimeType.startsWith("${kind.wireValue}/")) return@withContext null
|
||||
val maximumBytes = kind.maximumBufferedBytes
|
||||
val declaredLength = body.contentLength()
|
||||
if (declaredLength > maximumBytes) return@withContext null
|
||||
val buffer = Buffer()
|
||||
@@ -905,10 +992,33 @@ class GatewaySession(
|
||||
total += read
|
||||
if (total > maximumBytes) return@withContext null
|
||||
}
|
||||
GatewayLoadedImage(bytes = buffer.readByteArray(), mimeType = mimeType)
|
||||
bufferedMedia(bytes = buffer.readByteArray(), mimeType = mimeType)
|
||||
}
|
||||
}
|
||||
|
||||
private fun resolveTicketedMediaRequest(ticketedPath: String): TicketedMediaRequest? {
|
||||
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return null
|
||||
val rawPath = uri.rawPath ?: return null
|
||||
val rawQuery = uri.rawQuery ?: return null
|
||||
if (uri.isAbsolute || uri.rawAuthority != null || uri.rawFragment != null) return null
|
||||
val hasMediaTicket =
|
||||
rawQuery
|
||||
.split('&')
|
||||
.any { field -> field.substringBefore('=') == "mediaTicket" && field.substringAfter('=', "").isNotEmpty() }
|
||||
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !hasMediaTicket) return null
|
||||
val scheme = if (tlsConfig != null) "https" else "http"
|
||||
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
|
||||
val headers = mediaTransportHeaders()
|
||||
return TicketedMediaRequest(url = url, headers = headers)
|
||||
}
|
||||
|
||||
private fun mediaTransportHeaders(): Map<String, String> =
|
||||
if (tlsConfig == null) {
|
||||
emptyMap()
|
||||
} else {
|
||||
GatewayCustomHeaders.sanitized(customHeadersProvider?.invoke(endpoint.stableId).orEmpty())
|
||||
}
|
||||
|
||||
@OptIn(DelicateCoroutinesApi::class)
|
||||
suspend fun sendRequestFrame(
|
||||
method: String,
|
||||
|
||||
@@ -0,0 +1,738 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import ai.openclaw.app.chat.ChatMessageContent
|
||||
import ai.openclaw.app.gateway.GatewayLoadedMedia
|
||||
import ai.openclaw.app.gateway.GatewayMediaKind
|
||||
import ai.openclaw.app.i18n.nativeString
|
||||
import ai.openclaw.app.ui.design.ClawTheme
|
||||
import android.content.Context
|
||||
import android.media.AudioFocusRequest
|
||||
import android.media.AudioManager
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import androidx.annotation.OptIn
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.aspectRatio
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Mic
|
||||
import androidx.compose.material.icons.filled.Pause
|
||||
import androidx.compose.material.icons.filled.PlayArrow
|
||||
import androidx.compose.material.icons.filled.Videocam
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.Slider
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableLongStateOf
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.runtime.rememberUpdatedState
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.LifecycleEventObserver
|
||||
import androidx.lifecycle.compose.LocalLifecycleOwner
|
||||
import androidx.media3.common.AudioAttributes
|
||||
import androidx.media3.common.C
|
||||
import androidx.media3.common.MediaItem
|
||||
import androidx.media3.common.PlaybackException
|
||||
import androidx.media3.common.Player
|
||||
import androidx.media3.common.util.UnstableApi
|
||||
import androidx.media3.datasource.DefaultDataSource
|
||||
import androidx.media3.datasource.okhttp.OkHttpDataSource
|
||||
import androidx.media3.exoplayer.ExoPlayer
|
||||
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
|
||||
import androidx.media3.ui.AspectRatioFrameLayout
|
||||
import androidx.media3.ui.PlayerView
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.NonCancellable
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
|
||||
internal class ChatMediaPlaybackClaims<T>(
|
||||
private val pause: (T) -> Unit,
|
||||
private val release: (T) -> Unit,
|
||||
) {
|
||||
var active: T? = null
|
||||
private set
|
||||
|
||||
fun claim(value: T) {
|
||||
if (active === value) return
|
||||
val previous = active
|
||||
active = null
|
||||
previous?.let(release)
|
||||
active = value
|
||||
}
|
||||
|
||||
fun releaseIf(predicate: (T) -> Boolean): Boolean {
|
||||
val previous = active?.takeIf(predicate) ?: return false
|
||||
active = null
|
||||
release(previous)
|
||||
return true
|
||||
}
|
||||
|
||||
fun pauseIf(predicate: (T) -> Boolean): Boolean {
|
||||
val current = active?.takeIf(predicate) ?: return false
|
||||
pause(current)
|
||||
return true
|
||||
}
|
||||
|
||||
fun releaseActive() {
|
||||
val previous = active ?: return
|
||||
active = null
|
||||
release(previous)
|
||||
}
|
||||
}
|
||||
|
||||
private object ChatMediaPlaybackArbiter {
|
||||
private data class AudioFocusHandle(
|
||||
val manager: AudioManager,
|
||||
val request: AudioFocusRequest,
|
||||
)
|
||||
|
||||
private class ActivePlayback(
|
||||
val player: ExoPlayer,
|
||||
val onReleased: () -> Unit,
|
||||
) {
|
||||
var audioFocus: AudioFocusHandle? = null
|
||||
}
|
||||
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val claims = ChatMediaPlaybackClaims<ActivePlayback>(::pauseAndAbandon, ::stopAndRelease)
|
||||
private var playbackIntentGeneration = 0L
|
||||
|
||||
@Synchronized
|
||||
fun claimPlaybackIntent(preservePlayer: ExoPlayer? = null): Long {
|
||||
if (claims.active?.player !== preservePlayer) claims.releaseActive()
|
||||
playbackIntentGeneration += 1L
|
||||
return playbackIntentGeneration
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun isCurrentIntent(generation: Long): Boolean = playbackIntentGeneration == generation
|
||||
|
||||
@Synchronized
|
||||
fun cancelIntent(generation: Long) {
|
||||
if (playbackIntentGeneration == generation) playbackIntentGeneration += 1L
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun pauseAll() {
|
||||
playbackIntentGeneration += 1L
|
||||
claims.pauseIf { true }
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun registerPrepared(
|
||||
player: ExoPlayer,
|
||||
intentGeneration: Long,
|
||||
onReleased: () -> Unit,
|
||||
): Boolean {
|
||||
if (playbackIntentGeneration != intentGeneration) return false
|
||||
if (claims.active?.player === player) return true
|
||||
claims.claim(ActivePlayback(player = player, onReleased = onReleased))
|
||||
return true
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun requestPlayback(
|
||||
context: Context,
|
||||
player: ExoPlayer,
|
||||
intentGeneration: Long,
|
||||
onReleased: () -> Unit,
|
||||
): Boolean {
|
||||
if (playbackIntentGeneration != intentGeneration) return false
|
||||
val existing = claims.active?.takeIf { it.player === player }
|
||||
if (existing?.audioFocus != null) return true
|
||||
|
||||
val audioManager = context.getSystemService(AudioManager::class.java)
|
||||
val focusRequest =
|
||||
AudioFocusRequest
|
||||
.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT)
|
||||
.setAudioAttributes(
|
||||
android.media.AudioAttributes
|
||||
.Builder()
|
||||
.setUsage(android.media.AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(android.media.AudioAttributes.CONTENT_TYPE_UNKNOWN)
|
||||
.build(),
|
||||
).setWillPauseWhenDucked(true)
|
||||
.setOnAudioFocusChangeListener { change ->
|
||||
if (change < 0) {
|
||||
mainHandler.post { pause(player) }
|
||||
}
|
||||
}.build()
|
||||
if (audioManager.requestAudioFocus(focusRequest) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
|
||||
return false
|
||||
}
|
||||
val playback = existing ?: ActivePlayback(player = player, onReleased = onReleased).also(claims::claim)
|
||||
playback.audioFocus = AudioFocusHandle(manager = audioManager, request = focusRequest)
|
||||
return true
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun pause(player: ExoPlayer): Boolean = claims.pauseIf { it.player === player }
|
||||
|
||||
@Synchronized
|
||||
fun release(player: ExoPlayer): Boolean = claims.releaseIf { it.player === player }
|
||||
|
||||
private fun pauseAndAbandon(playback: ActivePlayback) {
|
||||
playback.player.pause()
|
||||
playback.audioFocus?.let { focus -> focus.manager.abandonAudioFocusRequest(focus.request) }
|
||||
playback.audioFocus = null
|
||||
}
|
||||
|
||||
private fun stopAndRelease(playback: ActivePlayback) {
|
||||
pauseAndAbandon(playback)
|
||||
playback.player.release()
|
||||
playback.onReleased()
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun ChatAudioPlayerCard(
|
||||
content: ChatMessageContent,
|
||||
playbackBlocked: Boolean,
|
||||
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
ChatMediaPlayerCard(
|
||||
content = content,
|
||||
kind = GatewayMediaKind.Audio,
|
||||
playbackBlocked = playbackBlocked,
|
||||
loadMedia = loadMedia,
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun ChatVideoPlayerCard(
|
||||
content: ChatMessageContent,
|
||||
playbackBlocked: Boolean,
|
||||
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
ChatMediaPlayerCard(
|
||||
content = content,
|
||||
kind = GatewayMediaKind.Video,
|
||||
playbackBlocked = playbackBlocked,
|
||||
loadMedia = loadMedia,
|
||||
)
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
@Composable
|
||||
private fun ChatMediaPlayerCard(
|
||||
content: ChatMessageContent,
|
||||
kind: GatewayMediaKind,
|
||||
playbackBlocked: Boolean,
|
||||
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
val currentPlaybackBlocked by rememberUpdatedState(playbackBlocked)
|
||||
val scope = rememberCoroutineScope()
|
||||
var player by remember(content.artifactId, kind) { mutableStateOf<ExoPlayer?>(null) }
|
||||
var tempFile by remember(content.artifactId, kind) { mutableStateOf<File?>(null) }
|
||||
var loading by remember(content.artifactId, kind) { mutableStateOf(false) }
|
||||
var error by remember(content.artifactId, kind) { mutableStateOf<String?>(null) }
|
||||
var isPlaying by remember(content.artifactId, kind) { mutableStateOf(false) }
|
||||
var positionMs by remember(content.artifactId, kind) { mutableLongStateOf(0L) }
|
||||
var durationMs by remember(content.artifactId, kind) { mutableLongStateOf(content.durationMs?.coerceAtLeast(0L) ?: 0L) }
|
||||
var playbackIntent by remember(content.artifactId, kind) { mutableLongStateOf(0L) }
|
||||
|
||||
fun clearPlayerState(
|
||||
released: ExoPlayer,
|
||||
releasedFile: File?,
|
||||
) {
|
||||
if (player === released) player = null
|
||||
if (tempFile === releasedFile) tempFile = null
|
||||
releasedFile?.delete()
|
||||
isPlaying = false
|
||||
positionMs = 0L
|
||||
}
|
||||
|
||||
fun disposeUnclaimedPlayer(
|
||||
released: ExoPlayer,
|
||||
releasedFile: File?,
|
||||
) {
|
||||
released.release()
|
||||
clearPlayerState(released, releasedFile)
|
||||
}
|
||||
|
||||
fun requestPlayback(
|
||||
requested: ExoPlayer,
|
||||
requestedFile: File?,
|
||||
intentGeneration: Long,
|
||||
): Boolean =
|
||||
ChatMediaPlaybackArbiter.requestPlayback(
|
||||
context = context,
|
||||
player = requested,
|
||||
intentGeneration = intentGeneration,
|
||||
onReleased = { clearPlayerState(requested, requestedFile) },
|
||||
)
|
||||
|
||||
fun registerPrepared(
|
||||
prepared: ExoPlayer,
|
||||
preparedFile: File?,
|
||||
intentGeneration: Long,
|
||||
): Boolean =
|
||||
ChatMediaPlaybackArbiter.registerPrepared(
|
||||
player = prepared,
|
||||
intentGeneration = intentGeneration,
|
||||
onReleased = { clearPlayerState(prepared, preparedFile) },
|
||||
)
|
||||
|
||||
fun pause() {
|
||||
player?.let(ChatMediaPlaybackArbiter::pause)
|
||||
}
|
||||
|
||||
fun play() {
|
||||
if (playbackBlocked) return
|
||||
val existing = player
|
||||
if (existing != null) {
|
||||
val intentGeneration = ChatMediaPlaybackArbiter.claimPlaybackIntent(preservePlayer = existing)
|
||||
playbackIntent = intentGeneration
|
||||
error = null
|
||||
if (requestPlayback(existing, tempFile, intentGeneration)) {
|
||||
if (existing.playbackState == Player.STATE_ENDED) existing.seekToDefaultPosition()
|
||||
existing.play()
|
||||
} else {
|
||||
error = nativeString("Audio playback is unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
val artifactId = content.artifactId?.trim()?.takeIf(String::isNotEmpty)
|
||||
if (artifactId == null) {
|
||||
error = nativeString("Media unavailable")
|
||||
return
|
||||
}
|
||||
if (loading) return
|
||||
val intentGeneration = ChatMediaPlaybackArbiter.claimPlaybackIntent()
|
||||
playbackIntent = intentGeneration
|
||||
loading = true
|
||||
error = null
|
||||
scope.launch {
|
||||
val loaded =
|
||||
try {
|
||||
loadMedia(artifactId, kind)
|
||||
} catch (error: CancellationException) {
|
||||
throw error
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
if (loaded == null) {
|
||||
loading = false
|
||||
error = nativeString("Media unavailable")
|
||||
return@launch
|
||||
}
|
||||
if (!ChatMediaPlaybackArbiter.isCurrentIntent(intentGeneration)) {
|
||||
loading = false
|
||||
return@launch
|
||||
}
|
||||
val prepared = prepareMediaSource(context = context, loaded = loaded)
|
||||
if (prepared == null) {
|
||||
loading = false
|
||||
error = nativeString("Media unavailable")
|
||||
return@launch
|
||||
}
|
||||
if (!ChatMediaPlaybackArbiter.isCurrentIntent(intentGeneration)) {
|
||||
prepared.tempFile?.delete()
|
||||
loading = false
|
||||
return@launch
|
||||
}
|
||||
tempFile?.delete()
|
||||
tempFile = prepared.tempFile
|
||||
val created =
|
||||
runCatching { buildMediaPlayer(context = context, source = prepared) }.getOrElse {
|
||||
prepared.tempFile?.delete()
|
||||
tempFile = null
|
||||
loading = false
|
||||
error = nativeString("Media unavailable")
|
||||
return@launch
|
||||
}
|
||||
created.addListener(
|
||||
object : Player.Listener {
|
||||
override fun onIsPlayingChanged(value: Boolean) {
|
||||
isPlaying = value
|
||||
}
|
||||
|
||||
override fun onPlaybackStateChanged(playbackState: Int) {
|
||||
if (playbackState == Player.STATE_ENDED) {
|
||||
ChatMediaPlaybackArbiter.pause(created)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onPlayerError(playbackException: PlaybackException) {
|
||||
if (!ChatMediaPlaybackArbiter.release(created)) {
|
||||
disposeUnclaimedPlayer(created, prepared.tempFile)
|
||||
}
|
||||
error = nativeString("Media unavailable")
|
||||
}
|
||||
},
|
||||
)
|
||||
player = created
|
||||
loading = false
|
||||
if (!registerPrepared(created, prepared.tempFile, intentGeneration)) {
|
||||
disposeUnclaimedPlayer(created, prepared.tempFile)
|
||||
return@launch
|
||||
}
|
||||
val shouldPlay =
|
||||
!currentPlaybackBlocked &&
|
||||
lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)
|
||||
if (shouldPlay && requestPlayback(created, prepared.tempFile, intentGeneration)) {
|
||||
created.play()
|
||||
} else if (shouldPlay) {
|
||||
error = nativeString("Audio playback is unavailable")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LaunchedEffect(player) {
|
||||
val activePlayer = player ?: return@LaunchedEffect
|
||||
while (currentCoroutineContext().isActive) {
|
||||
positionMs = activePlayer.currentPosition.coerceAtLeast(0L)
|
||||
activePlayer.duration.takeIf { it != C.TIME_UNSET && it > 0L }?.let { durationMs = it }
|
||||
delay(if (activePlayer.isPlaying) 250L else 1_000L)
|
||||
}
|
||||
}
|
||||
LaunchedEffect(playbackBlocked) {
|
||||
if (playbackBlocked) ChatMediaPlaybackArbiter.pauseAll()
|
||||
}
|
||||
DisposableEffect(lifecycleOwner, player) {
|
||||
val observer =
|
||||
LifecycleEventObserver { _, event ->
|
||||
if (event == Lifecycle.Event.ON_STOP) ChatMediaPlaybackArbiter.pauseAll()
|
||||
}
|
||||
lifecycleOwner.lifecycle.addObserver(observer)
|
||||
onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
|
||||
}
|
||||
DisposableEffect(content.artifactId, kind) {
|
||||
onDispose {
|
||||
ChatMediaPlaybackArbiter.cancelIntent(playbackIntent)
|
||||
player?.let { activePlayer ->
|
||||
val activeFile = tempFile
|
||||
if (!ChatMediaPlaybackArbiter.release(activePlayer)) {
|
||||
activePlayer.release()
|
||||
activeFile?.delete()
|
||||
}
|
||||
}
|
||||
tempFile?.delete()
|
||||
}
|
||||
}
|
||||
|
||||
if (kind == GatewayMediaKind.Video) {
|
||||
VideoPlayerSurface(
|
||||
content = content,
|
||||
player = player,
|
||||
loading = loading,
|
||||
isPlaying = isPlaying,
|
||||
playbackBlocked = playbackBlocked,
|
||||
error = error,
|
||||
onToggle = { if (isPlaying) pause() else play() },
|
||||
)
|
||||
} else {
|
||||
AudioPlayerSurface(
|
||||
content = content,
|
||||
loading = loading,
|
||||
isPlaying = isPlaying,
|
||||
playbackBlocked = playbackBlocked,
|
||||
error = error,
|
||||
positionMs = positionMs,
|
||||
durationMs = durationMs,
|
||||
onToggle = { if (isPlaying) pause() else play() },
|
||||
onSeek = { value ->
|
||||
val target = value.toLong().coerceIn(0L, durationMs.coerceAtLeast(0L))
|
||||
positionMs = target
|
||||
player?.seekTo(target)
|
||||
},
|
||||
seekEnabled = player != null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun AudioPlayerSurface(
|
||||
content: ChatMessageContent,
|
||||
loading: Boolean,
|
||||
isPlaying: Boolean,
|
||||
playbackBlocked: Boolean,
|
||||
error: String?,
|
||||
positionMs: Long,
|
||||
durationMs: Long,
|
||||
onToggle: () -> Unit,
|
||||
onSeek: (Float) -> Unit,
|
||||
seekEnabled: Boolean,
|
||||
) {
|
||||
Surface(
|
||||
shape = RoundedCornerShape(10.dp),
|
||||
color = ClawTheme.colors.surfacePressed.copy(alpha = 0.72f),
|
||||
border = BorderStroke(1.dp, ClawTheme.colors.border.copy(alpha = 0.6f)),
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 10.dp, vertical = 8.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(4.dp),
|
||||
) {
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Surface(
|
||||
onClick = onToggle,
|
||||
enabled = !playbackBlocked && !loading,
|
||||
shape = CircleShape,
|
||||
color = ClawTheme.colors.primary,
|
||||
contentColor = ClawTheme.colors.primaryText,
|
||||
) {
|
||||
Box(modifier = Modifier.size(34.dp), contentAlignment = Alignment.Center) {
|
||||
if (loading) {
|
||||
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
|
||||
} else {
|
||||
Icon(
|
||||
imageVector = if (isPlaying) Icons.Default.Pause else Icons.Default.PlayArrow,
|
||||
contentDescription = if (isPlaying) nativeString("Pause audio") else nativeString("Play audio"),
|
||||
modifier = Modifier.size(19.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
Icon(Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(16.dp), tint = ClawTheme.colors.textMuted)
|
||||
Column(modifier = Modifier.weight(1f)) {
|
||||
Text(
|
||||
text = content.fileName?.takeIf(String::isNotBlank) ?: nativeString("Voice note"),
|
||||
style = ClawTheme.type.body,
|
||||
color = ClawTheme.colors.text,
|
||||
)
|
||||
val status = error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null
|
||||
status?.let { Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted) }
|
||||
}
|
||||
}
|
||||
Slider(
|
||||
value = positionMs.coerceIn(0L, durationMs.coerceAtLeast(0L)).toFloat(),
|
||||
onValueChange = onSeek,
|
||||
valueRange = 0f..durationMs.coerceAtLeast(1L).toFloat(),
|
||||
enabled = seekEnabled && durationMs > 0L && playerControlsAvailable(error, playbackBlocked),
|
||||
)
|
||||
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) {
|
||||
Text(formatVoiceNoteDuration(positionMs), style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
|
||||
Text(formatVoiceNoteDuration(durationMs), style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
@Composable
|
||||
private fun VideoPlayerSurface(
|
||||
content: ChatMessageContent,
|
||||
player: ExoPlayer?,
|
||||
loading: Boolean,
|
||||
isPlaying: Boolean,
|
||||
playbackBlocked: Boolean,
|
||||
error: String?,
|
||||
onToggle: () -> Unit,
|
||||
) {
|
||||
val ratio =
|
||||
remember(content.width, content.height) {
|
||||
val width = content.width?.takeIf { it > 0 }
|
||||
val height = content.height?.takeIf { it > 0 }
|
||||
if (width != null && height != null) width.toFloat() / height.toFloat() else 16f / 9f
|
||||
}
|
||||
Column(verticalArrangement = Arrangement.spacedBy(5.dp)) {
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxWidth()
|
||||
.aspectRatio(ratio.coerceIn(0.5f, 2.4f))
|
||||
.background(ClawTheme.colors.surfacePressed, RoundedCornerShape(10.dp)),
|
||||
contentAlignment = Alignment.Center,
|
||||
) {
|
||||
if (player != null) {
|
||||
AndroidView(
|
||||
factory = { viewContext ->
|
||||
PlayerView(viewContext).apply {
|
||||
useController = false
|
||||
resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FIT
|
||||
this.player = player
|
||||
}
|
||||
},
|
||||
update = { it.player = player },
|
||||
modifier = Modifier.matchParentSize(),
|
||||
)
|
||||
} else {
|
||||
Icon(Icons.Default.Videocam, contentDescription = null, modifier = Modifier.size(36.dp), tint = ClawTheme.colors.textMuted)
|
||||
}
|
||||
Box(
|
||||
modifier = Modifier.matchParentSize().clickable(enabled = !playbackBlocked && !loading, onClick = onToggle),
|
||||
contentAlignment = Alignment.Center,
|
||||
) {
|
||||
if (loading) {
|
||||
CircularProgressIndicator(modifier = Modifier.size(28.dp), strokeWidth = 2.dp)
|
||||
} else if (!isPlaying) {
|
||||
Surface(shape = CircleShape, color = ClawTheme.colors.primary, contentColor = ClawTheme.colors.primaryText) {
|
||||
Icon(Icons.Default.PlayArrow, contentDescription = nativeString("Play video"), modifier = Modifier.padding(10.dp).size(24.dp))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Text(
|
||||
text = content.fileName?.takeIf(String::isNotBlank) ?: nativeString("Video"),
|
||||
style = ClawTheme.type.caption,
|
||||
color = ClawTheme.colors.textMuted,
|
||||
)
|
||||
(error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null)?.let {
|
||||
Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun playerControlsAvailable(
|
||||
error: String?,
|
||||
playbackBlocked: Boolean,
|
||||
): Boolean = error == null && !playbackBlocked
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
private suspend fun prepareMediaSource(
|
||||
context: Context,
|
||||
loaded: GatewayLoadedMedia,
|
||||
): PreparedMediaSource? {
|
||||
return when (loaded) {
|
||||
is GatewayLoadedMedia.Buffered -> {
|
||||
val file = writeBufferedMediaFile(context = context, bytes = loaded.bytes) ?: return null
|
||||
PreparedMediaSource(
|
||||
uri = file.toURI().toString(),
|
||||
mimeType = loaded.mimeType,
|
||||
headers = loaded.headers,
|
||||
client = loaded.client,
|
||||
tempFile = file,
|
||||
)
|
||||
}
|
||||
is GatewayLoadedMedia.Streaming ->
|
||||
PreparedMediaSource(
|
||||
uri = loaded.url,
|
||||
mimeType = loaded.mimeType,
|
||||
headers = loaded.headers,
|
||||
client = loaded.client,
|
||||
tempFile = null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun writeBufferedMediaFile(
|
||||
context: Context,
|
||||
bytes: ByteArray,
|
||||
): File? {
|
||||
var created: File? = null
|
||||
return try {
|
||||
withContext(Dispatchers.IO) {
|
||||
File.createTempFile("chat-media-", ".media", context.cacheDir).also { file ->
|
||||
created = file
|
||||
file.writeBytes(bytes)
|
||||
}
|
||||
}
|
||||
} catch (error: CancellationException) {
|
||||
withContext(NonCancellable + Dispatchers.IO) { created?.delete() }
|
||||
throw error
|
||||
} catch (_: Throwable) {
|
||||
withContext(NonCancellable + Dispatchers.IO) { created?.delete() }
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(UnstableApi::class)
|
||||
private fun buildMediaPlayer(
|
||||
context: Context,
|
||||
source: PreparedMediaSource,
|
||||
): ExoPlayer {
|
||||
val httpFactory = OkHttpDataSource.Factory(source.client).setDefaultRequestProperties(source.headers)
|
||||
val dataSourceFactory = DefaultDataSource.Factory(context, httpFactory)
|
||||
val player =
|
||||
ExoPlayer
|
||||
.Builder(context)
|
||||
.setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory))
|
||||
.build()
|
||||
player.setAudioAttributes(
|
||||
AudioAttributes
|
||||
.Builder()
|
||||
.setUsage(C.USAGE_MEDIA)
|
||||
.setContentType(C.AUDIO_CONTENT_TYPE_UNKNOWN)
|
||||
.build(),
|
||||
false,
|
||||
)
|
||||
player.setMediaItem(
|
||||
MediaItem
|
||||
.Builder()
|
||||
.setUri(source.uri)
|
||||
.apply { source.mimeType?.let(::setMimeType) }
|
||||
.build(),
|
||||
)
|
||||
player.prepare()
|
||||
return player
|
||||
}
|
||||
|
||||
private data class PreparedMediaSource(
|
||||
val uri: String,
|
||||
val mimeType: String?,
|
||||
val headers: Map<String, String>,
|
||||
val client: okhttp3.OkHttpClient,
|
||||
val tempFile: File?,
|
||||
)
|
||||
|
||||
internal fun ChatMessageContent.isVideoAttachment(): Boolean = type == "video" || mimeType?.startsWith("video/") == true
|
||||
|
||||
internal fun ChatMessageContent.hasPlayableMediaArtifact(): Boolean = !artifactId.isNullOrBlank()
|
||||
|
||||
@Composable
|
||||
internal fun ChatMediaAttachmentLabel(content: ChatMessageContent) {
|
||||
val isAudio = content.isAudioAttachment()
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Icon(
|
||||
imageVector = if (isAudio) Icons.Default.Mic else Icons.Default.Videocam,
|
||||
contentDescription = null,
|
||||
modifier = Modifier.size(16.dp),
|
||||
tint = ClawTheme.colors.textMuted,
|
||||
)
|
||||
Text(
|
||||
text =
|
||||
content.fileName?.takeIf(String::isNotBlank)
|
||||
?: if (isAudio) nativeString("Voice note") else nativeString("Video"),
|
||||
style = ClawTheme.type.body,
|
||||
color = ClawTheme.colors.text,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
content.durationMs?.let { duration ->
|
||||
Text(
|
||||
text = formatVoiceNoteDuration(duration),
|
||||
style = ClawTheme.type.caption,
|
||||
color = ClawTheme.colors.textMuted,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,8 @@ import ai.openclaw.app.chat.OUTBOX_BRANCH_CHANGED_ERROR
|
||||
import ai.openclaw.app.chat.chatOutboxDisplayError
|
||||
import ai.openclaw.app.chat.normalizeVisibleChatMessageRole
|
||||
import ai.openclaw.app.gateway.GatewayLoadedImage
|
||||
import ai.openclaw.app.gateway.GatewayLoadedMedia
|
||||
import ai.openclaw.app.gateway.GatewayMediaKind
|
||||
import ai.openclaw.app.i18n.nativeString
|
||||
import ai.openclaw.app.i18n.nativeStringResource
|
||||
import ai.openclaw.app.tools.ToolDisplayRegistry
|
||||
@@ -107,6 +109,8 @@ internal fun ChatMessageBubble(
|
||||
onToggleListen: ((String, String) -> Unit)? = null,
|
||||
imageResolverReady: Boolean = false,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage? = { null },
|
||||
inlineMediaPlaybackBlocked: Boolean = false,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia? = { _, _ -> null },
|
||||
) {
|
||||
val role = normalizeVisibleChatMessageRole(message.role) ?: return
|
||||
val style = bubbleStyle(role)
|
||||
@@ -117,7 +121,7 @@ internal fun ChatMessageBubble(
|
||||
when (part.type) {
|
||||
"text" -> !part.text.isNullOrBlank()
|
||||
"image" -> !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank()
|
||||
else -> part.isAudioAttachment()
|
||||
else -> part.isAudioAttachment() || part.isVideoAttachment()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,6 +152,8 @@ internal fun ChatMessageBubble(
|
||||
textColor = mobileText,
|
||||
imageResolverReady = imageResolverReady,
|
||||
loadImageArtifact = loadImageArtifact,
|
||||
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
|
||||
loadMediaArtifact = loadMediaArtifact,
|
||||
)
|
||||
ChatMessageLinkPreview(messageId = message.id, role = role, content = displayableContent)
|
||||
messageSpeech?.let { speech ->
|
||||
@@ -235,6 +241,8 @@ private fun ChatMessageBody(
|
||||
textColor: Color,
|
||||
imageResolverReady: Boolean,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
inlineMediaPlaybackBlocked: Boolean,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
for (part in content) {
|
||||
@@ -243,7 +251,19 @@ private fun ChatMessageBody(
|
||||
val text = part.text ?: continue
|
||||
ChatMarkdown(text = text, textColor = textColor)
|
||||
}
|
||||
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
|
||||
part.isAudioAttachment() && part.hasPlayableMediaArtifact() ->
|
||||
ChatAudioPlayerCard(
|
||||
content = part,
|
||||
playbackBlocked = inlineMediaPlaybackBlocked,
|
||||
loadMedia = loadMediaArtifact,
|
||||
)
|
||||
part.isVideoAttachment() && part.hasPlayableMediaArtifact() ->
|
||||
ChatVideoPlayerCard(
|
||||
content = part,
|
||||
playbackBlocked = inlineMediaPlaybackBlocked,
|
||||
loadMedia = loadMediaArtifact,
|
||||
)
|
||||
part.isAudioAttachment() || part.isVideoAttachment() -> ChatMediaAttachmentLabel(content = part)
|
||||
part.type == "image" && !part.base64.isNullOrBlank() ->
|
||||
ChatBase64Image(base64 = part.base64, mimeType = part.mimeType)
|
||||
part.type == "image" && !part.artifactId.isNullOrBlank() ->
|
||||
|
||||
@@ -33,6 +33,8 @@ import ai.openclaw.app.chat.resolveChatComposerOwner
|
||||
import ai.openclaw.app.chat.resolveGatewayDefaultAgentId
|
||||
import ai.openclaw.app.currentAppLanguage
|
||||
import ai.openclaw.app.gateway.GatewayLoadedImage
|
||||
import ai.openclaw.app.gateway.GatewayLoadedMedia
|
||||
import ai.openclaw.app.gateway.GatewayMediaKind
|
||||
import ai.openclaw.app.i18n.NativeText
|
||||
import ai.openclaw.app.i18n.joinedNativeText
|
||||
import ai.openclaw.app.i18n.nativeString
|
||||
@@ -296,6 +298,7 @@ fun ChatScreen(
|
||||
val micCooldown by viewModel.micCooldown.collectAsState()
|
||||
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
|
||||
val talkModeListening by viewModel.talkModeListening.collectAsState()
|
||||
val inlineMediaPlaybackBlocked = messageSpeechState != null || talkModeEnabled || talkModeListening
|
||||
val thinkingSupported =
|
||||
chatThinkingSupported(
|
||||
selection = thinkingLevelSelection,
|
||||
@@ -743,8 +746,10 @@ fun ChatScreen(
|
||||
},
|
||||
speechState = messageSpeechState,
|
||||
onToggleListen = viewModel::toggleChatMessageSpeech,
|
||||
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
|
||||
resolveInlineWidgetResource = viewModel::resolveInlineWidgetResource,
|
||||
loadImageArtifact = viewModel::loadChatImageArtifact,
|
||||
loadMediaArtifact = viewModel::loadChatMediaArtifact,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
|
||||
@@ -1263,8 +1268,10 @@ private fun ChatMessageList(
|
||||
onForkMessage: (String) -> Unit,
|
||||
speechState: MessageSpeechState?,
|
||||
onToggleListen: (String, String) -> Unit,
|
||||
inlineMediaPlaybackBlocked: Boolean,
|
||||
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
val baseTimeline =
|
||||
@@ -1338,9 +1345,11 @@ private fun ChatMessageList(
|
||||
onForkMessage = onForkMessage,
|
||||
speechState = speechState,
|
||||
onToggleListen = onToggleListen,
|
||||
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
|
||||
inlineWidgetResolverReady = healthOk,
|
||||
resolveInlineWidgetResource = resolveInlineWidgetResource,
|
||||
loadImageArtifact = loadImageArtifact,
|
||||
loadMediaArtifact = loadMediaArtifact,
|
||||
)
|
||||
is ChatTimelineItem.OutboxCommand ->
|
||||
ChatOutboxBubble(
|
||||
@@ -1382,9 +1391,11 @@ private fun ChatMessageList(
|
||||
onForkMessage = onForkMessage,
|
||||
speechState = null,
|
||||
onToggleListen = onToggleListen,
|
||||
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
|
||||
inlineWidgetResolverReady = healthOk,
|
||||
resolveInlineWidgetResource = resolveInlineWidgetResource,
|
||||
loadImageArtifact = loadImageArtifact,
|
||||
loadMediaArtifact = loadMediaArtifact,
|
||||
)
|
||||
ChatTimelineItem.Thinking -> {
|
||||
val run = workingRun
|
||||
@@ -1632,9 +1643,11 @@ private fun ChatBubble(
|
||||
onForkMessage: (String) -> Unit,
|
||||
speechState: MessageSpeechState?,
|
||||
onToggleListen: (String, String) -> Unit,
|
||||
inlineMediaPlaybackBlocked: Boolean,
|
||||
inlineWidgetResolverReady: Boolean,
|
||||
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
|
||||
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
|
||||
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
|
||||
) {
|
||||
val normalizedRole = role.trim().lowercase(Locale.US)
|
||||
val isUser = normalizedRole == "user"
|
||||
@@ -1650,7 +1663,7 @@ private fun ChatBubble(
|
||||
visible
|
||||
}
|
||||
"canvas" -> normalizedRole == "assistant" && part.widget != null
|
||||
else -> part.isAudioAttachment()
|
||||
else -> part.isAudioAttachment() || part.isVideoAttachment()
|
||||
}
|
||||
}
|
||||
val omittedImageCount = (visibleImageCount - 4).coerceAtLeast(0)
|
||||
@@ -1716,7 +1729,19 @@ private fun ChatBubble(
|
||||
when {
|
||||
part.type == "text" && !collapsibleUserText -> ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live)
|
||||
part.type == "text" -> Unit
|
||||
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
|
||||
part.isAudioAttachment() && part.hasPlayableMediaArtifact() ->
|
||||
ChatAudioPlayerCard(
|
||||
content = part,
|
||||
playbackBlocked = inlineMediaPlaybackBlocked,
|
||||
loadMedia = loadMediaArtifact,
|
||||
)
|
||||
part.isVideoAttachment() && part.hasPlayableMediaArtifact() ->
|
||||
ChatVideoPlayerCard(
|
||||
content = part,
|
||||
playbackBlocked = inlineMediaPlaybackBlocked,
|
||||
loadMedia = loadMediaArtifact,
|
||||
)
|
||||
part.isAudioAttachment() || part.isVideoAttachment() -> ChatMediaAttachmentLabel(content = part)
|
||||
part.type == "image" ->
|
||||
if (!part.base64.isNullOrBlank()) {
|
||||
ChatBase64Image(
|
||||
|
||||
@@ -36,6 +36,7 @@ class AndroidLicenseNoticesTest {
|
||||
|
||||
assertEquals(
|
||||
listOf(
|
||||
"AndroidX Media3",
|
||||
"AndroidX Room",
|
||||
"AndroidX Wear",
|
||||
"Bouncy Castle Provider",
|
||||
|
||||
+50
-3
@@ -320,6 +320,18 @@ class ChatMessageContentParsingTest {
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun derivesArtifactIdentityForManagedAudioAndVideoBlocks() {
|
||||
val attachmentId = "22222222-2222-4222-8222-222222222222"
|
||||
val url = "/api/chat/media/outgoing/main/$attachmentId/full"
|
||||
|
||||
assertEquals("artifact_managed_media_$attachmentId", managedMediaArtifactId(url))
|
||||
assertEquals(
|
||||
"artifact_managed_media_$attachmentId",
|
||||
parseChatMessageContent(Json.parseToJsonElement("""{"type":"video","mimeType":"video/mp4","url":"$url"}"""))?.artifactId,
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dropsOversizedInlineImageContentBeforeRendering() {
|
||||
val oversized = "A".repeat(CHAT_IMAGE_MAX_BASE64_CHARS + 1)
|
||||
@@ -335,14 +347,27 @@ class ChatMessageContentParsingTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun parsesDirectAndAttachmentAudioBlocks() {
|
||||
fun dropsInlineAudioAndVideoContentThatRequiresManagedArtifacts() {
|
||||
val audio = Json.parseToJsonElement("""{"type":"audio","mimeType":"audio/mpeg","content":"audio-bytes"}""")
|
||||
val video = Json.parseToJsonElement("""{"type":"video","mimeType":"video/mp4","content":"video-bytes"}""")
|
||||
|
||||
assertEquals(ChatMessageContent(type = "audio", mimeType = "audio/mpeg"), parseChatMessageContent(audio))
|
||||
assertEquals(ChatMessageContent(type = "video", mimeType = "video/mp4"), parseChatMessageContent(video))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun parsesDirectAndAttachmentAudioVideoBlocks() {
|
||||
val direct =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"audio","mimeType":"audio/mp4","fileName":"voice.m4a"}""",
|
||||
)
|
||||
val attachment =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mpeg","label":"reply.mp3"}}""",
|
||||
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mpeg","label":"reply.mp3","artifactId":"artifact_managed_media_33333333-3333-4333-8333-333333333333","url":"/api/chat/media/outgoing/main/33333333-3333-4333-8333-333333333333/full","sizeBytes":4096,"durationMs":2100}}""",
|
||||
)
|
||||
val video =
|
||||
Json.parseToJsonElement(
|
||||
"""{"type":"attachment","attachment":{"kind":"video","mimeType":"video/mp4","fileName":"demo.mp4","artifactId":"artifact_managed_media_44444444-4444-4444-8444-444444444444","url":"/api/chat/media/outgoing/main/44444444-4444-4444-8444-444444444444/full","sizeBytes":8192,"durationMs":5300,"width":1920,"height":1080}}""",
|
||||
)
|
||||
|
||||
assertEquals(
|
||||
@@ -350,9 +375,31 @@ class ChatMessageContentParsingTest {
|
||||
parseChatMessageContent(direct),
|
||||
)
|
||||
assertEquals(
|
||||
ChatMessageContent(type = "audio", mimeType = "audio/mpeg", fileName = "reply.mp3"),
|
||||
ChatMessageContent(
|
||||
type = "audio",
|
||||
mimeType = "audio/mpeg",
|
||||
fileName = "reply.mp3",
|
||||
artifactId = "artifact_managed_media_33333333-3333-4333-8333-333333333333",
|
||||
url = "/api/chat/media/outgoing/main/33333333-3333-4333-8333-333333333333/full",
|
||||
sizeBytes = 4096,
|
||||
durationMs = 2100,
|
||||
),
|
||||
parseChatMessageContent(attachment),
|
||||
)
|
||||
assertEquals(
|
||||
ChatMessageContent(
|
||||
type = "video",
|
||||
mimeType = "video/mp4",
|
||||
fileName = "demo.mp4",
|
||||
artifactId = "artifact_managed_media_44444444-4444-4444-8444-444444444444",
|
||||
url = "/api/chat/media/outgoing/main/44444444-4444-4444-8444-444444444444/full",
|
||||
width = 1920,
|
||||
height = 1080,
|
||||
sizeBytes = 8192,
|
||||
durationMs = 5300,
|
||||
),
|
||||
parseChatMessageContent(video),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -76,6 +76,44 @@ class RoomChatTranscriptCacheTest {
|
||||
assertEquals(listOf("run-1:user", null, null), loaded.map { it.idempotencyKey })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun transcriptRoundTripKeepsManagedAudioAndVideoMetadata() =
|
||||
runTest {
|
||||
val store = cache()
|
||||
val audio =
|
||||
ChatMessageContent(
|
||||
type = "audio",
|
||||
mimeType = "audio/mpeg",
|
||||
fileName = "reply.mp3",
|
||||
artifactId = "artifact_managed_media_33333333-3333-4333-8333-333333333333",
|
||||
durationMs = 2_100,
|
||||
)
|
||||
val video =
|
||||
ChatMessageContent(
|
||||
type = "video",
|
||||
mimeType = "video/mp4",
|
||||
fileName = "demo.mp4",
|
||||
artifactId = "artifact_managed_media_44444444-4444-4444-8444-444444444444",
|
||||
durationMs = 5_300,
|
||||
width = 1920,
|
||||
height = 1080,
|
||||
)
|
||||
store.saveTranscript(
|
||||
gatewayId = "gateway-a",
|
||||
agentId = "main",
|
||||
sessionKey = "main",
|
||||
messages =
|
||||
listOf(
|
||||
ChatMessage(id = "audio", role = "assistant", content = listOf(audio), timestampMs = 10),
|
||||
ChatMessage(id = "video", role = "assistant", content = listOf(video), timestampMs = 11),
|
||||
),
|
||||
)
|
||||
|
||||
val loaded = store.loadTranscript("gateway-a", "main", "main")
|
||||
|
||||
assertEquals(listOf(audio, video), loaded.map { it.content.single() })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun legacyStringArrayTranscriptRowsRemainReadable() =
|
||||
runTest {
|
||||
|
||||
+24
-4
@@ -63,7 +63,7 @@ private class NoopDeviceAuthStore : DeviceAuthTokenStore {
|
||||
@Config(sdk = [34])
|
||||
class GatewaySessionCustomHeadersTest {
|
||||
@Test
|
||||
fun managedImageDownload_usesArtifactTicketWithoutGatewayBearer() =
|
||||
fun managedMediaDownload_usesArtifactTicketWithoutGatewayBearer() =
|
||||
runBlocking {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
val json = Json { ignoreUnknownKeys = true }
|
||||
@@ -73,6 +73,9 @@ class GatewaySessionCustomHeadersTest {
|
||||
val attachmentId = "11111111-1111-4111-8111-111111111111"
|
||||
val artifactId = "artifact_managed_image_$attachmentId"
|
||||
val imagePath = "/api/chat/media/outgoing/main/$attachmentId/full?mediaTicket=ticket"
|
||||
val videoAttachmentId = "22222222-2222-4222-8222-222222222222"
|
||||
val videoArtifactId = "artifact_managed_media_$videoAttachmentId"
|
||||
val videoPath = "/api/chat/media/outgoing/main/$videoAttachmentId/full?mediaTicket=video-ticket"
|
||||
val server =
|
||||
MockWebServer().apply {
|
||||
dispatcher =
|
||||
@@ -106,9 +109,20 @@ class GatewaySessionCustomHeadersTest {
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"snapshot":{"sessionDefaults":{"mainSessionKey":"main"}}}}""",
|
||||
)
|
||||
"artifacts.download" ->
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
|
||||
)
|
||||
if (frame["params"]
|
||||
?.jsonObject
|
||||
?.get("artifactId")
|
||||
?.jsonPrimitive
|
||||
?.content == videoArtifactId
|
||||
) {
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"artifact":{"id":"$videoArtifactId","type":"video","mimeType":"video/mp4","download":{"mode":"url"}},"url":"$videoPath"}}""",
|
||||
)
|
||||
} else {
|
||||
webSocket.send(
|
||||
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -164,6 +178,12 @@ class GatewaySessionCustomHeadersTest {
|
||||
val request = withTimeout(TEST_TIMEOUT_MS) { imageRequest.await() }
|
||||
assertNull(request.getHeader("Authorization"))
|
||||
assertEquals("image/*", request.getHeader("Accept"))
|
||||
|
||||
val streamed =
|
||||
session.loadMediaArtifact(stableId, "main", "main", videoArtifactId, GatewayMediaKind.Video) as GatewayLoadedMedia.Streaming
|
||||
assertEquals("http://127.0.0.1:${server.port}$videoPath", streamed.url)
|
||||
assertEquals("video/*", streamed.headers["Accept"])
|
||||
assertEquals("video/mp4", streamed.mimeType)
|
||||
} finally {
|
||||
session.disconnectAndJoin()
|
||||
scope.cancel()
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package ai.openclaw.app.ui.chat
|
||||
|
||||
import ai.openclaw.app.chat.ChatMessage
|
||||
import ai.openclaw.app.chat.ChatMessageContent
|
||||
import androidx.compose.ui.test.assertCountEquals
|
||||
import androidx.compose.ui.test.assertIsDisplayed
|
||||
import androidx.compose.ui.test.junit4.v2.createComposeRule
|
||||
import androidx.compose.ui.test.onAllNodesWithContentDescription
|
||||
import androidx.compose.ui.test.onNodeWithText
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertSame
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
class ChatMediaPlayerTest {
|
||||
private class FakePlayer(
|
||||
var positionMs: Long = 0L,
|
||||
) {
|
||||
var paused = false
|
||||
var released = false
|
||||
var playCount = 0
|
||||
|
||||
fun pause() {
|
||||
paused = true
|
||||
}
|
||||
|
||||
fun play() {
|
||||
paused = false
|
||||
playCount += 1
|
||||
}
|
||||
|
||||
fun release() {
|
||||
released = true
|
||||
}
|
||||
}
|
||||
|
||||
@get:Rule
|
||||
val composeRule = createComposeRule()
|
||||
|
||||
@Test
|
||||
fun claimHandoffReleasesPreviousPlaybackInstance() {
|
||||
val first = FakePlayer()
|
||||
val second = FakePlayer()
|
||||
val claims = ChatMediaPlaybackClaims<FakePlayer>(FakePlayer::pause, FakePlayer::release)
|
||||
|
||||
claims.claim(first)
|
||||
claims.claim(second)
|
||||
|
||||
assertTrue(first.released)
|
||||
assertFalse(second.released)
|
||||
assertSame(second, claims.active)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun pauseThenPlayResumesPositionWithoutRedownload() {
|
||||
var downloadCount = 0
|
||||
val player = FakePlayer(positionMs = 4_200L).also { downloadCount += 1 }
|
||||
val claims = ChatMediaPlaybackClaims<FakePlayer>(FakePlayer::pause, FakePlayer::release)
|
||||
|
||||
claims.claim(player)
|
||||
claims.pauseIf { it === player }
|
||||
claims.claim(player)
|
||||
player.play()
|
||||
|
||||
assertEquals(1, downloadCount)
|
||||
assertEquals(4_200L, player.positionMs)
|
||||
assertEquals(1, player.playCount)
|
||||
assertFalse(player.paused)
|
||||
assertFalse(player.released)
|
||||
assertSame(player, claims.active)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun legacyMediaPartsRenderLabelsWithoutPlayControlsOrClaims() {
|
||||
val audio =
|
||||
ChatMessageContent(
|
||||
type = "audio",
|
||||
mimeType = "audio/mpeg",
|
||||
fileName = "legacy.mp3",
|
||||
durationMs = 4_000,
|
||||
)
|
||||
val video =
|
||||
ChatMessageContent(
|
||||
type = "video",
|
||||
mimeType = "video/mp4",
|
||||
fileName = "legacy.mp4",
|
||||
durationMs = 9_000,
|
||||
)
|
||||
var loadCount = 0
|
||||
|
||||
composeRule.setContent {
|
||||
ChatMessageBubble(
|
||||
message =
|
||||
ChatMessage(
|
||||
id = "legacy-media",
|
||||
role = "assistant",
|
||||
content = listOf(audio, video),
|
||||
timestampMs = 1,
|
||||
),
|
||||
loadMediaArtifact = { _, _ ->
|
||||
loadCount += 1
|
||||
null
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
composeRule.onNodeWithText("legacy.mp3").assertIsDisplayed()
|
||||
composeRule.onNodeWithText("legacy.mp4").assertIsDisplayed()
|
||||
composeRule.onNodeWithText("0:04").assertIsDisplayed()
|
||||
composeRule.onNodeWithText("0:09").assertIsDisplayed()
|
||||
composeRule.onAllNodesWithContentDescription("Play audio").assertCountEquals(0)
|
||||
composeRule.onAllNodesWithContentDescription("Play video").assertCountEquals(0)
|
||||
composeRule.runOnIdle {
|
||||
assertEquals(0, loadCount)
|
||||
assertFalse(audio.hasPlayableMediaArtifact())
|
||||
assertFalse(video.hasPlayableMediaArtifact())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ ksp = "2.3.10"
|
||||
ktlint-gradle = "14.2.0"
|
||||
kotlin = "2.4.10"
|
||||
material = "1.14.0"
|
||||
media3 = "1.10.1"
|
||||
okhttp = "5.4.0"
|
||||
play-services-wearable = "20.0.1"
|
||||
barcode-scanning = "17.3.0"
|
||||
@@ -87,6 +88,9 @@ kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutine
|
||||
kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" }
|
||||
kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization-json" }
|
||||
material = { module = "com.google.android.material:material", version.ref = "material" }
|
||||
media3-datasource-okhttp = { module = "androidx.media3:media3-datasource-okhttp", version.ref = "media3" }
|
||||
media3-exoplayer = { module = "androidx.media3:media3-exoplayer", version.ref = "media3" }
|
||||
media3-ui = { module = "androidx.media3:media3-ui", version.ref = "media3" }
|
||||
mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp" }
|
||||
okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
|
||||
play-services-wearable = { module = "com.google.android.gms:play-services-wearable", version.ref = "play-services-wearable" }
|
||||
|
||||
Reference in New Issue
Block a user