feat(android): inline audio/video players with Media3 (#115916)

* feat(android): add inline media players

* fix(android): release inactive media players

* fix(android): preserve paused media playback

* test(android): include Media3 license notice

* chore(android): refresh native i18n inventory
This commit is contained in:
Peter Steinberger
2026-07-29 10:26:45 -04:00
committed by GitHub
parent 26b236dae3
commit 80176dab8d
18 changed files with 1600 additions and 306 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,18 @@
AndroidX Media3
Artifacts:
- androidx.media3:media3-datasource-okhttp:1.10.1
- androidx.media3:media3-exoplayer:1.10.1
- androidx.media3:media3-ui:1.10.1
Copyright 2016 The Android Open Source Project
Licensed under the Apache License, Version 2.0.
You may obtain a copy of the License at:
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+3
View File
@@ -336,6 +336,9 @@ dependencies {
ksp(libs.androidx.room.compiler)
implementation(libs.androidx.exifinterface)
implementation(libs.okhttp)
implementation(libs.media3.datasource.okhttp)
implementation(libs.media3.exoplayer)
implementation(libs.media3.ui)
implementation(libs.bcprov)
implementation(libs.coil.compose)
implementation(libs.coil.svg)
@@ -23,6 +23,7 @@ import ai.openclaw.app.chat.SessionRewindResult
import ai.openclaw.app.chat.defaultChatThinkingLevelSelection
import ai.openclaw.app.chat.resolveChatComposerOwner
import ai.openclaw.app.gateway.GatewayEndpoint
import ai.openclaw.app.gateway.GatewayMediaKind
import ai.openclaw.app.gateway.GatewayRegistryEntry
import ai.openclaw.app.gateway.GatewayRegistryEntryKind
import ai.openclaw.app.gateway.GatewayUpdateAvailableSummary
@@ -1310,6 +1311,11 @@ class MainViewModel private constructor(
internal suspend fun loadChatImageArtifact(artifactId: String) = ensureRuntime().loadChatImageArtifact(artifactId)
internal suspend fun loadChatMediaArtifact(
artifactId: String,
kind: GatewayMediaKind,
) = ensureRuntime().loadChatMediaArtifact(artifactId, kind)
fun requestCanvasRehydrate(source: String = "screen_tab") {
ensureRuntime().requestCanvasRehydrate(source = source, force = true)
}
@@ -39,6 +39,7 @@ import ai.openclaw.app.gateway.DeviceIdentityStore
import ai.openclaw.app.gateway.GatewayDiscovery
import ai.openclaw.app.gateway.GatewayEndpoint
import ai.openclaw.app.gateway.GatewayEvent
import ai.openclaw.app.gateway.GatewayMediaKind
import ai.openclaw.app.gateway.GatewayMethod
import ai.openclaw.app.gateway.GatewayRegistryEntry
import ai.openclaw.app.gateway.GatewayRegistryEntryKind
@@ -4915,6 +4916,11 @@ class NodeRuntime private constructor(
internal suspend fun loadChatImageArtifact(artifactId: String) = chat.loadImageArtifact(artifactId)
internal suspend fun loadChatMediaArtifact(
artifactId: String,
kind: GatewayMediaKind,
) = chat.loadMediaArtifact(artifactId, kind)
fun loadChat(
sessionKey: String,
ownerAgentId: String? = null,
@@ -2,6 +2,8 @@ package ai.openclaw.app.chat
import ai.openclaw.app.GatewayModelSummary
import ai.openclaw.app.gateway.GatewayLoadedImage
import ai.openclaw.app.gateway.GatewayLoadedMedia
import ai.openclaw.app.gateway.GatewayMediaKind
import ai.openclaw.app.gateway.GatewayRequestDefinitiveFailure
import ai.openclaw.app.gateway.GatewayRequestNotEnqueued
import ai.openclaw.app.gateway.GatewayRequestOutcomeUnknown
@@ -57,7 +59,7 @@ internal const val SESSION_LIST_FETCH_LIMIT = 200
private val QUESTION_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
private val SWARM_REFRESH_RETRY_DELAYS_MS = longArrayOf(1_000L, 2_000L, 4_000L)
private const val SESSION_EDITOR_MAX_BASE64_CHARS = ((OUTBOX_MAX_COMMAND_ATTACHMENT_BYTES + 2) / 3) * 4
private val MANAGED_IMAGE_PATH_REGEX =
private val MANAGED_MEDIA_PATH_REGEX =
Regex("^/api/chat/media/outgoing/[^/]+/([0-9a-fA-F-]{36})/full(?:\\?.*)?$")
internal fun chatOutboxQueueFailureText(): NativeText = ChatController.queueFailureText()
@@ -120,6 +122,13 @@ class ChatController internal constructor(
agentId: String?,
artifactId: String,
) -> GatewayLoadedImage? = { _, _, _, _ -> null },
private val loadGatewayMediaArtifact: suspend (
gatewayId: String?,
sessionKey: String,
agentId: String?,
artifactId: String,
kind: GatewayMediaKind,
) -> GatewayLoadedMedia? = { _, _, _, _, _ -> null },
private val commandOutbox: ChatCommandOutbox? = null,
private val recordModelRecent: (String) -> Unit = {},
private val onSessionDeleted: (ChatSessionDeletion) -> Unit = {},
@@ -154,6 +163,9 @@ class ChatController internal constructor(
loadGatewayImageArtifact = { gatewayId, sessionKey, agentId, artifactId ->
session.loadImageArtifact(gatewayId, sessionKey, agentId, artifactId)
},
loadGatewayMediaArtifact = { gatewayId, sessionKey, agentId, artifactId, kind ->
session.loadMediaArtifact(gatewayId, sessionKey, agentId, artifactId, kind)
},
commandOutbox = commandOutbox,
recordModelRecent = recordModelRecent,
onSessionDeleted = onSessionDeleted,
@@ -171,6 +183,22 @@ class ChatController internal constructor(
)
}
suspend fun loadMediaArtifact(
artifactId: String,
kind: GatewayMediaKind,
): GatewayLoadedMedia? {
val normalizedArtifactId = artifactId.trim().takeIf(String::isNotEmpty) ?: return null
if (kind == GatewayMediaKind.Image) return null
val sessionKey = normalizeRequestedSessionKey(_sessionKey.value)
return loadGatewayMediaArtifact(
currentCacheScope()?.gatewayId,
sessionKey,
resolveAgentIdForSessionKey(sessionKey),
normalizedArtifactId,
kind,
)
}
private var appliedMainSessionKey = "main"
private val cacheMutationMutex = Mutex()
private val defaultAgentPersistenceMutex = Mutex()
@@ -6671,7 +6699,7 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
text = obj["text"].asStringOrNull() ?: obj["content"].asStringOrNull(),
)
"image", "audio" -> {
"image", "audio", "video" -> {
val type = obj["type"].asStringOrNull() ?: "image"
val inlineContent = obj["content"].asStringOrNull()?.takeIf { it.isNotBlank() }
val url = obj["url"].asStringOrNull()
@@ -6679,25 +6707,42 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
type = type,
mimeType = obj["mimeType"].asStringOrNull(),
fileName = obj["fileName"].asStringOrNull(),
artifactId = obj["artifactId"].asStringOrNull() ?: managedImageArtifactId(url),
artifactId =
obj["artifactId"].asStringOrNull()
?: if (type == "image") managedImageArtifactId(url) else managedMediaArtifactId(url),
url = url,
openUrl = obj["openUrl"].asStringOrNull(),
alt = obj["alt"].asStringOrNull(),
width = obj["width"].asLongOrNull()?.toInt(),
height = obj["height"].asLongOrNull()?.toInt(),
sizeBytes = obj["sizeBytes"].asLongOrNull(),
base64 = inlineContent?.takeIf { type != "image" || it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
base64 = inlineContent?.takeIf { type == "image" && it.length <= CHAT_IMAGE_MAX_BASE64_CHARS },
durationMs = obj["durationMs"].asLongOrNull(),
)
}
"attachment" -> {
val attachment = obj["attachment"].asObjectOrNull() ?: return null
val mimeType = attachment["mimeType"].asStringOrNull()
if (attachment["kind"].asStringOrNull() != "audio" && mimeType?.startsWith("audio/") != true) return null
val type =
when {
attachment["kind"].asStringOrNull() == "audio" || mimeType?.startsWith("audio/") == true -> "audio"
attachment["kind"].asStringOrNull() == "video" || mimeType?.startsWith("video/") == true -> "video"
else -> return null
}
val url = attachment["url"].asStringOrNull()
ChatMessageContent(
type = "audio",
type = type,
mimeType = mimeType,
fileName = attachment["label"].asStringOrNull(),
fileName = attachment["fileName"].asStringOrNull() ?: attachment["label"].asStringOrNull(),
artifactId = attachment["artifactId"].asStringOrNull() ?: managedMediaArtifactId(url),
url = url,
openUrl = attachment["openUrl"].asStringOrNull(),
alt = attachment["alt"].asStringOrNull(),
width = attachment["width"].asLongOrNull()?.toInt(),
height = attachment["height"].asLongOrNull()?.toInt(),
sizeBytes = attachment["sizeBytes"].asLongOrNull(),
durationMs = attachment["durationMs"].asLongOrNull(),
)
}
@@ -6730,13 +6775,22 @@ internal fun parseChatMessageContent(el: JsonElement): ChatMessageContent? {
}
internal fun managedImageArtifactId(rawUrl: String?): String? {
val attachmentId = managedMediaAttachmentId(rawUrl) ?: return null
return "artifact_managed_image_$attachmentId"
}
internal fun managedMediaArtifactId(rawUrl: String?): String? {
val attachmentId = managedMediaAttachmentId(rawUrl) ?: return null
return "artifact_managed_media_$attachmentId"
}
private fun managedMediaAttachmentId(rawUrl: String?): String? {
val match =
rawUrl
?.trim()
?.let(MANAGED_IMAGE_PATH_REGEX::matchEntire)
?.let(MANAGED_MEDIA_PATH_REGEX::matchEntire)
?: return null
val attachmentId = runCatching { UUID.fromString(match.groupValues[1]).toString() }.getOrNull() ?: return null
return "artifact_managed_image_$attachmentId"
return runCatching { UUID.fromString(match.groupValues[1]).toString() }.getOrNull()
}
internal fun parseChatMessageContents(obj: JsonObject): List<ChatMessageContent> {
@@ -63,7 +63,7 @@ data class ChatTranscriptAnchorState(
)
/**
* One content part in a chat message; images carry either bounded base64 or a managed artifact reference.
* One content part in a chat message; media carries either bounded base64 or a managed artifact reference.
*/
data class ChatMessageContent(
val type: String = "text",
@@ -31,6 +31,7 @@ private data class CachedMessageContent(
val width: Int? = null,
val height: Int? = null,
val sizeBytes: Long? = null,
val durationMs: Long? = null,
)
/**
@@ -108,7 +109,7 @@ internal data class CachedMessageEntity(
val sessionKey: String,
val rowOrder: Int,
val role: String,
// JSON array of text and managed-image references; attachment bytes are never persisted.
// JSON array of text and managed-media references; attachment bytes are never persisted.
val textPartsJson: String,
val timestampMs: Long?,
// Kept so live history reconciliation can match cached rows by identity key.
@@ -318,6 +319,7 @@ class RoomChatTranscriptCache internal constructor(
width = part.width,
height = part.height,
sizeBytes = part.sizeBytes,
durationMs = part.durationMs,
)
},
timestampMs = row.timestampMs,
@@ -399,8 +401,8 @@ class RoomChatTranscriptCache internal constructor(
val gateway = scopedGatewayId(gatewayId) ?: return
val agent = scopedAgentId(agentId) ?: return
val key = sessionKey.trim().takeIf { it.isNotEmpty() } ?: return
// Persist small managed-image references, never attachment bytes. This keeps generated images
// visible offline without turning the disposable transcript cache into a binary store.
// Persist small managed-media references, never attachment bytes. Cards remain visible offline
// even though their short-lived download capability must be reacquired after reconnecting.
val rows =
messages
.mapNotNull { message ->
@@ -423,6 +425,20 @@ class RoomChatTranscriptCache internal constructor(
height = part.height,
sizeBytes = part.sizeBytes,
)
part.type == "audio" || part.type == "video" ->
CachedMessageContent(
type = part.type,
mimeType = part.mimeType,
fileName = part.fileName,
artifactId = part.artifactId,
url = part.url,
openUrl = part.openUrl,
alt = part.alt,
width = part.width,
height = part.height,
sizeBytes = part.sizeBytes,
durationMs = part.durationMs,
)
else -> null
}
}
@@ -35,6 +35,7 @@ import okhttp3.WebSocket
import okhttp3.WebSocketListener
import okio.Buffer
import java.net.URI
import java.util.Base64
import java.util.Locale
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
@@ -60,6 +61,32 @@ data class GatewayLoadedImage(
val mimeType: String,
)
enum class GatewayMediaKind(
val wireValue: String,
val maximumBufferedBytes: Long,
) {
Image("image", 12L * 1024L * 1024L),
Audio("audio", 16L * 1024L * 1024L),
Video("video", 0L),
}
sealed interface GatewayLoadedMedia {
data class Buffered(
val bytes: ByteArray,
val mimeType: String,
val headers: Map<String, String>,
val client: OkHttpClient,
) : GatewayLoadedMedia
/** A short-lived gateway capability resolved with the exact socket's HTTP transport policy. */
data class Streaming(
val url: String,
val headers: Map<String, String>,
val client: OkHttpClient,
val mimeType: String?,
) : GatewayLoadedMedia
}
/**
* Role, scopes, commands, and permission snapshot sent with the connect frame.
*/
@@ -629,6 +656,24 @@ class GatewaySession(
agentId: String?,
artifactId: String,
): GatewayLoadedImage? {
val loaded =
loadMediaArtifact(
expectedEndpointStableId = expectedEndpointStableId,
sessionKey = sessionKey,
agentId = agentId,
artifactId = artifactId,
kind = GatewayMediaKind.Image,
) as? GatewayLoadedMedia.Buffered ?: return null
return GatewayLoadedImage(bytes = loaded.bytes, mimeType = loaded.mimeType)
}
suspend fun loadMediaArtifact(
expectedEndpointStableId: String?,
sessionKey: String,
agentId: String?,
artifactId: String,
kind: GatewayMediaKind,
): GatewayLoadedMedia? {
val conn = readyConnection(expectedEndpointStableId) ?: return null
val params =
buildJsonObject {
@@ -640,16 +685,35 @@ class GatewaySession(
if (!response.ok) {
throw GatewayRequestRejected(response.error ?: ErrorShape("UNAVAILABLE", "artifact download failed"))
}
val ticketedPath =
response.payloadJson
?.let(::parseJsonOrNull)
.asObjectOrNull()
?.get("url")
val payload = response.payloadJson?.let(::parseJsonOrNull).asObjectOrNull() ?: return null
val artifact = payload["artifact"].asObjectOrNull()
// Older gateways returned only the managed-image URL. Audio/video shipped with the typed
// artifact envelope, so keep the narrow image fallback without weakening their type gate.
if (artifact == null && kind != GatewayMediaKind.Image) return null
if (artifact != null && artifact["type"].asStringOrNull()?.trim()?.lowercase(Locale.ROOT) != kind.wireValue) return null
val mimeType =
artifact
?.get("mimeType")
.asStringOrNull()
?.trim()
?.takeIf(String::isNotEmpty)
?: return null
val loaded = conn.loadTicketedImage(ticketedPath) ?: return null
?.lowercase(Locale.ROOT)
if (mimeType != null && !mimeType.startsWith("${kind.wireValue}/")) return null
val loaded =
payload["data"].asStringOrNull()?.let { encoded ->
if (kind == GatewayMediaKind.Video) return@let null
val maximumEncodedLength = ((kind.maximumBufferedBytes + 2L) / 3L) * 4L
if (encoded.length.toLong() > maximumEncodedLength) return@let null
val bytes = runCatching { Base64.getDecoder().decode(encoded) }.getOrNull() ?: return@let null
if (bytes.size.toLong() > kind.maximumBufferedBytes) return@let null
val resolvedMimeType = mimeType ?: return@let null
conn.bufferedMedia(bytes = bytes, mimeType = resolvedMimeType)
} ?: payload["url"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty)?.let { ticketedPath ->
if (kind == GatewayMediaKind.Video) {
conn.resolveTicketedMediaStream(ticketedPath, mimeType)
} else {
conn.loadTicketedMedia(ticketedPath, kind)
}
} ?: return null
return synchronized(lifecycleLock) {
loaded.takeIf { currentConnection === conn && conn.isReady() }
}
@@ -773,6 +837,11 @@ class GatewaySession(
val error: ErrorShape?,
)
private data class TicketedMediaRequest(
val url: String,
val headers: Map<String, String>,
)
private data class ConnectedGateway(
val pluginSurfaceUrls: Map<String, String>,
val mainSessionKey: String?,
@@ -869,22 +938,40 @@ class GatewaySession(
}
}
suspend fun loadTicketedImage(ticketedPath: String): GatewayLoadedImage? =
fun resolveTicketedMediaStream(
ticketedPath: String,
mimeType: String?,
): GatewayLoadedMedia.Streaming? {
val request = resolveTicketedMediaRequest(ticketedPath) ?: return null
return GatewayLoadedMedia.Streaming(
url = request.url,
headers = request.headers + ("Accept" to "video/*"),
client = client,
mimeType = mimeType,
)
}
fun bufferedMedia(
bytes: ByteArray,
mimeType: String,
): GatewayLoadedMedia.Buffered =
GatewayLoadedMedia.Buffered(
bytes = bytes,
mimeType = mimeType,
headers = mediaTransportHeaders(),
client = client,
)
suspend fun loadTicketedMedia(
ticketedPath: String,
kind: GatewayMediaKind,
): GatewayLoadedMedia.Buffered? =
withContext(Dispatchers.IO) {
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return@withContext null
val rawPath = uri.rawPath ?: return@withContext null
val rawQuery = uri.rawQuery ?: return@withContext null
if (uri.isAbsolute || uri.rawAuthority != null || uri.rawFragment != null) return@withContext null
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !rawQuery.contains("mediaTicket=")) {
return@withContext null
}
val scheme = if (tlsConfig != null) "https" else "http"
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
val request = Request.Builder().url(url).header("Accept", "image/*")
if (tlsConfig != null) {
for ((name, value) in GatewayCustomHeaders.sanitized(customHeadersProvider?.invoke(endpoint.stableId).orEmpty())) {
request.header(name, value)
}
if (kind == GatewayMediaKind.Video) return@withContext null
val resolved = resolveTicketedMediaRequest(ticketedPath) ?: return@withContext null
val request = Request.Builder().url(resolved.url).header("Accept", "${kind.wireValue}/*")
for ((name, value) in resolved.headers) {
request.header(name, value)
}
val call = client.newCall(request.build())
call.timeout().timeout(20, java.util.concurrent.TimeUnit.SECONDS)
@@ -892,8 +979,8 @@ class GatewaySession(
if (!response.isSuccessful) return@withContext null
val body = response.body
val mimeType = body.contentType()?.toString()?.lowercase(Locale.ROOT) ?: return@withContext null
if (!mimeType.startsWith("image/")) return@withContext null
val maximumBytes = 12L * 1024L * 1024L
if (!mimeType.startsWith("${kind.wireValue}/")) return@withContext null
val maximumBytes = kind.maximumBufferedBytes
val declaredLength = body.contentLength()
if (declaredLength > maximumBytes) return@withContext null
val buffer = Buffer()
@@ -905,10 +992,33 @@ class GatewaySession(
total += read
if (total > maximumBytes) return@withContext null
}
GatewayLoadedImage(bytes = buffer.readByteArray(), mimeType = mimeType)
bufferedMedia(bytes = buffer.readByteArray(), mimeType = mimeType)
}
}
private fun resolveTicketedMediaRequest(ticketedPath: String): TicketedMediaRequest? {
val uri = runCatching { URI(ticketedPath) }.getOrNull() ?: return null
val rawPath = uri.rawPath ?: return null
val rawQuery = uri.rawQuery ?: return null
if (uri.isAbsolute || uri.rawAuthority != null || uri.rawFragment != null) return null
val hasMediaTicket =
rawQuery
.split('&')
.any { field -> field.substringBefore('=') == "mediaTicket" && field.substringAfter('=', "").isNotEmpty() }
if (!rawPath.startsWith("/api/chat/media/outgoing/") || !hasMediaTicket) return null
val scheme = if (tlsConfig != null) "https" else "http"
val url = "$scheme://${formatGatewayAuthority(endpoint.host, endpoint.port)}$ticketedPath"
val headers = mediaTransportHeaders()
return TicketedMediaRequest(url = url, headers = headers)
}
private fun mediaTransportHeaders(): Map<String, String> =
if (tlsConfig == null) {
emptyMap()
} else {
GatewayCustomHeaders.sanitized(customHeadersProvider?.invoke(endpoint.stableId).orEmpty())
}
@OptIn(DelicateCoroutinesApi::class)
suspend fun sendRequestFrame(
method: String,
@@ -0,0 +1,738 @@
package ai.openclaw.app.ui.chat
import ai.openclaw.app.chat.ChatMessageContent
import ai.openclaw.app.gateway.GatewayLoadedMedia
import ai.openclaw.app.gateway.GatewayMediaKind
import ai.openclaw.app.i18n.nativeString
import ai.openclaw.app.ui.design.ClawTheme
import android.content.Context
import android.media.AudioFocusRequest
import android.media.AudioManager
import android.os.Handler
import android.os.Looper
import androidx.annotation.OptIn
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.aspectRatio
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.Pause
import androidx.compose.material.icons.filled.PlayArrow
import androidx.compose.material.icons.filled.Videocam
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.Slider
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableLongStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleEventObserver
import androidx.lifecycle.compose.LocalLifecycleOwner
import androidx.media3.common.AudioAttributes
import androidx.media3.common.C
import androidx.media3.common.MediaItem
import androidx.media3.common.PlaybackException
import androidx.media3.common.Player
import androidx.media3.common.util.UnstableApi
import androidx.media3.datasource.DefaultDataSource
import androidx.media3.datasource.okhttp.OkHttpDataSource
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
import androidx.media3.ui.AspectRatioFrameLayout
import androidx.media3.ui.PlayerView
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import java.io.File
internal class ChatMediaPlaybackClaims<T>(
private val pause: (T) -> Unit,
private val release: (T) -> Unit,
) {
var active: T? = null
private set
fun claim(value: T) {
if (active === value) return
val previous = active
active = null
previous?.let(release)
active = value
}
fun releaseIf(predicate: (T) -> Boolean): Boolean {
val previous = active?.takeIf(predicate) ?: return false
active = null
release(previous)
return true
}
fun pauseIf(predicate: (T) -> Boolean): Boolean {
val current = active?.takeIf(predicate) ?: return false
pause(current)
return true
}
fun releaseActive() {
val previous = active ?: return
active = null
release(previous)
}
}
private object ChatMediaPlaybackArbiter {
private data class AudioFocusHandle(
val manager: AudioManager,
val request: AudioFocusRequest,
)
private class ActivePlayback(
val player: ExoPlayer,
val onReleased: () -> Unit,
) {
var audioFocus: AudioFocusHandle? = null
}
private val mainHandler = Handler(Looper.getMainLooper())
private val claims = ChatMediaPlaybackClaims<ActivePlayback>(::pauseAndAbandon, ::stopAndRelease)
private var playbackIntentGeneration = 0L
@Synchronized
fun claimPlaybackIntent(preservePlayer: ExoPlayer? = null): Long {
if (claims.active?.player !== preservePlayer) claims.releaseActive()
playbackIntentGeneration += 1L
return playbackIntentGeneration
}
@Synchronized
fun isCurrentIntent(generation: Long): Boolean = playbackIntentGeneration == generation
@Synchronized
fun cancelIntent(generation: Long) {
if (playbackIntentGeneration == generation) playbackIntentGeneration += 1L
}
@Synchronized
fun pauseAll() {
playbackIntentGeneration += 1L
claims.pauseIf { true }
}
@Synchronized
fun registerPrepared(
player: ExoPlayer,
intentGeneration: Long,
onReleased: () -> Unit,
): Boolean {
if (playbackIntentGeneration != intentGeneration) return false
if (claims.active?.player === player) return true
claims.claim(ActivePlayback(player = player, onReleased = onReleased))
return true
}
@Synchronized
fun requestPlayback(
context: Context,
player: ExoPlayer,
intentGeneration: Long,
onReleased: () -> Unit,
): Boolean {
if (playbackIntentGeneration != intentGeneration) return false
val existing = claims.active?.takeIf { it.player === player }
if (existing?.audioFocus != null) return true
val audioManager = context.getSystemService(AudioManager::class.java)
val focusRequest =
AudioFocusRequest
.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT)
.setAudioAttributes(
android.media.AudioAttributes
.Builder()
.setUsage(android.media.AudioAttributes.USAGE_MEDIA)
.setContentType(android.media.AudioAttributes.CONTENT_TYPE_UNKNOWN)
.build(),
).setWillPauseWhenDucked(true)
.setOnAudioFocusChangeListener { change ->
if (change < 0) {
mainHandler.post { pause(player) }
}
}.build()
if (audioManager.requestAudioFocus(focusRequest) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
return false
}
val playback = existing ?: ActivePlayback(player = player, onReleased = onReleased).also(claims::claim)
playback.audioFocus = AudioFocusHandle(manager = audioManager, request = focusRequest)
return true
}
@Synchronized
fun pause(player: ExoPlayer): Boolean = claims.pauseIf { it.player === player }
@Synchronized
fun release(player: ExoPlayer): Boolean = claims.releaseIf { it.player === player }
private fun pauseAndAbandon(playback: ActivePlayback) {
playback.player.pause()
playback.audioFocus?.let { focus -> focus.manager.abandonAudioFocusRequest(focus.request) }
playback.audioFocus = null
}
private fun stopAndRelease(playback: ActivePlayback) {
pauseAndAbandon(playback)
playback.player.release()
playback.onReleased()
}
}
@Composable
internal fun ChatAudioPlayerCard(
content: ChatMessageContent,
playbackBlocked: Boolean,
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
) {
ChatMediaPlayerCard(
content = content,
kind = GatewayMediaKind.Audio,
playbackBlocked = playbackBlocked,
loadMedia = loadMedia,
)
}
@Composable
internal fun ChatVideoPlayerCard(
content: ChatMessageContent,
playbackBlocked: Boolean,
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
) {
ChatMediaPlayerCard(
content = content,
kind = GatewayMediaKind.Video,
playbackBlocked = playbackBlocked,
loadMedia = loadMedia,
)
}
@OptIn(UnstableApi::class)
@Composable
private fun ChatMediaPlayerCard(
content: ChatMessageContent,
kind: GatewayMediaKind,
playbackBlocked: Boolean,
loadMedia: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
) {
val context = LocalContext.current
val lifecycleOwner = LocalLifecycleOwner.current
val currentPlaybackBlocked by rememberUpdatedState(playbackBlocked)
val scope = rememberCoroutineScope()
var player by remember(content.artifactId, kind) { mutableStateOf<ExoPlayer?>(null) }
var tempFile by remember(content.artifactId, kind) { mutableStateOf<File?>(null) }
var loading by remember(content.artifactId, kind) { mutableStateOf(false) }
var error by remember(content.artifactId, kind) { mutableStateOf<String?>(null) }
var isPlaying by remember(content.artifactId, kind) { mutableStateOf(false) }
var positionMs by remember(content.artifactId, kind) { mutableLongStateOf(0L) }
var durationMs by remember(content.artifactId, kind) { mutableLongStateOf(content.durationMs?.coerceAtLeast(0L) ?: 0L) }
var playbackIntent by remember(content.artifactId, kind) { mutableLongStateOf(0L) }
fun clearPlayerState(
released: ExoPlayer,
releasedFile: File?,
) {
if (player === released) player = null
if (tempFile === releasedFile) tempFile = null
releasedFile?.delete()
isPlaying = false
positionMs = 0L
}
fun disposeUnclaimedPlayer(
released: ExoPlayer,
releasedFile: File?,
) {
released.release()
clearPlayerState(released, releasedFile)
}
fun requestPlayback(
requested: ExoPlayer,
requestedFile: File?,
intentGeneration: Long,
): Boolean =
ChatMediaPlaybackArbiter.requestPlayback(
context = context,
player = requested,
intentGeneration = intentGeneration,
onReleased = { clearPlayerState(requested, requestedFile) },
)
fun registerPrepared(
prepared: ExoPlayer,
preparedFile: File?,
intentGeneration: Long,
): Boolean =
ChatMediaPlaybackArbiter.registerPrepared(
player = prepared,
intentGeneration = intentGeneration,
onReleased = { clearPlayerState(prepared, preparedFile) },
)
fun pause() {
player?.let(ChatMediaPlaybackArbiter::pause)
}
fun play() {
if (playbackBlocked) return
val existing = player
if (existing != null) {
val intentGeneration = ChatMediaPlaybackArbiter.claimPlaybackIntent(preservePlayer = existing)
playbackIntent = intentGeneration
error = null
if (requestPlayback(existing, tempFile, intentGeneration)) {
if (existing.playbackState == Player.STATE_ENDED) existing.seekToDefaultPosition()
existing.play()
} else {
error = nativeString("Audio playback is unavailable")
}
return
}
val artifactId = content.artifactId?.trim()?.takeIf(String::isNotEmpty)
if (artifactId == null) {
error = nativeString("Media unavailable")
return
}
if (loading) return
val intentGeneration = ChatMediaPlaybackArbiter.claimPlaybackIntent()
playbackIntent = intentGeneration
loading = true
error = null
scope.launch {
val loaded =
try {
loadMedia(artifactId, kind)
} catch (error: CancellationException) {
throw error
} catch (_: Throwable) {
null
}
if (loaded == null) {
loading = false
error = nativeString("Media unavailable")
return@launch
}
if (!ChatMediaPlaybackArbiter.isCurrentIntent(intentGeneration)) {
loading = false
return@launch
}
val prepared = prepareMediaSource(context = context, loaded = loaded)
if (prepared == null) {
loading = false
error = nativeString("Media unavailable")
return@launch
}
if (!ChatMediaPlaybackArbiter.isCurrentIntent(intentGeneration)) {
prepared.tempFile?.delete()
loading = false
return@launch
}
tempFile?.delete()
tempFile = prepared.tempFile
val created =
runCatching { buildMediaPlayer(context = context, source = prepared) }.getOrElse {
prepared.tempFile?.delete()
tempFile = null
loading = false
error = nativeString("Media unavailable")
return@launch
}
created.addListener(
object : Player.Listener {
override fun onIsPlayingChanged(value: Boolean) {
isPlaying = value
}
override fun onPlaybackStateChanged(playbackState: Int) {
if (playbackState == Player.STATE_ENDED) {
ChatMediaPlaybackArbiter.pause(created)
}
}
override fun onPlayerError(playbackException: PlaybackException) {
if (!ChatMediaPlaybackArbiter.release(created)) {
disposeUnclaimedPlayer(created, prepared.tempFile)
}
error = nativeString("Media unavailable")
}
},
)
player = created
loading = false
if (!registerPrepared(created, prepared.tempFile, intentGeneration)) {
disposeUnclaimedPlayer(created, prepared.tempFile)
return@launch
}
val shouldPlay =
!currentPlaybackBlocked &&
lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)
if (shouldPlay && requestPlayback(created, prepared.tempFile, intentGeneration)) {
created.play()
} else if (shouldPlay) {
error = nativeString("Audio playback is unavailable")
}
}
}
LaunchedEffect(player) {
val activePlayer = player ?: return@LaunchedEffect
while (currentCoroutineContext().isActive) {
positionMs = activePlayer.currentPosition.coerceAtLeast(0L)
activePlayer.duration.takeIf { it != C.TIME_UNSET && it > 0L }?.let { durationMs = it }
delay(if (activePlayer.isPlaying) 250L else 1_000L)
}
}
LaunchedEffect(playbackBlocked) {
if (playbackBlocked) ChatMediaPlaybackArbiter.pauseAll()
}
DisposableEffect(lifecycleOwner, player) {
val observer =
LifecycleEventObserver { _, event ->
if (event == Lifecycle.Event.ON_STOP) ChatMediaPlaybackArbiter.pauseAll()
}
lifecycleOwner.lifecycle.addObserver(observer)
onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
}
DisposableEffect(content.artifactId, kind) {
onDispose {
ChatMediaPlaybackArbiter.cancelIntent(playbackIntent)
player?.let { activePlayer ->
val activeFile = tempFile
if (!ChatMediaPlaybackArbiter.release(activePlayer)) {
activePlayer.release()
activeFile?.delete()
}
}
tempFile?.delete()
}
}
if (kind == GatewayMediaKind.Video) {
VideoPlayerSurface(
content = content,
player = player,
loading = loading,
isPlaying = isPlaying,
playbackBlocked = playbackBlocked,
error = error,
onToggle = { if (isPlaying) pause() else play() },
)
} else {
AudioPlayerSurface(
content = content,
loading = loading,
isPlaying = isPlaying,
playbackBlocked = playbackBlocked,
error = error,
positionMs = positionMs,
durationMs = durationMs,
onToggle = { if (isPlaying) pause() else play() },
onSeek = { value ->
val target = value.toLong().coerceIn(0L, durationMs.coerceAtLeast(0L))
positionMs = target
player?.seekTo(target)
},
seekEnabled = player != null,
)
}
}
@Composable
private fun AudioPlayerSurface(
content: ChatMessageContent,
loading: Boolean,
isPlaying: Boolean,
playbackBlocked: Boolean,
error: String?,
positionMs: Long,
durationMs: Long,
onToggle: () -> Unit,
onSeek: (Float) -> Unit,
seekEnabled: Boolean,
) {
Surface(
shape = RoundedCornerShape(10.dp),
color = ClawTheme.colors.surfacePressed.copy(alpha = 0.72f),
border = BorderStroke(1.dp, ClawTheme.colors.border.copy(alpha = 0.6f)),
) {
Column(
modifier = Modifier.fillMaxWidth().padding(horizontal = 10.dp, vertical = 8.dp),
verticalArrangement = Arrangement.spacedBy(4.dp),
) {
Row(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Surface(
onClick = onToggle,
enabled = !playbackBlocked && !loading,
shape = CircleShape,
color = ClawTheme.colors.primary,
contentColor = ClawTheme.colors.primaryText,
) {
Box(modifier = Modifier.size(34.dp), contentAlignment = Alignment.Center) {
if (loading) {
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
} else {
Icon(
imageVector = if (isPlaying) Icons.Default.Pause else Icons.Default.PlayArrow,
contentDescription = if (isPlaying) nativeString("Pause audio") else nativeString("Play audio"),
modifier = Modifier.size(19.dp),
)
}
}
}
Icon(Icons.Default.Mic, contentDescription = null, modifier = Modifier.size(16.dp), tint = ClawTheme.colors.textMuted)
Column(modifier = Modifier.weight(1f)) {
Text(
text = content.fileName?.takeIf(String::isNotBlank) ?: nativeString("Voice note"),
style = ClawTheme.type.body,
color = ClawTheme.colors.text,
)
val status = error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null
status?.let { Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted) }
}
}
Slider(
value = positionMs.coerceIn(0L, durationMs.coerceAtLeast(0L)).toFloat(),
onValueChange = onSeek,
valueRange = 0f..durationMs.coerceAtLeast(1L).toFloat(),
enabled = seekEnabled && durationMs > 0L && playerControlsAvailable(error, playbackBlocked),
)
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) {
Text(formatVoiceNoteDuration(positionMs), style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
Text(formatVoiceNoteDuration(durationMs), style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
}
}
}
}
@OptIn(UnstableApi::class)
@Composable
private fun VideoPlayerSurface(
content: ChatMessageContent,
player: ExoPlayer?,
loading: Boolean,
isPlaying: Boolean,
playbackBlocked: Boolean,
error: String?,
onToggle: () -> Unit,
) {
val ratio =
remember(content.width, content.height) {
val width = content.width?.takeIf { it > 0 }
val height = content.height?.takeIf { it > 0 }
if (width != null && height != null) width.toFloat() / height.toFloat() else 16f / 9f
}
Column(verticalArrangement = Arrangement.spacedBy(5.dp)) {
Box(
modifier =
Modifier
.fillMaxWidth()
.aspectRatio(ratio.coerceIn(0.5f, 2.4f))
.background(ClawTheme.colors.surfacePressed, RoundedCornerShape(10.dp)),
contentAlignment = Alignment.Center,
) {
if (player != null) {
AndroidView(
factory = { viewContext ->
PlayerView(viewContext).apply {
useController = false
resizeMode = AspectRatioFrameLayout.RESIZE_MODE_FIT
this.player = player
}
},
update = { it.player = player },
modifier = Modifier.matchParentSize(),
)
} else {
Icon(Icons.Default.Videocam, contentDescription = null, modifier = Modifier.size(36.dp), tint = ClawTheme.colors.textMuted)
}
Box(
modifier = Modifier.matchParentSize().clickable(enabled = !playbackBlocked && !loading, onClick = onToggle),
contentAlignment = Alignment.Center,
) {
if (loading) {
CircularProgressIndicator(modifier = Modifier.size(28.dp), strokeWidth = 2.dp)
} else if (!isPlaying) {
Surface(shape = CircleShape, color = ClawTheme.colors.primary, contentColor = ClawTheme.colors.primaryText) {
Icon(Icons.Default.PlayArrow, contentDescription = nativeString("Play video"), modifier = Modifier.padding(10.dp).size(24.dp))
}
}
}
}
Text(
text = content.fileName?.takeIf(String::isNotBlank) ?: nativeString("Video"),
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
)
(error ?: if (playbackBlocked) nativeString("Paused for voice playback") else null)?.let {
Text(it, style = ClawTheme.type.caption, color = ClawTheme.colors.textMuted)
}
}
}
private fun playerControlsAvailable(
error: String?,
playbackBlocked: Boolean,
): Boolean = error == null && !playbackBlocked
@OptIn(UnstableApi::class)
private suspend fun prepareMediaSource(
context: Context,
loaded: GatewayLoadedMedia,
): PreparedMediaSource? {
return when (loaded) {
is GatewayLoadedMedia.Buffered -> {
val file = writeBufferedMediaFile(context = context, bytes = loaded.bytes) ?: return null
PreparedMediaSource(
uri = file.toURI().toString(),
mimeType = loaded.mimeType,
headers = loaded.headers,
client = loaded.client,
tempFile = file,
)
}
is GatewayLoadedMedia.Streaming ->
PreparedMediaSource(
uri = loaded.url,
mimeType = loaded.mimeType,
headers = loaded.headers,
client = loaded.client,
tempFile = null,
)
}
}
private suspend fun writeBufferedMediaFile(
context: Context,
bytes: ByteArray,
): File? {
var created: File? = null
return try {
withContext(Dispatchers.IO) {
File.createTempFile("chat-media-", ".media", context.cacheDir).also { file ->
created = file
file.writeBytes(bytes)
}
}
} catch (error: CancellationException) {
withContext(NonCancellable + Dispatchers.IO) { created?.delete() }
throw error
} catch (_: Throwable) {
withContext(NonCancellable + Dispatchers.IO) { created?.delete() }
null
}
}
@OptIn(UnstableApi::class)
private fun buildMediaPlayer(
context: Context,
source: PreparedMediaSource,
): ExoPlayer {
val httpFactory = OkHttpDataSource.Factory(source.client).setDefaultRequestProperties(source.headers)
val dataSourceFactory = DefaultDataSource.Factory(context, httpFactory)
val player =
ExoPlayer
.Builder(context)
.setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory))
.build()
player.setAudioAttributes(
AudioAttributes
.Builder()
.setUsage(C.USAGE_MEDIA)
.setContentType(C.AUDIO_CONTENT_TYPE_UNKNOWN)
.build(),
false,
)
player.setMediaItem(
MediaItem
.Builder()
.setUri(source.uri)
.apply { source.mimeType?.let(::setMimeType) }
.build(),
)
player.prepare()
return player
}
private data class PreparedMediaSource(
val uri: String,
val mimeType: String?,
val headers: Map<String, String>,
val client: okhttp3.OkHttpClient,
val tempFile: File?,
)
internal fun ChatMessageContent.isVideoAttachment(): Boolean = type == "video" || mimeType?.startsWith("video/") == true
internal fun ChatMessageContent.hasPlayableMediaArtifact(): Boolean = !artifactId.isNullOrBlank()
@Composable
internal fun ChatMediaAttachmentLabel(content: ChatMessageContent) {
val isAudio = content.isAudioAttachment()
Row(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Icon(
imageVector = if (isAudio) Icons.Default.Mic else Icons.Default.Videocam,
contentDescription = null,
modifier = Modifier.size(16.dp),
tint = ClawTheme.colors.textMuted,
)
Text(
text =
content.fileName?.takeIf(String::isNotBlank)
?: if (isAudio) nativeString("Voice note") else nativeString("Video"),
style = ClawTheme.type.body,
color = ClawTheme.colors.text,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.weight(1f),
)
content.durationMs?.let { duration ->
Text(
text = formatVoiceNoteDuration(duration),
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
)
}
}
}
@@ -11,6 +11,8 @@ import ai.openclaw.app.chat.OUTBOX_BRANCH_CHANGED_ERROR
import ai.openclaw.app.chat.chatOutboxDisplayError
import ai.openclaw.app.chat.normalizeVisibleChatMessageRole
import ai.openclaw.app.gateway.GatewayLoadedImage
import ai.openclaw.app.gateway.GatewayLoadedMedia
import ai.openclaw.app.gateway.GatewayMediaKind
import ai.openclaw.app.i18n.nativeString
import ai.openclaw.app.i18n.nativeStringResource
import ai.openclaw.app.tools.ToolDisplayRegistry
@@ -107,6 +109,8 @@ internal fun ChatMessageBubble(
onToggleListen: ((String, String) -> Unit)? = null,
imageResolverReady: Boolean = false,
loadImageArtifact: suspend (String) -> GatewayLoadedImage? = { null },
inlineMediaPlaybackBlocked: Boolean = false,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia? = { _, _ -> null },
) {
val role = normalizeVisibleChatMessageRole(message.role) ?: return
val style = bubbleStyle(role)
@@ -117,7 +121,7 @@ internal fun ChatMessageBubble(
when (part.type) {
"text" -> !part.text.isNullOrBlank()
"image" -> !part.base64.isNullOrBlank() || !part.artifactId.isNullOrBlank()
else -> part.isAudioAttachment()
else -> part.isAudioAttachment() || part.isVideoAttachment()
}
}
@@ -148,6 +152,8 @@ internal fun ChatMessageBubble(
textColor = mobileText,
imageResolverReady = imageResolverReady,
loadImageArtifact = loadImageArtifact,
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
loadMediaArtifact = loadMediaArtifact,
)
ChatMessageLinkPreview(messageId = message.id, role = role, content = displayableContent)
messageSpeech?.let { speech ->
@@ -235,6 +241,8 @@ private fun ChatMessageBody(
textColor: Color,
imageResolverReady: Boolean,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
inlineMediaPlaybackBlocked: Boolean,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
) {
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
for (part in content) {
@@ -243,7 +251,19 @@ private fun ChatMessageBody(
val text = part.text ?: continue
ChatMarkdown(text = text, textColor = textColor)
}
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
part.isAudioAttachment() && part.hasPlayableMediaArtifact() ->
ChatAudioPlayerCard(
content = part,
playbackBlocked = inlineMediaPlaybackBlocked,
loadMedia = loadMediaArtifact,
)
part.isVideoAttachment() && part.hasPlayableMediaArtifact() ->
ChatVideoPlayerCard(
content = part,
playbackBlocked = inlineMediaPlaybackBlocked,
loadMedia = loadMediaArtifact,
)
part.isAudioAttachment() || part.isVideoAttachment() -> ChatMediaAttachmentLabel(content = part)
part.type == "image" && !part.base64.isNullOrBlank() ->
ChatBase64Image(base64 = part.base64, mimeType = part.mimeType)
part.type == "image" && !part.artifactId.isNullOrBlank() ->
@@ -33,6 +33,8 @@ import ai.openclaw.app.chat.resolveChatComposerOwner
import ai.openclaw.app.chat.resolveGatewayDefaultAgentId
import ai.openclaw.app.currentAppLanguage
import ai.openclaw.app.gateway.GatewayLoadedImage
import ai.openclaw.app.gateway.GatewayLoadedMedia
import ai.openclaw.app.gateway.GatewayMediaKind
import ai.openclaw.app.i18n.NativeText
import ai.openclaw.app.i18n.joinedNativeText
import ai.openclaw.app.i18n.nativeString
@@ -296,6 +298,7 @@ fun ChatScreen(
val micCooldown by viewModel.micCooldown.collectAsState()
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
val talkModeListening by viewModel.talkModeListening.collectAsState()
val inlineMediaPlaybackBlocked = messageSpeechState != null || talkModeEnabled || talkModeListening
val thinkingSupported =
chatThinkingSupported(
selection = thinkingLevelSelection,
@@ -743,8 +746,10 @@ fun ChatScreen(
},
speechState = messageSpeechState,
onToggleListen = viewModel::toggleChatMessageSpeech,
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
resolveInlineWidgetResource = viewModel::resolveInlineWidgetResource,
loadImageArtifact = viewModel::loadChatImageArtifact,
loadMediaArtifact = viewModel::loadChatMediaArtifact,
modifier = Modifier.weight(1f),
)
@@ -1263,8 +1268,10 @@ private fun ChatMessageList(
onForkMessage: (String) -> Unit,
speechState: MessageSpeechState?,
onToggleListen: (String, String) -> Unit,
inlineMediaPlaybackBlocked: Boolean,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
modifier: Modifier = Modifier,
) {
val baseTimeline =
@@ -1338,9 +1345,11 @@ private fun ChatMessageList(
onForkMessage = onForkMessage,
speechState = speechState,
onToggleListen = onToggleListen,
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
inlineWidgetResolverReady = healthOk,
resolveInlineWidgetResource = resolveInlineWidgetResource,
loadImageArtifact = loadImageArtifact,
loadMediaArtifact = loadMediaArtifact,
)
is ChatTimelineItem.OutboxCommand ->
ChatOutboxBubble(
@@ -1382,9 +1391,11 @@ private fun ChatMessageList(
onForkMessage = onForkMessage,
speechState = null,
onToggleListen = onToggleListen,
inlineMediaPlaybackBlocked = inlineMediaPlaybackBlocked,
inlineWidgetResolverReady = healthOk,
resolveInlineWidgetResource = resolveInlineWidgetResource,
loadImageArtifact = loadImageArtifact,
loadMediaArtifact = loadMediaArtifact,
)
ChatTimelineItem.Thinking -> {
val run = workingRun
@@ -1632,9 +1643,11 @@ private fun ChatBubble(
onForkMessage: (String) -> Unit,
speechState: MessageSpeechState?,
onToggleListen: (String, String) -> Unit,
inlineMediaPlaybackBlocked: Boolean,
inlineWidgetResolverReady: Boolean,
resolveInlineWidgetResource: suspend (String, ChatWidgetResource?) -> ChatWidgetResource?,
loadImageArtifact: suspend (String) -> GatewayLoadedImage?,
loadMediaArtifact: suspend (String, GatewayMediaKind) -> GatewayLoadedMedia?,
) {
val normalizedRole = role.trim().lowercase(Locale.US)
val isUser = normalizedRole == "user"
@@ -1650,7 +1663,7 @@ private fun ChatBubble(
visible
}
"canvas" -> normalizedRole == "assistant" && part.widget != null
else -> part.isAudioAttachment()
else -> part.isAudioAttachment() || part.isVideoAttachment()
}
}
val omittedImageCount = (visibleImageCount - 4).coerceAtLeast(0)
@@ -1716,7 +1729,19 @@ private fun ChatBubble(
when {
part.type == "text" && !collapsibleUserText -> ChatText(text = part.text.orEmpty(), textColor = ClawTheme.colors.text, isStreaming = live)
part.type == "text" -> Unit
part.isAudioAttachment() -> VoiceNoteMessageRow(durationMs = part.durationMs)
part.isAudioAttachment() && part.hasPlayableMediaArtifact() ->
ChatAudioPlayerCard(
content = part,
playbackBlocked = inlineMediaPlaybackBlocked,
loadMedia = loadMediaArtifact,
)
part.isVideoAttachment() && part.hasPlayableMediaArtifact() ->
ChatVideoPlayerCard(
content = part,
playbackBlocked = inlineMediaPlaybackBlocked,
loadMedia = loadMediaArtifact,
)
part.isAudioAttachment() || part.isVideoAttachment() -> ChatMediaAttachmentLabel(content = part)
part.type == "image" ->
if (!part.base64.isNullOrBlank()) {
ChatBase64Image(
@@ -36,6 +36,7 @@ class AndroidLicenseNoticesTest {
assertEquals(
listOf(
"AndroidX Media3",
"AndroidX Room",
"AndroidX Wear",
"Bouncy Castle Provider",
@@ -320,6 +320,18 @@ class ChatMessageContentParsingTest {
)
}
@Test
fun derivesArtifactIdentityForManagedAudioAndVideoBlocks() {
val attachmentId = "22222222-2222-4222-8222-222222222222"
val url = "/api/chat/media/outgoing/main/$attachmentId/full"
assertEquals("artifact_managed_media_$attachmentId", managedMediaArtifactId(url))
assertEquals(
"artifact_managed_media_$attachmentId",
parseChatMessageContent(Json.parseToJsonElement("""{"type":"video","mimeType":"video/mp4","url":"$url"}"""))?.artifactId,
)
}
@Test
fun dropsOversizedInlineImageContentBeforeRendering() {
val oversized = "A".repeat(CHAT_IMAGE_MAX_BASE64_CHARS + 1)
@@ -335,14 +347,27 @@ class ChatMessageContentParsingTest {
}
@Test
fun parsesDirectAndAttachmentAudioBlocks() {
fun dropsInlineAudioAndVideoContentThatRequiresManagedArtifacts() {
val audio = Json.parseToJsonElement("""{"type":"audio","mimeType":"audio/mpeg","content":"audio-bytes"}""")
val video = Json.parseToJsonElement("""{"type":"video","mimeType":"video/mp4","content":"video-bytes"}""")
assertEquals(ChatMessageContent(type = "audio", mimeType = "audio/mpeg"), parseChatMessageContent(audio))
assertEquals(ChatMessageContent(type = "video", mimeType = "video/mp4"), parseChatMessageContent(video))
}
@Test
fun parsesDirectAndAttachmentAudioVideoBlocks() {
val direct =
Json.parseToJsonElement(
"""{"type":"audio","mimeType":"audio/mp4","fileName":"voice.m4a"}""",
)
val attachment =
Json.parseToJsonElement(
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mpeg","label":"reply.mp3"}}""",
"""{"type":"attachment","attachment":{"kind":"audio","mimeType":"audio/mpeg","label":"reply.mp3","artifactId":"artifact_managed_media_33333333-3333-4333-8333-333333333333","url":"/api/chat/media/outgoing/main/33333333-3333-4333-8333-333333333333/full","sizeBytes":4096,"durationMs":2100}}""",
)
val video =
Json.parseToJsonElement(
"""{"type":"attachment","attachment":{"kind":"video","mimeType":"video/mp4","fileName":"demo.mp4","artifactId":"artifact_managed_media_44444444-4444-4444-8444-444444444444","url":"/api/chat/media/outgoing/main/44444444-4444-4444-8444-444444444444/full","sizeBytes":8192,"durationMs":5300,"width":1920,"height":1080}}""",
)
assertEquals(
@@ -350,9 +375,31 @@ class ChatMessageContentParsingTest {
parseChatMessageContent(direct),
)
assertEquals(
ChatMessageContent(type = "audio", mimeType = "audio/mpeg", fileName = "reply.mp3"),
ChatMessageContent(
type = "audio",
mimeType = "audio/mpeg",
fileName = "reply.mp3",
artifactId = "artifact_managed_media_33333333-3333-4333-8333-333333333333",
url = "/api/chat/media/outgoing/main/33333333-3333-4333-8333-333333333333/full",
sizeBytes = 4096,
durationMs = 2100,
),
parseChatMessageContent(attachment),
)
assertEquals(
ChatMessageContent(
type = "video",
mimeType = "video/mp4",
fileName = "demo.mp4",
artifactId = "artifact_managed_media_44444444-4444-4444-8444-444444444444",
url = "/api/chat/media/outgoing/main/44444444-4444-4444-8444-444444444444/full",
width = 1920,
height = 1080,
sizeBytes = 8192,
durationMs = 5300,
),
parseChatMessageContent(video),
)
}
@Test
@@ -76,6 +76,44 @@ class RoomChatTranscriptCacheTest {
assertEquals(listOf("run-1:user", null, null), loaded.map { it.idempotencyKey })
}
@Test
fun transcriptRoundTripKeepsManagedAudioAndVideoMetadata() =
runTest {
val store = cache()
val audio =
ChatMessageContent(
type = "audio",
mimeType = "audio/mpeg",
fileName = "reply.mp3",
artifactId = "artifact_managed_media_33333333-3333-4333-8333-333333333333",
durationMs = 2_100,
)
val video =
ChatMessageContent(
type = "video",
mimeType = "video/mp4",
fileName = "demo.mp4",
artifactId = "artifact_managed_media_44444444-4444-4444-8444-444444444444",
durationMs = 5_300,
width = 1920,
height = 1080,
)
store.saveTranscript(
gatewayId = "gateway-a",
agentId = "main",
sessionKey = "main",
messages =
listOf(
ChatMessage(id = "audio", role = "assistant", content = listOf(audio), timestampMs = 10),
ChatMessage(id = "video", role = "assistant", content = listOf(video), timestampMs = 11),
),
)
val loaded = store.loadTranscript("gateway-a", "main", "main")
assertEquals(listOf(audio, video), loaded.map { it.content.single() })
}
@Test
fun legacyStringArrayTranscriptRowsRemainReadable() =
runTest {
@@ -63,7 +63,7 @@ private class NoopDeviceAuthStore : DeviceAuthTokenStore {
@Config(sdk = [34])
class GatewaySessionCustomHeadersTest {
@Test
fun managedImageDownload_usesArtifactTicketWithoutGatewayBearer() =
fun managedMediaDownload_usesArtifactTicketWithoutGatewayBearer() =
runBlocking {
val app = RuntimeEnvironment.getApplication()
val json = Json { ignoreUnknownKeys = true }
@@ -73,6 +73,9 @@ class GatewaySessionCustomHeadersTest {
val attachmentId = "11111111-1111-4111-8111-111111111111"
val artifactId = "artifact_managed_image_$attachmentId"
val imagePath = "/api/chat/media/outgoing/main/$attachmentId/full?mediaTicket=ticket"
val videoAttachmentId = "22222222-2222-4222-8222-222222222222"
val videoArtifactId = "artifact_managed_media_$videoAttachmentId"
val videoPath = "/api/chat/media/outgoing/main/$videoAttachmentId/full?mediaTicket=video-ticket"
val server =
MockWebServer().apply {
dispatcher =
@@ -106,9 +109,20 @@ class GatewaySessionCustomHeadersTest {
"""{"type":"res","id":"$id","ok":true,"payload":{"snapshot":{"sessionDefaults":{"mainSessionKey":"main"}}}}""",
)
"artifacts.download" ->
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
)
if (frame["params"]
?.jsonObject
?.get("artifactId")
?.jsonPrimitive
?.content == videoArtifactId
) {
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"artifact":{"id":"$videoArtifactId","type":"video","mimeType":"video/mp4","download":{"mode":"url"}},"url":"$videoPath"}}""",
)
} else {
webSocket.send(
"""{"type":"res","id":"$id","ok":true,"payload":{"url":"$imagePath"}}""",
)
}
}
}
},
@@ -164,6 +178,12 @@ class GatewaySessionCustomHeadersTest {
val request = withTimeout(TEST_TIMEOUT_MS) { imageRequest.await() }
assertNull(request.getHeader("Authorization"))
assertEquals("image/*", request.getHeader("Accept"))
val streamed =
session.loadMediaArtifact(stableId, "main", "main", videoArtifactId, GatewayMediaKind.Video) as GatewayLoadedMedia.Streaming
assertEquals("http://127.0.0.1:${server.port}$videoPath", streamed.url)
assertEquals("video/*", streamed.headers["Accept"])
assertEquals("video/mp4", streamed.mimeType)
} finally {
session.disconnectAndJoin()
scope.cancel()
@@ -0,0 +1,124 @@
package ai.openclaw.app.ui.chat
import ai.openclaw.app.chat.ChatMessage
import ai.openclaw.app.chat.ChatMessageContent
import androidx.compose.ui.test.assertCountEquals
import androidx.compose.ui.test.assertIsDisplayed
import androidx.compose.ui.test.junit4.v2.createComposeRule
import androidx.compose.ui.test.onAllNodesWithContentDescription
import androidx.compose.ui.test.onNodeWithText
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertSame
import org.junit.Assert.assertTrue
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
@RunWith(RobolectricTestRunner::class)
class ChatMediaPlayerTest {
private class FakePlayer(
var positionMs: Long = 0L,
) {
var paused = false
var released = false
var playCount = 0
fun pause() {
paused = true
}
fun play() {
paused = false
playCount += 1
}
fun release() {
released = true
}
}
@get:Rule
val composeRule = createComposeRule()
@Test
fun claimHandoffReleasesPreviousPlaybackInstance() {
val first = FakePlayer()
val second = FakePlayer()
val claims = ChatMediaPlaybackClaims<FakePlayer>(FakePlayer::pause, FakePlayer::release)
claims.claim(first)
claims.claim(second)
assertTrue(first.released)
assertFalse(second.released)
assertSame(second, claims.active)
}
@Test
fun pauseThenPlayResumesPositionWithoutRedownload() {
var downloadCount = 0
val player = FakePlayer(positionMs = 4_200L).also { downloadCount += 1 }
val claims = ChatMediaPlaybackClaims<FakePlayer>(FakePlayer::pause, FakePlayer::release)
claims.claim(player)
claims.pauseIf { it === player }
claims.claim(player)
player.play()
assertEquals(1, downloadCount)
assertEquals(4_200L, player.positionMs)
assertEquals(1, player.playCount)
assertFalse(player.paused)
assertFalse(player.released)
assertSame(player, claims.active)
}
@Test
fun legacyMediaPartsRenderLabelsWithoutPlayControlsOrClaims() {
val audio =
ChatMessageContent(
type = "audio",
mimeType = "audio/mpeg",
fileName = "legacy.mp3",
durationMs = 4_000,
)
val video =
ChatMessageContent(
type = "video",
mimeType = "video/mp4",
fileName = "legacy.mp4",
durationMs = 9_000,
)
var loadCount = 0
composeRule.setContent {
ChatMessageBubble(
message =
ChatMessage(
id = "legacy-media",
role = "assistant",
content = listOf(audio, video),
timestampMs = 1,
),
loadMediaArtifact = { _, _ ->
loadCount += 1
null
},
)
}
composeRule.onNodeWithText("legacy.mp3").assertIsDisplayed()
composeRule.onNodeWithText("legacy.mp4").assertIsDisplayed()
composeRule.onNodeWithText("0:04").assertIsDisplayed()
composeRule.onNodeWithText("0:09").assertIsDisplayed()
composeRule.onAllNodesWithContentDescription("Play audio").assertCountEquals(0)
composeRule.onAllNodesWithContentDescription("Play video").assertCountEquals(0)
composeRule.runOnIdle {
assertEquals(0, loadCount)
assertFalse(audio.hasPlayableMediaArtifact())
assertFalse(video.hasPlayableMediaArtifact())
}
}
}
+4
View File
@@ -29,6 +29,7 @@ ksp = "2.3.10"
ktlint-gradle = "14.2.0"
kotlin = "2.4.10"
material = "1.14.0"
media3 = "1.10.1"
okhttp = "5.4.0"
play-services-wearable = "20.0.1"
barcode-scanning = "17.3.0"
@@ -87,6 +88,9 @@ kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutine
kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" }
kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization-json" }
material = { module = "com.google.android.material:material", version.ref = "material" }
media3-datasource-okhttp = { module = "androidx.media3:media3-datasource-okhttp", version.ref = "media3" }
media3-exoplayer = { module = "androidx.media3:media3-exoplayer", version.ref = "media3" }
media3-ui = { module = "androidx.media3:media3-ui", version.ref = "media3" }
mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp" }
okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
play-services-wearable = { module = "com.google.android.gms:play-services-wearable", version.ref = "play-services-wearable" }