fix(android): preserve device work across lifecycle changes (#116593)

* fix(android): use configured SDK for screenshot AVDs

* fix(android): keep chat jump control clear of messages

* fix(wear): ignore stale control completions

* fix(android): advertise node permission changes

* fix(android): select play flavor for benchmarks

* test(android): decouple cron benchmark from copy

* fix(android): install play variant in perf scripts

* fix(wear): clear gateway control busy state

* fix(android): surface voice e2e connection errors

* test(android): cover common live node commands

* fix(android): retain share parsing across recreation

* fix(android): retain permission requests across recreation

* fix(wear): bind realtime audio to talk attempts

* fix(android): track active permission host

* fix(wear): own gateway control busy state

* fix(wear): admit realtime channels by attempt

* fix(android): refresh permissions on top resume

* fix(wear): serialize reconnect retirement with writes

* chore(i18n): refresh native source inventory

* fix(wear): retain pending audio through start RPC

* fix(wear): negotiate realtime channel compatibility

* test(wear): assert reconnect outcome without owner churn

* fix(android): move settings prompt with active host

* fix(wear): reject stale relay starts

* chore(changelog): leave notes to release generation
This commit is contained in:
Vincent Koc
2026-07-31 09:49:11 +08:00
committed by GitHub
parent 3b864a18c4
commit 88876f780b
48 changed files with 4838 additions and 750 deletions
File diff suppressed because it is too large Load Diff
+3 -1
View File
@@ -367,7 +367,9 @@ What it does:
- Reads `node.describe` command list from the selected Android node.
- Invokes advertised non-interactive commands.
- Skips `screen.record` in this suite (Android requires interactive per-invocation screen-capture consent).
- Skips `screen.record` and `talk.ptt.*` in this suite because they require
interactive capture. Use `apps/android/scripts/voice-e2e.sh` for microphone
and voice-path proof.
- Asserts command contracts (success or expected deterministic error for safe-invalid calls like `sms.send` and `notifications.actions`).
Common failure quick-fixes:
@@ -10,6 +10,7 @@ import android.util.Log
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.TimeoutCancellationException
import kotlinx.coroutines.cancel
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
@@ -164,10 +165,22 @@ class VoiceE2eService : Service() {
runtime: NodeRuntime,
timeoutMs: Long,
) {
withTimeout(timeoutMs) {
while (!runtime.isConnected.value) {
delay(100L)
try {
withTimeout(timeoutMs) {
while (!runtime.isConnected.value) {
voiceE2eTerminalGatewayFailure(runtime.gatewayConnectionProblem.value)?.let { error(it) }
delay(100L)
}
}
} catch (err: TimeoutCancellationException) {
throw IllegalStateException(
voiceE2eGatewayTimeoutMessage(
timeoutMs = timeoutMs,
statusText = runtime.statusText.value,
problem = runtime.gatewayConnectionProblem.value,
),
err,
)
}
}
@@ -198,3 +211,32 @@ private fun Intent.getDecodedStringExtra(name: String): String? {
}
return getStringExtra(name)
}
internal fun voiceE2eTerminalGatewayFailure(problem: GatewayConnectionProblem?): String? =
problem
?.takeIf { it.pauseReconnect && !it.canAutoRetry }
?.message
?.trim()
?.takeIf { it.isNotEmpty() }
internal fun voiceE2eGatewayTimeoutMessage(
timeoutMs: Long,
statusText: String,
problem: GatewayConnectionProblem?,
): String {
val detail =
problem
?.message
?.trim()
?.takeIf { it.isNotEmpty() }
?: statusText.trim().takeIf { it.isNotEmpty() }
return buildString {
append("Gateway connection timed out after ")
append(timeoutMs)
append(" ms")
if (detail != null) {
append(": ")
append(detail)
}
}
}
@@ -69,6 +69,13 @@
android:host="*"
android:path="/openclaw/wear/v1/realtime/audio" />
</intent-filter>
<intent-filter>
<action android:name="com.google.android.gms.wearable.CHANNEL_EVENT" />
<data
android:scheme="wear"
android:host="*"
android:pathPrefix="/openclaw/wear/v1/realtime/audio/" />
</intent-filter>
</service>
<service
android:name="androidx.appcompat.app.AppLocalesMetadataHolderService"
@@ -35,9 +35,6 @@ import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
/**
@@ -45,13 +42,12 @@ import kotlinx.coroutines.withContext
*/
class MainActivity : AppCompatActivity() {
private val viewModel: MainViewModel by viewModels()
private lateinit var permissionRequester: PermissionRequester
private val permissionRequester: PermissionRequester
get() = (application as NodeApp).permissionRequester
private var initializedViewModel: MainViewModel? = null
private var didStartViewModelCollectors = false
private var foreground = false
private val pendingIntentRouter = MainActivityPendingIntentRouter()
private val shareLaunchMutex = Mutex()
private val shareLaunchSlots = Semaphore(MAX_PENDING_CHAT_SHARES)
private val runtimeUiStarter = MainActivityRuntimeUiStarter()
private var screenshotScene: AndroidScreenshotScene? = null
@@ -59,7 +55,7 @@ class MainActivity : AppCompatActivity() {
super.onCreate(savedInstanceState)
pendingIntentRouter.setInitialIntent(intent)
WindowCompat.setDecorFitsSystemWindows(window, false)
permissionRequester = PermissionRequester(this)
permissionRequester.attach(this)
if (BuildConfig.DEBUG) {
screenshotScene = parseAndroidScreenshotModeIntent(intent)
if (screenshotScene != null) hideScreenshotModeStatusBar()
@@ -108,7 +104,20 @@ class MainActivity : AppCompatActivity() {
initializedViewModel?.setForeground(true)
}
override fun onTopResumedActivityChanged(isTopResumedActivity: Boolean) {
super.onTopResumedActivityChanged(isTopResumedActivity)
// minSdk 31 guarantees this callback and lets multi-resume select the actually interactive task.
updateTopResumedPermissionHost(
isTopResumedActivity = isTopResumedActivity,
activate = { permissionRequester.activate(this) },
deactivate = { permissionRequester.deactivate(this) },
refreshPermissionSurface = { initializedViewModel?.refreshNodePermissionSurface() },
)
}
override fun onStop() {
// Top-resumed ownership normally clears first; this also covers abnormal lifecycle ordering.
permissionRequester.deactivate(this)
foreground = false
if (shouldNotifyRuntimeBackgrounded(isChangingConfigurations)) {
initializedViewModel?.setForeground(false)
@@ -116,6 +125,11 @@ class MainActivity : AppCompatActivity() {
super.onStop()
}
override fun onDestroy() {
permissionRequester.detach(this)
super.onDestroy()
}
override fun onNewIntent(intent: android.content.Intent) {
super.onNewIntent(intent)
setIntent(intent)
@@ -123,9 +137,7 @@ class MainActivity : AppCompatActivity() {
pendingIntentRouter.onNewIntent(intent) { routedIntent ->
initializedViewModel?.let { handleLaunchIntent(viewModel = it, intent = routedIntent) }
}
if (!accepted) {
Toast.makeText(this, nativeString("Too many shares are waiting to be added."), Toast.LENGTH_SHORT).show()
}
if (!accepted) return
}
override fun onRequestPermissionsResult(
@@ -135,9 +147,8 @@ class MainActivity : AppCompatActivity() {
) {
// AppCompatActivity marks this callback @CallSuper; it preserves Fragment and ActivityResult dispatch.
super.onRequestPermissionsResult(requestCode, permissions, grantResults)
if (::permissionRequester.isInitialized) {
permissionRequester.onRequestPermissionsResult(requestCode, permissions, grantResults)
}
permissionRequester.onRequestPermissionsResult(requestCode, permissions, grantResults)
initializedViewModel?.refreshNodePermissionSurface()
}
/**
@@ -154,6 +165,7 @@ class MainActivity : AppCompatActivity() {
pendingIntentRouter.activate { initialIntent ->
handleLaunchIntent(viewModel = readyViewModel, intent = initialIntent)
}
readyViewModel.reportShareLaunchOverflow(pendingIntentRouter.takeShareOverflowCount())
}
/**
@@ -192,6 +204,22 @@ class MainActivity : AppCompatActivity() {
}
}
}
lifecycleScope.launch {
repeatOnLifecycle(Lifecycle.State.STARTED) {
readyViewModel.shareLaunchOverflowRevision.collect { revision ->
if (revision == 0L) return@collect
repeat(readyViewModel.takeShareLaunchOverflowCount()) {
Toast
.makeText(
this@MainActivity,
nativeString("Too many shares are waiting to be added."),
Toast.LENGTH_SHORT,
).show()
}
}
}
}
}
/**
@@ -202,23 +230,7 @@ class MainActivity : AppCompatActivity() {
intent: Intent?,
) {
if (intent?.isShareLaunchIntent() == true) {
if (!shareLaunchSlots.tryAcquire()) {
Toast.makeText(this, nativeString("Too many shares are waiting to be added."), Toast.LENGTH_SHORT).show()
return
}
val owner = viewModel.captureChatShareOwner()
lifecycleScope
.launch {
shareLaunchMutex.withLock {
val request =
withContext(Dispatchers.IO) {
parseShareLaunchIntent(intent, contentResolver::getType)
} ?: return@withLock
if (!viewModel.handleShareLaunch(request, owner)) {
Toast.makeText(this@MainActivity, nativeString("Too many shares are waiting to be added."), Toast.LENGTH_SHORT).show()
}
}
}.invokeOnCompletion { shareLaunchSlots.release() }
viewModel.handleShareLaunchIntent(intent)
return
}
parseHomeDestinationIntent(intent)?.let { destination ->
@@ -242,6 +254,7 @@ internal class MainActivityPendingIntentRouter {
private var sequence = 0L
private val pendingShareIntents = ArrayDeque<PendingLaunchIntent>()
private var pendingNonShareIntent: PendingLaunchIntent? = null
private var shareOverflowCount = 0
fun setInitialIntent(intent: Intent?) {
if (!activated && intent != null) store(intent = intent, initial = true)
@@ -275,6 +288,11 @@ internal class MainActivityPendingIntentRouter {
return true
}
fun takeShareOverflowCount(): Int =
shareOverflowCount.also {
shareOverflowCount = 0
}
private fun store(
intent: Intent,
initial: Boolean,
@@ -284,7 +302,10 @@ internal class MainActivityPendingIntentRouter {
pendingNonShareIntent = pending
return true
}
if (pendingShareIntents.size >= MAX_PENDING_CHAT_SHARES) return false
if (pendingShareIntents.size >= MAX_PENDING_CHAT_SHARES) {
shareOverflowCount += 1
return false
}
pendingShareIntents.addLast(pending)
return true
}
@@ -305,6 +326,20 @@ internal class MainActivityInitialIntentGate {
internal fun shouldNotifyRuntimeBackgrounded(isChangingConfigurations: Boolean): Boolean = !isChangingConfigurations
internal fun updateTopResumedPermissionHost(
isTopResumedActivity: Boolean,
activate: () -> Unit,
deactivate: () -> Unit,
refreshPermissionSurface: () -> Unit,
) {
if (isTopResumedActivity) {
activate()
refreshPermissionSurface()
} else {
deactivate()
}
}
/** Preserves one-shot runtime UI startup while allowing screenshot fixtures to skip side effects. */
internal class MainActivityRuntimeUiStarter {
private var completed = false
@@ -47,6 +47,8 @@ import ai.openclaw.app.voice.VoiceConversationEntry
import ai.openclaw.app.voice.VoiceWakePreferences
import android.Manifest
import android.app.Application
import android.content.Intent
import android.net.Uri
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.SavedStateHandle
@@ -65,7 +67,9 @@ import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import java.util.UUID
import java.util.concurrent.atomic.AtomicLong
@@ -276,17 +280,33 @@ class MainViewModel private constructor(
app: Application,
private val prefs: SecurePrefs,
savedStateHandle: SavedStateHandle,
private val resolveShareMimeType: (Uri) -> String?,
shareLaunchCapacity: Int,
) : AndroidViewModel(app) {
constructor(
app: Application,
savedStateHandle: SavedStateHandle,
) : this(app, (app as NodeApp).prefs, savedStateHandle)
) : this(
app = app,
prefs = (app as NodeApp).prefs,
savedStateHandle = savedStateHandle,
resolveShareMimeType = app.contentResolver::getType,
shareLaunchCapacity = MAX_PENDING_CHAT_SHARES,
)
internal constructor(
app: NodeApp,
prefs: SecurePrefs,
savedStateHandle: SavedStateHandle,
) : this(app as Application, prefs, savedStateHandle)
resolveShareMimeType: (Uri) -> String? = app.contentResolver::getType,
shareLaunchCapacity: Int = MAX_PENDING_CHAT_SHARES,
) : this(
app = app as Application,
prefs = prefs,
savedStateHandle = savedStateHandle,
resolveShareMimeType = resolveShareMimeType,
shareLaunchCapacity = shareLaunchCapacity,
)
private val nodeApp = app as NodeApp
private val runtimeRef = MutableStateFlow<NodeRuntime?>(null)
@@ -296,6 +316,10 @@ class MainViewModel private constructor(
// Multiple MainActivity instances can overlap across sender tasks; the process owns one queue.
private val chatShareDraftSeq = nodeApp.chatShareDraftSeq
private val chatShareDraftQueue = nodeApp.chatShareDraftQueue
private val shareLaunchMutex = Mutex()
private val shareLaunchSlots = Semaphore(shareLaunchCapacity)
private val shareLaunchOverflowLock = Any()
private var pendingShareLaunchOverflowCount = 0
// One bounded heap-only slot follows the ViewModel across Activity recreation.
// Detail disposal clears it; process death drops it with the ViewModel.
@@ -352,6 +376,8 @@ class MainViewModel private constructor(
val chatShareDraft: StateFlow<ChatShareDraft?> = chatShareDraftQueue.head
internal val chatShareDrafts: StateFlow<List<ChatShareDraft>> = chatShareDraftQueue.queued
internal val chatShareDraftOwnerRevision: StateFlow<Long> = chatShareDraftQueue.ownerRevision
private val shareLaunchOverflowRevisionMutable = MutableStateFlow(0L)
internal val shareLaunchOverflowRevision: StateFlow<Long> = shareLaunchOverflowRevisionMutable.asStateFlow()
private val pendingAssistantAutoSendMutable = MutableStateFlow<PendingAssistantAutoSend?>(null)
internal val pendingAssistantAutoSend: StateFlow<PendingAssistantAutoSend?> = pendingAssistantAutoSendMutable
private val _assistantAutoSendInFlight = MutableStateFlow(false)
@@ -706,6 +732,10 @@ class MainViewModel private constructor(
runtimeRef.value?.setForeground(value)
}
fun refreshNodePermissionSurface() {
runtimeRef.value?.refreshNodePermissionSurface()
}
fun setDisplayName(value: String) {
prefs.setDisplayName(value)
}
@@ -964,8 +994,49 @@ class MainViewModel private constructor(
setChatDraft(request.prompt?.let { ChatDraft(text = it, placement = ChatDraftPlacement.Replace, owner = owner) })
}
/**
* Owns share admission through queue insertion so Activity recreation cannot cancel accepted work.
*/
internal fun handleShareLaunchIntent(intent: Intent): Boolean {
if (!shareLaunchSlots.tryAcquire()) {
reportShareLaunchOverflow()
return false
}
val retainedIntent = Intent(intent)
val owner = captureChatShareOwner()
viewModelScope.launch(start = CoroutineStart.UNDISPATCHED) {
try {
shareLaunchMutex.withLock {
val request =
withContext(Dispatchers.IO) {
parseShareLaunchIntent(retainedIntent, resolveShareMimeType)
} ?: return@withLock
if (!enqueueShareLaunch(request, owner)) reportShareLaunchOverflow()
}
} finally {
shareLaunchSlots.release()
}
}
return true
}
internal fun reportShareLaunchOverflow(count: Int = 1) {
if (count <= 0) return
synchronized(shareLaunchOverflowLock) {
pendingShareLaunchOverflowCount += count
shareLaunchOverflowRevisionMutable.value += 1
}
}
internal fun takeShareLaunchOverflowCount(): Int =
synchronized(shareLaunchOverflowLock) {
pendingShareLaunchOverflowCount.also {
pendingShareLaunchOverflowCount = 0
}
}
/** Opens shared content as a fresh composer draft; sending still requires an explicit tap. */
internal fun handleShareLaunch(
private fun enqueueShareLaunch(
request: ShareLaunchRequest,
owner: ChatComposerOwner,
): Boolean {
@@ -24,6 +24,7 @@ class NodeApp : Application() {
// System share senders can create overlapping Activity tasks; keep one bounded process queue.
internal val chatShareDraftSeq = AtomicLong()
internal val chatShareDraftQueue = ChatShareDraftQueue()
internal val permissionRequester by lazy { PermissionRequester(this) }
private val runtimeScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val runtimeLock = Any()
@@ -96,6 +96,7 @@ import ai.openclaw.app.node.asObjectOrNull
import ai.openclaw.app.node.asStringOrNull
import ai.openclaw.app.node.invokeErrorFromThrowable
import ai.openclaw.app.node.parseHexColorArgb
import ai.openclaw.app.node.readAndroidPermissionSnapshot
import ai.openclaw.app.protocol.OpenClawCanvasA2UIAction
import ai.openclaw.app.systemagent.SystemAgentChatController
import ai.openclaw.app.systemagent.SystemAgentChatState
@@ -119,6 +120,7 @@ import ai.openclaw.app.wear.WearProxyBridge
import ai.openclaw.app.wear.WearProxyController
import ai.openclaw.app.wear.WearProxyGatewayException
import ai.openclaw.app.wear.WearProxyModel
import ai.openclaw.app.wear.WearRealtimeAttemptOwner
import ai.openclaw.app.wear.WearRealtimeTalkController
import ai.openclaw.app.wear.wearConnectionFailure
import ai.openclaw.wear.shared.WearMessage
@@ -194,6 +196,36 @@ private fun execApprovalResolveFailureMessage(): String = nativeText("Could not
internal typealias GatewayDataRequestOverride =
suspend (stableId: String, method: String, paramsJson: String?) -> String
internal suspend fun startWearRealtimeTalkWhileCurrent(
owner: WearRealtimeAttemptOwner,
isCurrent: suspend (WearRealtimeAttemptOwner) -> Boolean,
start: suspend (onSessionActivated: () -> Unit) -> Boolean,
stop: suspend (WearRealtimeAttemptOwner) -> Unit,
): Boolean {
if (!isCurrent(owner)) return false
var relayStarted = false
var committed = false
try {
val startReturned =
start {
// The controller invokes this synchronously at activation, before a
// canceled caller can lose the successful suspend result.
relayStarted = true
}
if (!startReturned || !isCurrent(owner)) return false
committed = true
return true
} finally {
// Relay creation suspends outside the channel registry. Never leave a late
// session alive when replacement or cancellation wins before commit.
if (relayStarted && !committed) {
withContext(NonCancellable) {
stop(owner)
}
}
}
}
private class ExecApprovalWriteOutcomeUnknown : IllegalStateException("approval resolve response was not authoritative")
private class GatewayApprovalRpcUnavailable : IllegalStateException("Gateway approval RPC catalog is inconsistent")
@@ -907,11 +939,23 @@ class NodeRuntime private constructor(
locationPreciseEnabled = { locationPreciseEnabled.value },
)
private val permissionSnapshot = {
readAndroidPermissionSnapshot(
context = appContext,
smsEnabled = SensitiveFeatureConfig.smsEnabled,
callLogEnabled = SensitiveFeatureConfig.callLogEnabled,
photosEnabled = SensitiveFeatureConfig.photosEnabled,
backgroundLocationEnabled = SensitiveFeatureConfig.backgroundLocationEnabled,
)
}
private val deviceHandler: DeviceHandler =
DeviceHandler(
DeviceHandler.withPermissionSnapshot(
appContext = appContext,
smsEnabled = SensitiveFeatureConfig.smsEnabled,
callLogEnabled = SensitiveFeatureConfig.callLogEnabled,
photosEnabled = SensitiveFeatureConfig.photosEnabled,
permissionSnapshot = permissionSnapshot,
)
private val notificationsHandler: NotificationsHandler =
@@ -985,12 +1029,14 @@ class NodeRuntime private constructor(
SensitiveFeatureConfig.accessibilityControlEnabled && mobileUiHandler.isConnected.value
},
inlineWidgetsAvailable = { WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE) },
permissionSnapshot = permissionSnapshot,
manualTls = { endpoint ->
prefs.gatewayRegistry.entries.value
.firstOrNull { it.stableId == endpoint.stableId }
?.tls ?: manualTls.value
},
)
private var lastNodePermissions = connectionManager.buildPermissions()
private var lastVoiceWakeCapabilityEnabled = isVoiceWakeCapabilityEnabled()
private val invokeDispatcher: InvokeDispatcher =
@@ -1510,8 +1556,8 @@ class NodeRuntime private constructor(
},
connectGateway = { refreshGatewayConnection() },
disconnectGateway = { disconnect() },
startRealtimeTalk = { nodeId, sessionKey, attemptId, language ->
if (startWearRealtimeTalk(nodeId, sessionKey, attemptId, language)) wearRealtimeTalkSnapshot.value else null
startRealtimeTalk = { nodeId, sessionKey, attemptId, language, attemptScopedAudio ->
if (startWearRealtimeTalk(nodeId, sessionKey, attemptId, language, attemptScopedAudio)) wearRealtimeTalkSnapshot.value else null
},
stopRealtimeTalk = { nodeId, attemptId ->
if (stopWearRealtimeTalk(nodeId, attemptId)) wearRealtimeTalkSnapshot.value else null
@@ -2042,6 +2088,8 @@ class NodeRuntime private constructor(
val talkModeConversation: StateFlow<List<VoiceConversationEntry>>
get() = talkMode.conversation
private val wearRealtimeLifecycleMutex = Mutex()
private val wearRealtimeTalkControllerLazy: Lazy<WearRealtimeTalkController> =
lazy {
WearRealtimeTalkController(
@@ -2057,15 +2105,17 @@ class NodeRuntime private constructor(
onError(error.message)
}
},
sendWatchFrame = { nodeId, type, payload ->
sendWatchFrame = { owner, type, payload ->
val app = appContext as? NodeApp ?: error("Wear channel owner is unavailable")
app.wearRealtimeChannels.send(nodeId, type, payload)
app.wearRealtimeChannels.send(owner, type, payload)
},
onSnapshot = { snapshot ->
wearProxyBridge()?.publishTalk(WearRealtimeTalkCodec.encode(snapshot))
},
onForceCloseWatchChannel = { nodeId ->
scope.launch { (appContext as? NodeApp)?.wearRealtimeChannels?.close(nodeId) }
onForceCloseWatchChannel = { owner ->
scope.launch {
(appContext as? NodeApp)?.wearRealtimeChannels?.close(owner)
}
},
)
}
@@ -2081,27 +2131,68 @@ class NodeRuntime private constructor(
sessionKey: String,
attemptId: String,
language: String?,
attemptScopedAudio: Boolean,
): Boolean {
if (talkModeEnabled.value || micEnabled.value || micCooldown.value) return false
val app = appContext as? NodeApp ?: return false
val claim =
app.wearRealtimeChannels.claim(
nodeId = nodeId,
attemptId = attemptId,
attemptScopedAudio = attemptScopedAudio,
) ?: return false
val owner = claim.owner
val resolvedLanguage = talkMode.resolveRealtimeLanguageHint(language)
return wearRealtimeTalkController.start(nodeId, sessionKey, attemptId, resolvedLanguage)
var started = false
return try {
started =
wearRealtimeLifecycleMutex.withLock {
if (talkModeEnabled.value || micEnabled.value || micCooldown.value) {
return@withLock false
}
startWearRealtimeTalkWhileCurrent(
owner = owner,
isCurrent = app.wearRealtimeChannels::isCurrent,
start = { onSessionActivated ->
wearRealtimeTalkController.start(
owner = owner,
sessionKey = sessionKey,
language = resolvedLanguage,
onSessionActivated = onSessionActivated,
)
},
stop = { staleOwner ->
wearRealtimeTalkController.stop(staleOwner)
},
)
}
started
} finally {
if (!started && claim.newlyAcquired) app.wearRealtimeChannels.release(owner)
}
}
internal suspend fun stopWearRealtimeTalk(
nodeId: String? = null,
attemptId: String? = null,
): Boolean {
// The watch closes its channel after receiving the stop response. Closing
// here races the response and makes a normal stop look like link failure.
return wearRealtimeTalkController.stop(nodeId, attemptId)
}
): Boolean =
wearRealtimeLifecycleMutex.withLock {
// The watch closes its channel after receiving the stop response. Closing
// here races the response and makes a normal stop look like link failure.
wearRealtimeTalkController.stop(nodeId, attemptId)
}
internal suspend fun stopWearRealtimeTalk(owner: WearRealtimeAttemptOwner): Boolean =
wearRealtimeLifecycleMutex.withLock {
wearRealtimeTalkController.stop(owner)
}
internal fun appendWearRealtimeAudio(
nodeId: String,
owner: WearRealtimeAttemptOwner,
payload: ByteArray,
) {
if (wearRealtimeTalkControllerLazy.isInitialized()) {
wearRealtimeTalkController.appendAudio(nodeId, payload)
wearRealtimeTalkController.appendAudio(owner, payload)
}
}
@@ -2991,6 +3082,7 @@ class NodeRuntime private constructor(
voiceLifecycleEpoch.incrementAndGet()
}
if (value) {
refreshNodePermissionSurface()
refreshVoiceWakeCapabilitySurfaceIfChanged()
reconnectPreferredGatewayOnForeground()
scope.launch {
@@ -3263,6 +3355,15 @@ class NodeRuntime private constructor(
resolvePreferredGatewayEndpoint()?.let(::connect)
}
/**
* Reconnect a live node only when Android authority changed since its last connect.
*/
fun refreshNodePermissionSurface() {
val permissions = connectionManager.buildPermissions()
if (permissions == lastNodePermissions) return
refreshNodeSurfaceAfterSettingsChange()
}
fun setDisplayName(value: String) {
prefs.setDisplayName(value)
}
@@ -4247,12 +4348,14 @@ class NodeRuntime private constructor(
tls,
)
}
val nodeConnectOptions = connectionManager.buildNodeConnectOptions()
lastNodePermissions = nodeConnectOptions.permissions
nodeSession.connect(
endpoint,
auth.token,
auth.bootstrapToken,
auth.password,
connectionManager.buildNodeConnectOptions(),
nodeConnectOptions,
tls,
)
if (reconnect && operatorAuth != null) {
@@ -2,6 +2,7 @@ package ai.openclaw.app
import ai.openclaw.app.i18n.nativeString
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
@@ -15,12 +16,20 @@ import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleEventObserver
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.filterNotNull
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeout
import java.util.IdentityHashMap
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.coroutines.resume
@@ -28,28 +37,83 @@ import kotlin.coroutines.resume
* Serializes Android runtime-permission prompts behind coroutine-friendly request calls.
*/
class PermissionRequester internal constructor(
private val activity: ComponentActivity,
private val permissionRequestLauncher: (Array<String>, Int) -> Unit,
context: Context,
private val requestCodeAllocator: PermissionRequestCodeAllocator = PermissionRequestCodeAllocator(),
) {
private data class ActivityHost(
val activity: ComponentActivity,
val permissionRequestLauncher: (Array<String>, Int) -> Unit,
)
private data class ActiveActivityHost(
val host: ActivityHost,
val activation: Long,
)
private data class PendingPermissionRequest(
val requestCode: Int,
val permissions: List<String>,
val deferred: CompletableDeferred<Map<String, Boolean>>,
)
constructor(activity: ComponentActivity) : this(
activity = activity,
permissionRequestLauncher = { permissions, requestCode ->
ActivityCompat.requestPermissions(activity, permissions, requestCode)
},
)
private enum class RationaleResult {
Proceed,
Decline,
HostLost,
}
private enum class SettingsResult {
Shown,
HostLost,
}
private val appContext = context.applicationContext
private val mutex = Mutex()
private val activityHostLock = Any()
private val permissionRequestsLock = Any()
private val mainHandler = Handler(Looper.getMainLooper())
private val activityHosts = IdentityHashMap<ComponentActivity, ActivityHost>()
private val activeActivitySequences = IdentityHashMap<ComponentActivity, Long>()
private val activeActivityHost = MutableStateFlow<ActiveActivityHost?>(null)
private var nextActivityActivation = 0L
private val pendingPermissionRequests = mutableMapOf<Int, PendingPermissionRequest>()
internal fun attach(
activity: ComponentActivity,
permissionRequestLauncher: (Array<String>, Int) -> Unit = { permissions, requestCode ->
ActivityCompat.requestPermissions(activity, permissions, requestCode)
},
) {
synchronized(activityHostLock) {
activityHosts[activity] = ActivityHost(activity, permissionRequestLauncher)
publishActiveActivityHostLocked()
}
}
internal fun activate(activity: ComponentActivity) {
synchronized(activityHostLock) {
check(activityHosts.containsKey(activity)) { "permission Activity must attach before activation" }
nextActivityActivation += 1
activeActivitySequences[activity] = nextActivityActivation
publishActiveActivityHostLocked()
}
}
internal fun deactivate(activity: ComponentActivity) {
synchronized(activityHostLock) {
activeActivitySequences.remove(activity)
publishActiveActivityHostLocked()
}
}
internal fun detach(activity: ComponentActivity) {
synchronized(activityHostLock) {
activeActivitySequences.remove(activity)
activityHosts.remove(activity)
publishActiveActivityHostLocked()
}
}
/**
* Request missing Android runtime permissions and return the final grant state for every requested permission.
*/
@@ -61,29 +125,22 @@ class PermissionRequester internal constructor(
while (true) {
val missing =
permissions.filter { perm ->
ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED
ContextCompat.checkSelfPermission(appContext, perm) != PackageManager.PERMISSION_GRANTED
}
if (missing.isEmpty()) {
return permissions.associateWith { true }
}
val needsRationale =
missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) }
if (needsRationale) {
val proceed = showRationaleDialog(missing)
if (!proceed) {
return permissions.associateWith { perm ->
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
}
if (!confirmRationaleIfNeeded(missing, timeoutMs)) {
return permissions.associateWith { perm ->
ContextCompat.checkSelfPermission(appContext, perm) == PackageManager.PERMISSION_GRANTED
}
}
val deferred = CompletableDeferred<Map<String, Boolean>>()
val request = reservePermissionRequest(missing, deferred)
try {
withContext(Dispatchers.Main) {
permissionRequestLauncher(missing.toTypedArray(), request.requestCode)
}
launchPermissionRequest(missing, request.requestCode, timeoutMs)
} catch (err: Throwable) {
clearPermissionRequest(request)
throw err
@@ -100,17 +157,11 @@ class PermissionRequester internal constructor(
val merged =
permissions.associateWith { perm ->
val nowGranted =
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
ContextCompat.checkSelfPermission(appContext, perm) == PackageManager.PERMISSION_GRANTED
result[perm] == true || nowGranted
}
val denied =
merged.filterValues { !it }.keys.filter {
!ActivityCompat.shouldShowRequestPermissionRationale(activity, it)
}
if (denied.isNotEmpty()) {
showSettingsDialog(denied)
}
showSettingsForPermanentDenials(merged, timeoutMs)
return merged
}
@@ -157,23 +208,135 @@ class PermissionRequester internal constructor(
}
}
private suspend fun showRationaleDialog(permissions: List<String>): Boolean =
withContext(Dispatchers.Main) {
if (activity.isFinishing || activity.isDestroyed) {
return@withContext false
private fun publishActiveActivityHostLocked() {
val active =
activeActivitySequences.entries.maxByOrNull { it.value }?.let { entry ->
activityHosts[entry.key]?.let { host ->
ActiveActivityHost(host = host, activation = entry.value)
}
}
activeActivityHost.value = active
}
private suspend fun awaitActiveActivityHost(timeoutMs: Long): ActiveActivityHost =
withTimeout(timeoutMs) {
activeActivityHost
.filterNotNull()
.first { active ->
!active.host.activity.isFinishing && !active.host.activity.isDestroyed
}
}
private suspend fun launchPermissionRequest(
permissions: List<String>,
requestCode: Int,
timeoutMs: Long,
) {
withTimeout(timeoutMs) {
var rejected: ActiveActivityHost? = null
while (true) {
val active =
activeActivityHost
.filterNotNull()
.first { candidate ->
candidate != rejected &&
!candidate.host.activity.isFinishing &&
!candidate.host.activity.isDestroyed
}
val launched =
withContext(Dispatchers.Main) {
if (activeActivityHost.value != active) return@withContext false
val host = active.host
if (host.activity.isFinishing || host.activity.isDestroyed) return@withContext false
host.permissionRequestLauncher(permissions.toTypedArray(), requestCode)
true
}
if (launched) return@withTimeout
rejected = active
}
}
}
private suspend fun confirmRationaleIfNeeded(
permissions: List<String>,
timeoutMs: Long,
): Boolean =
withTimeout(timeoutMs) {
while (true) {
val active = awaitActiveActivityHost(timeoutMs)
val needsRationale =
withContext(Dispatchers.Main) {
if (!isCurrentActiveHost(active)) return@withContext null
permissions.any { permission ->
ActivityCompat.shouldShowRequestPermissionRationale(active.host.activity, permission)
}
} ?: continue
if (!needsRationale) return@withTimeout true
when (showRationaleDialog(active, permissions)) {
RationaleResult.Proceed -> return@withTimeout true
RationaleResult.Decline -> return@withTimeout false
RationaleResult.HostLost -> Unit
}
}
error("unreachable")
}
private suspend fun showSettingsForPermanentDenials(
grants: Map<String, Boolean>,
timeoutMs: Long,
) {
if (grants.values.none { granted -> !granted }) return
withTimeout(timeoutMs) {
while (true) {
val active = awaitActiveActivityHost(timeoutMs)
val denied =
withContext(Dispatchers.Main) {
if (!isCurrentActiveHost(active)) return@withContext null
grants
.filterValues { granted -> !granted }
.keys
.filter { permission ->
!ActivityCompat.shouldShowRequestPermissionRationale(active.host.activity, permission)
}
} ?: continue
if (denied.isEmpty()) return@withTimeout
when (showSettingsDialog(active, denied)) {
SettingsResult.Shown -> return@withTimeout
SettingsResult.HostLost -> Unit
}
}
}
}
private fun isCurrentActiveHost(active: ActiveActivityHost): Boolean =
activeActivityHost.value == active &&
!active.host.activity.isFinishing &&
!active.host.activity.isDestroyed
private suspend fun showRationaleDialog(
active: ActiveActivityHost,
permissions: List<String>,
): RationaleResult =
withContext(Dispatchers.Main) {
if (!isCurrentActiveHost(active)) {
return@withContext RationaleResult.HostLost
}
val activity = active.host.activity
suspendCancellableCoroutine { cont ->
val lifecycle = activity.lifecycle
var dialog: AlertDialog? = null
var observer: LifecycleEventObserver? = null
var hostLossJob: Job? = null
val finished = AtomicBoolean(false)
val removeObserver = {
observer?.let(lifecycle::removeObserver)
observer = null
}
fun finish(result: Boolean?) {
fun finish(result: RationaleResult?) {
if (!finished.compareAndSet(false, true)) return
hostLossJob?.cancel()
hostLossJob = null
removeObserver()
dialog?.dismiss()
if (result != null) {
@@ -183,62 +346,106 @@ class PermissionRequester internal constructor(
val actualObserver =
LifecycleEventObserver { _, event ->
if (event != Lifecycle.Event.ON_DESTROY) return@LifecycleEventObserver
// Do not resume a destroyed Activity with a positive result.
finish(false)
finish(RationaleResult.HostLost)
}
observer = actualObserver
lifecycle.addObserver(actualObserver)
hostLossJob =
CoroutineScope(cont.context)
.launch(start = CoroutineStart.LAZY) {
activeActivityHost.first { current -> current != active }
finish(RationaleResult.HostLost)
}.also(Job::start)
cont.invokeOnCancellation {
mainHandler.post {
finish(null)
}
}
if (finished.get()) return@suspendCancellableCoroutine
dialog =
AlertDialog
.Builder(activity)
.setTitle(nativeString("Permission required"))
.setMessage(buildRationaleMessage(permissions))
.setPositiveButton(nativeString("Continue")) { _, _ -> finish(true) }
.setNegativeButton(nativeString("Not now")) { _, _ -> finish(false) }
.setOnCancelListener { finish(false) }
.setPositiveButton(nativeString("Continue")) { _, _ -> finish(RationaleResult.Proceed) }
.setNegativeButton(nativeString("Not now")) { _, _ -> finish(RationaleResult.Decline) }
.setOnCancelListener { finish(RationaleResult.Decline) }
.show()
}
}
private suspend fun showSettingsDialog(permissions: List<String>) =
private suspend fun showSettingsDialog(
active: ActiveActivityHost,
permissions: List<String>,
): SettingsResult =
withContext(Dispatchers.Main) {
if (activity.isFinishing || activity.isDestroyed) return@withContext
val lifecycle = activity.lifecycle
var dialog: AlertDialog? = null
var observer: LifecycleEventObserver? = null
val removeObserver = {
observer?.let(lifecycle::removeObserver)
observer = null
if (!isCurrentActiveHost(active)) {
return@withContext SettingsResult.HostLost
}
val actualObserver =
LifecycleEventObserver { _, event ->
if (event != Lifecycle.Event.ON_DESTROY) return@LifecycleEventObserver
val activity = active.host.activity
suspendCancellableCoroutine { cont ->
val lifecycle = activity.lifecycle
var dialog: AlertDialog? = null
var observer: LifecycleEventObserver? = null
var hostLossJob: Job? = null
val finished = AtomicBoolean(false)
val removeObserver = {
observer?.let(lifecycle::removeObserver)
observer = null
}
fun finish(result: SettingsResult?) {
if (!finished.compareAndSet(false, true)) return
hostLossJob?.cancel()
hostLossJob = null
removeObserver()
dialog?.dismiss()
if (result != null) {
cont.resume(result)
}
}
observer = actualObserver
lifecycle.addObserver(actualObserver)
dialog =
AlertDialog
.Builder(activity)
.setTitle(nativeString("Enable permission in Settings"))
.setMessage(buildSettingsMessage(permissions))
.setPositiveButton(nativeString("Open Settings")) { _, _ ->
if (activity.isFinishing || activity.isDestroyed) return@setPositiveButton
val intent =
Intent(
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
Uri.fromParts("package", activity.packageName, null),
)
activity.startActivity(intent)
}.setNegativeButton(nativeString("Cancel"), null)
.setOnDismissListener { removeObserver() }
.show()
val actualObserver =
LifecycleEventObserver { _, event ->
if (event != Lifecycle.Event.ON_DESTROY) return@LifecycleEventObserver
finish(SettingsResult.HostLost)
}
observer = actualObserver
lifecycle.addObserver(actualObserver)
hostLossJob =
CoroutineScope(cont.context)
.launch(start = CoroutineStart.LAZY) {
activeActivityHost.first { current -> current != active }
finish(SettingsResult.HostLost)
}.also(Job::start)
cont.invokeOnCancellation {
mainHandler.post {
finish(null)
}
}
if (finished.get()) return@suspendCancellableCoroutine
dialog =
AlertDialog
.Builder(activity)
.setTitle(nativeString("Enable permission in Settings"))
.setMessage(buildSettingsMessage(permissions))
.setPositiveButton(nativeString("Open Settings")) { _, _ ->
if (!isCurrentActiveHost(active)) {
finish(SettingsResult.HostLost)
return@setPositiveButton
}
val intent =
Intent(
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
Uri.fromParts("package", activity.packageName, null),
)
activity.startActivity(intent)
finish(SettingsResult.Shown)
}.setNegativeButton(nativeString("Cancel")) { _, _ ->
finish(SettingsResult.Shown)
}.setOnCancelListener { finish(SettingsResult.Shown) }
.setOnDismissListener { finish(SettingsResult.Shown) }
.show()
}
}
private fun buildRationaleMessage(permissions: List<String>): String {
@@ -0,0 +1,91 @@
package ai.openclaw.app.node
import ai.openclaw.app.hasPhotoReadPermission
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.os.Build
import androidx.core.content.ContextCompat
/**
* Canonical Android authority snapshot shared by node approval and device.permissions.
*/
internal data class AndroidPermissionSnapshot(
val camera: Boolean,
val microphone: Boolean,
val location: Boolean,
val locationPrecise: Boolean,
val locationBackground: Boolean,
val smsSend: Boolean,
val smsRead: Boolean,
val notificationListener: Boolean,
val notifications: Boolean,
val photos: Boolean,
val contactsRead: Boolean,
val contactsWrite: Boolean,
val calendarRead: Boolean,
val calendarWrite: Boolean,
val callLog: Boolean,
val motion: Boolean,
) {
/**
* Keep independently grantable authority separate so any widening requires node reapproval.
*/
fun gatewayPermissions(): Map<String, Boolean> =
linkedMapOf(
"camera" to camera,
"microphone" to microphone,
"location" to location,
"locationPrecise" to locationPrecise,
"locationBackground" to locationBackground,
"smsSend" to smsSend,
"smsRead" to smsRead,
"notificationListener" to notificationListener,
"notifications" to notifications,
"photos" to photos,
"contactsRead" to contactsRead,
"contactsWrite" to contactsWrite,
"calendarRead" to calendarRead,
"calendarWrite" to calendarWrite,
"callLog" to callLog,
"motion" to motion,
)
}
internal fun readAndroidPermissionSnapshot(
context: Context,
smsEnabled: Boolean,
callLogEnabled: Boolean,
photosEnabled: Boolean,
backgroundLocationEnabled: Boolean,
): AndroidPermissionSnapshot {
fun hasPermission(permission: String): Boolean = ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED
val locationFine = hasPermission(Manifest.permission.ACCESS_FINE_LOCATION)
val locationCoarse = hasPermission(Manifest.permission.ACCESS_COARSE_LOCATION)
val telephonyAvailable = context.packageManager.hasSystemFeature(PackageManager.FEATURE_TELEPHONY)
return AndroidPermissionSnapshot(
camera = hasPermission(Manifest.permission.CAMERA),
microphone = hasPermission(Manifest.permission.RECORD_AUDIO),
location = locationFine || locationCoarse,
locationPrecise = locationFine,
locationBackground =
backgroundLocationEnabled &&
(locationFine || locationCoarse) &&
hasPermission(Manifest.permission.ACCESS_BACKGROUND_LOCATION),
smsSend = smsEnabled && telephonyAvailable && hasPermission(Manifest.permission.SEND_SMS),
smsRead = smsEnabled && telephonyAvailable && hasPermission(Manifest.permission.READ_SMS),
notificationListener = DeviceNotificationListenerService.isAccessEnabled(context),
notifications =
Build.VERSION.SDK_INT < 33 ||
hasPermission(Manifest.permission.POST_NOTIFICATIONS),
photos = photosEnabled && hasPhotoReadPermission(context),
contactsRead = hasPermission(Manifest.permission.READ_CONTACTS),
contactsWrite = hasPermission(Manifest.permission.WRITE_CONTACTS),
calendarRead = hasPermission(Manifest.permission.READ_CALENDAR),
calendarWrite = hasPermission(Manifest.permission.WRITE_CALENDAR),
callLog = callLogEnabled && hasPermission(Manifest.permission.READ_CALL_LOG),
motion = hasPermission(Manifest.permission.ACTIVITY_RECOGNITION),
)
}
@@ -14,7 +14,7 @@ import android.os.Build
/**
* Builds gateway connect metadata from current Android permissions, settings, and device identity.
*/
class ConnectionManager(
class ConnectionManager internal constructor(
private val prefs: SecurePrefs,
private val cameraEnabled: () -> Boolean,
private val locationMode: () -> LocationMode,
@@ -29,6 +29,7 @@ class ConnectionManager(
private val voiceWakeAvailable: () -> Boolean,
private val mobileUiAvailable: () -> Boolean,
private val inlineWidgetsAvailable: () -> Boolean,
private val permissionSnapshot: () -> AndroidPermissionSnapshot,
private val manualTls: (GatewayEndpoint) -> Boolean,
) {
companion object {
@@ -158,6 +159,9 @@ class ConnectionManager(
/** Builds the gateway-advertised capability list from current permission and feature state. */
fun buildCapabilities(): List<String> = InvokeCommandRegistry.advertisedCapabilities(runtimeFlags())
/** Builds the current independently grantable Android permission surface. */
fun buildPermissions(): Map<String, Boolean> = permissionSnapshot().gatewayPermissions()
/**
* Debug Android builds advertise a dev version so gateway logs do not look like release clients.
*/
@@ -213,7 +217,7 @@ class ConnectionManager(
scopes = emptyList(),
caps = buildCapabilities(),
commands = buildInvokeCommands(),
permissions = emptyMap(),
permissions = buildPermissions(),
client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"),
userAgent = buildUserAgent(),
)
@@ -3,8 +3,6 @@ package ai.openclaw.app.node
import ai.openclaw.app.BuildConfig
import ai.openclaw.app.SensitiveFeatureConfig
import ai.openclaw.app.gateway.GatewaySession
import ai.openclaw.app.hasPhotoReadPermission
import android.Manifest
import android.annotation.SuppressLint
import android.app.ActivityManager
import android.content.Context
@@ -20,7 +18,6 @@ import android.os.Environment
import android.os.PowerManager
import android.os.StatFs
import android.os.SystemClock
import androidx.core.content.ContextCompat
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
@@ -118,27 +115,56 @@ class DeviceHandler private constructor(
private val callLogEnabled: Boolean = SensitiveFeatureConfig.callLogEnabled,
private val photosEnabled: Boolean = SensitiveFeatureConfig.photosEnabled,
private val appSource: DeviceAppSource = AndroidDeviceAppSource(appContext),
private val permissionSnapshot: () -> AndroidPermissionSnapshot,
) {
constructor(
appContext: Context,
smsEnabled: Boolean = SensitiveFeatureConfig.smsEnabled,
callLogEnabled: Boolean = SensitiveFeatureConfig.callLogEnabled,
photosEnabled: Boolean = SensitiveFeatureConfig.photosEnabled,
backgroundLocationEnabled: Boolean = SensitiveFeatureConfig.backgroundLocationEnabled,
) : this(
appContext = appContext,
smsEnabled = smsEnabled,
callLogEnabled = callLogEnabled,
photosEnabled = photosEnabled,
appSource = AndroidDeviceAppSource(appContext),
permissionSnapshot = {
readAndroidPermissionSnapshot(
context = appContext,
smsEnabled = smsEnabled,
callLogEnabled = callLogEnabled,
photosEnabled = photosEnabled,
backgroundLocationEnabled = backgroundLocationEnabled,
)
},
)
companion object {
internal fun withPermissionSnapshot(
appContext: Context,
smsEnabled: Boolean,
callLogEnabled: Boolean,
photosEnabled: Boolean,
permissionSnapshot: () -> AndroidPermissionSnapshot,
): DeviceHandler =
DeviceHandler(
appContext = appContext,
smsEnabled = smsEnabled,
callLogEnabled = callLogEnabled,
photosEnabled = photosEnabled,
appSource = AndroidDeviceAppSource(appContext),
permissionSnapshot = permissionSnapshot,
)
internal fun forTesting(
appContext: Context,
appSource: DeviceAppSource,
smsEnabled: Boolean = SensitiveFeatureConfig.smsEnabled,
callLogEnabled: Boolean = SensitiveFeatureConfig.callLogEnabled,
photosEnabled: Boolean = SensitiveFeatureConfig.photosEnabled,
backgroundLocationEnabled: Boolean = SensitiveFeatureConfig.backgroundLocationEnabled,
permissionSnapshot: (() -> AndroidPermissionSnapshot)? = null,
): DeviceHandler =
DeviceHandler(
appContext = appContext,
@@ -146,6 +172,16 @@ class DeviceHandler private constructor(
callLogEnabled = callLogEnabled,
photosEnabled = photosEnabled,
appSource = appSource,
permissionSnapshot =
permissionSnapshot ?: {
readAndroidPermissionSnapshot(
context = appContext,
smsEnabled = smsEnabled,
callLogEnabled = callLogEnabled,
photosEnabled = photosEnabled,
backgroundLocationEnabled = backgroundLocationEnabled,
)
},
)
/**
@@ -316,24 +352,8 @@ class DeviceHandler private constructor(
}
private fun permissionsPayloadJson(): String {
val snapshot = permissionSnapshot()
val canSendSms = appContext.packageManager.hasSystemFeature(PackageManager.FEATURE_TELEPHONY)
val smsSendGranted = hasPermission(Manifest.permission.SEND_SMS)
val smsReadGranted = hasPermission(Manifest.permission.READ_SMS)
val notificationAccess = DeviceNotificationListenerService.isAccessEnabled(appContext)
val photosGranted =
if (!photosEnabled) {
false
} else {
hasPhotoReadPermission(appContext)
}
val motionGranted = hasPermission(Manifest.permission.ACTIVITY_RECOGNITION)
val notificationsGranted =
if (Build.VERSION.SDK_INT >= 33) {
// POST_NOTIFICATIONS exists only on Android 13+.
hasPermission(Manifest.permission.POST_NOTIFICATIONS)
} else {
true
}
return buildJsonObject {
put(
"permissions",
@@ -341,23 +361,21 @@ class DeviceHandler private constructor(
put(
"camera",
permissionStateJson(
granted = hasPermission(Manifest.permission.CAMERA),
granted = snapshot.camera,
promptableWhenDenied = true,
),
)
put(
"microphone",
permissionStateJson(
granted = hasPermission(Manifest.permission.RECORD_AUDIO),
granted = snapshot.microphone,
promptableWhenDenied = true,
),
)
put(
"location",
permissionStateJson(
granted =
hasPermission(Manifest.permission.ACCESS_FINE_LOCATION) ||
hasPermission(Manifest.permission.ACCESS_COARSE_LOCATION),
granted = snapshot.location,
promptableWhenDenied = true,
),
)
@@ -370,8 +388,8 @@ class DeviceHandler private constructor(
if (hasAnySmsCapability(
smsEnabled,
canSendSms,
smsSendGranted,
smsReadGranted,
snapshot.smsSend,
snapshot.smsRead,
)
) {
"granted"
@@ -380,21 +398,31 @@ class DeviceHandler private constructor(
},
),
)
put("promptable", JsonPrimitive(isSmsPromptable(smsEnabled, canSendSms, smsSendGranted, smsReadGranted)))
put(
"promptable",
JsonPrimitive(
isSmsPromptable(
smsEnabled,
canSendSms,
snapshot.smsSend,
snapshot.smsRead,
),
),
)
put(
"capabilities",
buildJsonObject {
put(
"send",
permissionStateJson(
granted = smsEnabled && smsSendGranted && canSendSms,
granted = snapshot.smsSend,
promptableWhenDenied = smsEnabled && canSendSms,
),
)
put(
"read",
permissionStateJson(
granted = smsEnabled && smsReadGranted && canSendSms,
granted = snapshot.smsRead,
promptableWhenDenied = smsEnabled && canSendSms,
),
)
@@ -405,53 +433,49 @@ class DeviceHandler private constructor(
put(
"notificationListener",
permissionStateJson(
granted = notificationAccess,
granted = snapshot.notificationListener,
promptableWhenDenied = true,
),
)
put(
"notifications",
permissionStateJson(
granted = notificationsGranted,
granted = snapshot.notifications,
promptableWhenDenied = true,
),
)
put(
"photos",
permissionStateJson(
granted = photosGranted,
granted = snapshot.photos,
promptableWhenDenied = photosEnabled,
),
)
put(
"contacts",
permissionStateJson(
granted =
hasPermission(Manifest.permission.READ_CONTACTS) &&
hasPermission(Manifest.permission.WRITE_CONTACTS),
granted = snapshot.contactsRead && snapshot.contactsWrite,
promptableWhenDenied = true,
),
)
put(
"calendar",
permissionStateJson(
granted =
hasPermission(Manifest.permission.READ_CALENDAR) &&
hasPermission(Manifest.permission.WRITE_CALENDAR),
granted = snapshot.calendarRead && snapshot.calendarWrite,
promptableWhenDenied = true,
),
)
put(
"callLog",
permissionStateJson(
granted = callLogEnabled && hasPermission(Manifest.permission.READ_CALL_LOG),
granted = snapshot.callLog,
promptableWhenDenied = callLogEnabled,
),
)
put(
"motion",
permissionStateJson(
granted = motionGranted,
granted = snapshot.motion,
promptableWhenDenied = true,
),
)
@@ -617,11 +641,6 @@ class DeviceHandler private constructor(
put("promptable", JsonPrimitive(!granted && promptableWhenDenied))
}
private fun hasPermission(permission: String): Boolean =
(
ContextCompat.checkSelfPermission(appContext, permission) == PackageManager.PERMISSION_GRANTED
)
private fun mapMemoryPressure(
totalBytes: Long,
availableBytes: Long,
@@ -159,6 +159,7 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.text.style.TextDecoration
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.lifecycle.Lifecycle
@@ -199,6 +200,14 @@ internal fun resolveInitialChatLoadSessionKey(
return main
}
/** Reserves a viewport strip so the jump-to-latest target never covers chat content. */
internal fun chatReaderListBottomInset(showJumpToLatest: Boolean): Dp =
if (showJumpToLatest) {
56.dp
} else {
0.dp
}
internal enum class ChatComposerTrailingAction {
StartTalk,
StopTalk,
@@ -1349,7 +1358,10 @@ private fun ChatMessageList(
Box(modifier = modifier.fillMaxWidth()) {
LazyColumn(
modifier = Modifier.fillMaxSize(),
modifier =
Modifier
.fillMaxSize()
.padding(bottom = chatReaderListBottomInset(readerScroll.showJumpToLatest)),
state = readerScroll.listState,
reverseLayout = true,
verticalArrangement = Arrangement.spacedBy(5.dp),
@@ -51,7 +51,7 @@ internal class WearProxyController(
private val connectGateway: suspend () -> Unit = {},
private val disconnectGateway: suspend () -> Unit = {},
private val startRealtimeTalk:
suspend (nodeId: String, sessionKey: String, attemptId: String, language: String?) -> WearRealtimeTalkSnapshot? = { _, _, _, _ -> null },
suspend (nodeId: String, sessionKey: String, attemptId: String, language: String?, attemptScopedAudio: Boolean) -> WearRealtimeTalkSnapshot? = { _, _, _, _, _ -> null },
private val stopRealtimeTalk: suspend (nodeId: String, attemptId: String) -> WearRealtimeTalkSnapshot? = { _, _ -> null },
) {
suspend fun handle(
@@ -91,7 +91,7 @@ internal class WearProxyController(
params: JsonObject,
): JsonElement {
if (sourceNodeId.isBlank()) throw WearProxyInvalidRequest("Missing Watch node")
params.requireOnly("sessionKey", "attemptId", "language")
params.requireOnly("sessionKey", "attemptId", "language", "attemptScopedAudio")
val sessionKey = params.stringParam("sessionKey", MAX_SESSION_KEY_CHARS)
val attemptId = params.stringParam("attemptId", MAX_ATTEMPT_ID_CHARS)
val language =
@@ -100,8 +100,9 @@ internal class WearProxyController(
?.lowercase(Locale.ROOT)
?.takeIf { value -> value.length == 2 && value.all { it in 'a'..'z' } }
?: if ("language" in params) throw WearProxyInvalidRequest("Invalid language") else null
val attemptScopedAudio = params.optionalBooleanParam("attemptScopedAudio") ?: false
val snapshot =
startRealtimeTalk(sourceNodeId, sessionKey, attemptId, language)
startRealtimeTalk(sourceNodeId, sessionKey, attemptId, language, attemptScopedAudio)
?: throw WearProxyGatewayException("action_rejected", "Real-Time Talk is unavailable")
return WearRealtimeTalkCodec.encode(snapshot)
}
@@ -580,6 +581,12 @@ private fun JsonObject.optionalIntParam(
return value
}
private fun JsonObject.optionalBooleanParam(name: String): Boolean? {
if (name !in this) return null
return this[name].booleanPrimitiveOrNull()
?: throw WearProxyInvalidRequest("Invalid $name")
}
private fun JsonElement.asObject(method: String): JsonObject = this as? JsonObject ?: throw WearProxyGatewayException("invalid_response", "$method returned an invalid response")
private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray
@@ -19,6 +19,10 @@ class WearProxyListenerService : WearableListenerService() {
override fun onChannelOpened(channel: ChannelClient.Channel) {
val app = application as? NodeApp ?: return
app.wearRealtimeChannels.accept(channel, app::ensureBackgroundRuntime)
app.wearRealtimeChannels.accept(
channel = channel,
appendAudio = { owner, payload -> app.ensureBackgroundRuntime().appendWearRealtimeAudio(owner, payload) },
stopTalk = { owner -> app.ensureBackgroundRuntime().stopWearRealtimeTalk(owner) },
)
}
}
@@ -1,6 +1,5 @@
package ai.openclaw.app.wear
import ai.openclaw.app.NodeRuntime
import ai.openclaw.wear.shared.WearProtocol
import ai.openclaw.wear.shared.WearRealtimeAudioFrameType
import ai.openclaw.wear.shared.WearRealtimeAudioFraming
@@ -8,8 +7,11 @@ import android.content.Context
import com.google.android.gms.wearable.ChannelClient
import com.google.android.gms.wearable.Wearable
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.delay
import kotlinx.coroutines.ensureActive
@@ -20,80 +22,539 @@ import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
import java.io.InputStream
import java.io.OutputStream
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicLong
internal data class WearRealtimeAttemptOwner(
val nodeId: String,
val attemptId: String,
val channelGeneration: Long,
)
internal data class WearRealtimeChannelClaim(
val owner: WearRealtimeAttemptOwner,
val newlyAcquired: Boolean,
)
internal data class WearRealtimeChannelResources(
val input: InputStream,
val output: OutputStream,
)
internal interface WearRealtimeChannelTransport {
suspend fun open(channel: ChannelClient.Channel): WearRealtimeChannelResources?
suspend fun close(
channel: ChannelClient.Channel,
resources: WearRealtimeChannelResources?,
)
}
private class GoogleWearRealtimeChannelTransport(
context: Context,
) : WearRealtimeChannelTransport {
private val client = Wearable.getChannelClient(context.applicationContext)
override suspend fun open(channel: ChannelClient.Channel): WearRealtimeChannelResources? {
val input = runCatching { client.getInputStream(channel).awaitWearTask() }.getOrNull()
val output = runCatching { client.getOutputStream(channel).awaitWearTask() }.getOrNull()
if (input == null || output == null) {
input.closeQuietly()
output.closeQuietly()
runCatching { client.close(channel).awaitWearTask() }
return null
}
return WearRealtimeChannelResources(input, output)
}
override suspend fun close(
channel: ChannelClient.Channel,
resources: WearRealtimeChannelResources?,
) {
resources?.input.closeQuietly()
resources?.output.closeQuietly()
runCatching { client.close(channel).awaitWearTask() }
}
}
internal class WearRealtimeChannelRegistry(
context: Context,
private val scope: CoroutineScope,
private val transport: WearRealtimeChannelTransport,
private val connectionReadyTimeoutMillis: Long = DEFAULT_CONNECTION_READY_TIMEOUT_MILLIS,
private val pendingConnectionTimeoutMillis: Long = WearProtocol.REALTIME_AUDIO_PENDING_CHANNEL_TIMEOUT_MILLIS,
private val retireCallbackTimeoutMillis: Long = DEFAULT_RETIRE_CALLBACK_TIMEOUT_MILLIS,
private val maxStagedConnectionsPerNode: Int = DEFAULT_MAX_STAGED_CONNECTIONS_PER_NODE,
private val maxStagedConnections: Int = DEFAULT_MAX_STAGED_CONNECTIONS,
) {
private val channelClient = Wearable.getChannelClient(context.applicationContext)
private val connections = ConcurrentHashMap<String, Connection>()
private data class ChannelKey(
val nodeId: String,
val path: String,
)
private data class ChannelPromotion(
val connection: Connection,
val displaced: Connection?,
val claimSequence: Long,
)
private sealed interface ChannelClaimSelection {
data class Claimed(
val claim: WearRealtimeChannelClaim,
) : ChannelClaimSelection
data class Promote(
val promotion: ChannelPromotion,
) : ChannelClaimSelection
data object Wait : ChannelClaimSelection
data object Superseded : ChannelClaimSelection
}
constructor(context: Context, scope: CoroutineScope) : this(
scope,
GoogleWearRealtimeChannelTransport(context),
)
private val lifecycleMutex = Mutex()
private val channelGeneration = AtomicLong()
private val claimSequence = AtomicLong()
private val connections = mutableMapOf<String, Connection>()
private val cleanupScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
// Channel receipt is not attempt order. Stage exact paths until a start claim reserves promotion.
private val pendingConnections = mutableMapOf<ChannelKey, Connection>()
private val promotingConnections = mutableMapOf<ChannelKey, Connection>()
private val latestClaimSequences = mutableMapOf<String, Long>()
private val openingConnectionsByNode = mutableMapOf<String, Int>()
private var openingConnectionCount = 0
fun accept(
channel: ChannelClient.Channel,
runtime: () -> NodeRuntime,
appendAudio: (owner: WearRealtimeAttemptOwner, payload: ByteArray) -> Unit,
stopTalk: suspend (owner: WearRealtimeAttemptOwner) -> Unit,
) {
if (channel.path != WearProtocol.REALTIME_AUDIO_CHANNEL_PATH || channel.nodeId.isBlank()) {
scope.launch { runCatching { channelClient.close(channel).awaitWearTask() } }
if (!WearProtocol.isRealtimeAudioChannelPath(channel.path) || channel.nodeId.isBlank()) {
scope.launch { transport.close(channel, null) }
return
}
val generation = channelGeneration.incrementAndGet()
scope.launch(Dispatchers.IO) {
val input = runCatching { channelClient.getInputStream(channel).awaitWearTask() }.getOrNull()
val output = runCatching { channelClient.getOutputStream(channel).awaitWearTask() }.getOrNull()
if (input == null || output == null) {
input.closeQuietly()
output.closeQuietly()
runCatching { channelClient.close(channel).awaitWearTask() }
if (!reserveOpeningSlot(channel.nodeId)) {
runCatching { transport.close(channel, null) }
return@launch
}
val connection = Connection(channel, input, output)
connections.put(channel.nodeId, connection)?.close(channelClient)
var openingSlotReserved = true
try {
while (connections[channel.nodeId] === connection) {
val frame = WearRealtimeAudioFraming.read(input) ?: break
if (frame.type != WearRealtimeAudioFrameType.INPUT_PCM) break
runtime().appendWearRealtimeAudio(channel.nodeId, frame.payload)
val resources = transport.open(channel) ?: return@launch
val connection = Connection(channel, resources, generation, stopTalk)
var published = false
var activated = false
var displacedActive: Connection? = null
val displacedPending =
lifecycleMutex.withLock {
releaseOpeningSlotLocked(channel.nodeId)
openingSlotReserved = false
val active = connections[channel.nodeId]
val pending = pendingConnections[connection.key]
if (
active?.generation?.let { it > generation } == true ||
pending?.generation?.let { it > generation } == true
) {
return@withLock null
}
published = true
connection.ready = true
if (
WearProtocol.isAttemptScopedRealtimeAudioChannelPath(channel.path) &&
active?.takeIf { it.ready && !it.retirementStarted.get() }?.channel?.path == channel.path
) {
// Attempt-scoped reconnects can inherit the live owner before the old reader reports EOF.
active.ready = false
connection.owner = active.owner
connection.claimSequence = active.claimSequence
active.owner = null
connections[channel.nodeId] = connection
displacedActive = active
activated = true
pendingConnections.remove(connection.key)
} else {
pendingConnections.put(connection.key, connection)
}
}
if (!published) {
connection.retire(transport)
return@launch
}
displacedPending?.retire(transport)
displacedActive?.retire(transport)
if (activated) {
connection.activation.complete(true)
} else {
schedulePendingExpiry(connection)
}
try {
// The Watch starts capture after the start RPC; do not consume its PCM before that claim owns this path.
if (!connection.activation.await()) return@launch
while (isKnown(connection)) {
val frame = WearRealtimeAudioFraming.read(resources.input) ?: break
if (frame.type != WearRealtimeAudioFrameType.INPUT_PCM) break
val owner =
lifecycleMutex.withLock {
connection.owner.takeIf {
connection.ready && connections[channel.nodeId] === connection
}
}
if (owner != null) appendAudio(owner, frame.payload)
}
} catch (err: CancellationException) {
currentCoroutineContext().ensureActive()
} catch (_: Throwable) {
// A malformed frame or transport failure owns this channel only.
} finally {
retireKnownConnection(connection)
}
} catch (err: CancellationException) {
currentCoroutineContext().ensureActive()
} catch (_: Throwable) {
// A malformed frame or transport failure owns this channel only.
} finally {
if (connections.remove(channel.nodeId, connection)) {
connection.close(channelClient)
runCatching { runtime().stopWearRealtimeTalk(channel.nodeId) }
if (openingSlotReserved) {
withContext(NonCancellable) {
releaseOpeningSlot(channel.nodeId)
}
}
}
}
}
suspend fun claim(
nodeId: String,
attemptId: String,
attemptScopedAudio: Boolean = true,
): WearRealtimeChannelClaim? {
val expectedPath =
if (attemptScopedAudio) {
WearProtocol.realtimeAudioChannelPath(attemptId)
} else {
WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH
}
val key = ChannelKey(nodeId, expectedPath)
val sequence = claimSequence.incrementAndGet()
lifecycleMutex.withLock {
latestClaimSequences[nodeId] = sequence
}
val deadlineNanos =
System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(connectionReadyTimeoutMillis)
while (true) {
currentCoroutineContext().ensureActive()
when (val selection = reserveClaim(nodeId, attemptId, expectedPath, key, sequence)) {
is ChannelClaimSelection.Claimed -> return selection.claim
is ChannelClaimSelection.Promote ->
return completePromotion(nodeId, attemptId, key, selection.promotion)
ChannelClaimSelection.Superseded -> return null
ChannelClaimSelection.Wait -> {
val remainingNanos = deadlineNanos - System.nanoTime()
if (remainingNanos <= 0L) return null
val waitMillis =
TimeUnit.NANOSECONDS
.toMillis(remainingNanos)
.coerceIn(1L, CONNECTION_POLL_MILLIS)
delay(waitMillis)
}
}
}
}
private suspend fun reserveClaim(
nodeId: String,
attemptId: String,
expectedPath: String,
key: ChannelKey,
sequence: Long,
): ChannelClaimSelection =
lifecycleMutex.withLock {
if (latestClaimSequences[nodeId] != sequence) return@withLock ChannelClaimSelection.Superseded
if (promotingConnections.keys.any { promotingKey -> promotingKey.nodeId == nodeId }) {
return@withLock ChannelClaimSelection.Wait
}
val connection = pendingConnections.remove(key)
if (connection != null) {
val displaced = connections[nodeId]
// Reservation is the ordering boundary. Later claims wait for this bounded handoff, then may replace it.
connection.ready = false
displaced?.ready = false
promotingConnections[key] = connection
return@withLock ChannelClaimSelection.Promote(
ChannelPromotion(
connection = connection,
displaced = displaced,
claimSequence = sequence,
),
)
}
connections[nodeId]?.let { active ->
if (active.claimSequence > sequence) return@withLock ChannelClaimSelection.Superseded
val current = active.owner
if (active.ready && active.channel.path == expectedPath) {
if (current?.attemptId == attemptId) {
active.claimSequence = sequence
return@withLock ChannelClaimSelection.Claimed(
WearRealtimeChannelClaim(current, newlyAcquired = false),
)
}
if (current == null) {
val owner = WearRealtimeAttemptOwner(nodeId, attemptId, active.generation)
active.owner = owner
active.claimSequence = sequence
return@withLock ChannelClaimSelection.Claimed(
WearRealtimeChannelClaim(owner, newlyAcquired = true),
)
}
}
}
ChannelClaimSelection.Wait
}
private suspend fun reserveOpeningSlot(nodeId: String): Boolean =
lifecycleMutex.withLock {
val stagedForNode =
pendingConnections.keys.count { key -> key.nodeId == nodeId } +
promotingConnections.keys.count { key -> key.nodeId == nodeId } +
(openingConnectionsByNode[nodeId] ?: 0)
val stagedTotal = pendingConnections.size + promotingConnections.size + openingConnectionCount
if (
stagedForNode >= maxStagedConnectionsPerNode ||
stagedTotal >= maxStagedConnections
) {
return@withLock false
}
openingConnectionsByNode[nodeId] = (openingConnectionsByNode[nodeId] ?: 0) + 1
openingConnectionCount += 1
true
}
private suspend fun releaseOpeningSlot(nodeId: String) {
lifecycleMutex.withLock {
releaseOpeningSlotLocked(nodeId)
}
}
private fun releaseOpeningSlotLocked(nodeId: String) {
val count = checkNotNull(openingConnectionsByNode[nodeId])
if (count == 1) {
openingConnectionsByNode.remove(nodeId)
} else {
openingConnectionsByNode[nodeId] = count - 1
}
openingConnectionCount -= 1
}
private suspend fun completePromotion(
nodeId: String,
attemptId: String,
key: ChannelKey,
promotion: ChannelPromotion,
): WearRealtimeChannelClaim? {
var connection = promotion.connection
var promoted = false
try {
// Discovery already succeeded, so finish this bounded handoff even if the polling deadline has elapsed.
promotion.displaced?.let { retireCurrentConnection(it) }
while (true) {
currentCoroutineContext().ensureActive()
var superseded: Connection? = null
var owner: WearRealtimeAttemptOwner? = null
val valid =
lifecycleMutex.withLock {
if (promotingConnections[key] !== connection || connection.retirementStarted.get()) {
return@withLock false
}
val replacement =
pendingConnections[key]?.takeIf {
it.generation > connection.generation && !it.retirementStarted.get()
}
if (replacement != null) {
pendingConnections.remove(key, replacement)
replacement.ready = false
promotingConnections[key] = replacement
superseded = connection
connection = replacement
} else {
promotingConnections.remove(key, connection)
owner = WearRealtimeAttemptOwner(nodeId, attemptId, connection.generation)
connection.owner = owner
connection.claimSequence = promotion.claimSequence
connection.ready = true
connections[nodeId] = connection
promoted = true
}
true
}
if (!valid) return null
val retired = superseded
if (retired != null) {
// A reconnect can arrive while owner cleanup runs. Publish the newest exact-path channel.
retired.retire(transport)
continue
}
val committedOwner = checkNotNull(owner)
connection.activation.complete(true)
return WearRealtimeChannelClaim(committedOwner, newlyAcquired = true)
}
} finally {
if (!promoted) retirePromotingConnection(connection)
}
}
suspend fun send(
nodeId: String,
owner: WearRealtimeAttemptOwner,
type: WearRealtimeAudioFrameType,
payload: ByteArray,
) {
val connection =
withTimeoutOrNull(CONNECTION_READY_TIMEOUT_MILLIS) {
var current = connections[nodeId]
while (current == null) {
delay(CONNECTION_POLL_MILLIS)
current = connections[nodeId]
}
current
lifecycleMutex.withLock {
connections[owner.nodeId]?.takeIf { it.owner == owner }
} ?: error("Wear realtime audio channel is unavailable")
connection.write(type, payload)
}
suspend fun close(nodeId: String) {
connections.remove(nodeId)?.close(channelClient)
suspend fun release(owner: WearRealtimeAttemptOwner) {
lifecycleMutex.withLock {
connections[owner.nodeId]
?.takeIf { it.owner == owner }
?.owner = null
}
}
suspend fun isCurrent(owner: WearRealtimeAttemptOwner): Boolean =
lifecycleMutex.withLock {
connections[owner.nodeId]?.let { connection ->
connection.ready &&
!connection.retirementStarted.get() &&
connection.owner == owner
} == true
}
suspend fun close(owner: WearRealtimeAttemptOwner) {
val connection =
lifecycleMutex.withLock {
connections[owner.nodeId]
?.takeIf { it.owner == owner }
?.also { it.ready = false }
}
connection?.let { retireCurrentConnection(it) }
}
private fun schedulePendingExpiry(connection: Connection) {
scope.launch {
delay(pendingConnectionTimeoutMillis)
val expired =
lifecycleMutex.withLock {
pendingConnections.remove(connection.key, connection)
}
if (expired) connection.retire(transport)
}
}
private suspend fun isKnown(item: Connection): Boolean = lifecycleMutex.withLock { isKnownLocked(item) }
private fun isKnownLocked(item: Connection): Boolean =
connections[item.channel.nodeId] === item ||
pendingConnections[item.key] === item ||
promotingConnections[item.key] === item
private fun isCurrentLocked(item: Connection): Boolean = connections[item.channel.nodeId] === item
private suspend fun retireCurrentConnection(connection: Connection) {
withContext(NonCancellable) {
lifecycleMutex.withLock {
if (isCurrentLocked(connection)) connection.ready = false
}
connection.retire(transport)
lifecycleMutex.withLock {
if (isCurrentLocked(connection)) connections.remove(connection.channel.nodeId)
}
}
}
private suspend fun retirePromotingConnection(connection: Connection) {
withContext(NonCancellable) {
lifecycleMutex.withLock {
promotingConnections.remove(connection.key, connection)
}
connection.retire(transport)
}
}
private suspend fun retireKnownConnection(connection: Connection) {
withContext(NonCancellable) {
lifecycleMutex.withLock {
if (isCurrentLocked(connection)) connection.ready = false
pendingConnections.remove(connection.key, connection)
promotingConnections.remove(connection.key, connection)
}
connection.retire(transport)
lifecycleMutex.withLock {
if (isCurrentLocked(connection)) connections.remove(connection.channel.nodeId)
}
}
}
private suspend fun Connection.retire(transport: WearRealtimeChannelTransport) {
if (retirementStarted.compareAndSet(false, true)) {
val retiringOwner = owner
withContext(NonCancellable) {
try {
activation.complete(false)
retiringOwner?.let { owner ->
val stopJob =
cleanupScope.launch {
runCatching { stopTalk(owner) }
}
withTimeoutOrNull(retireCallbackTimeoutMillis) {
stopJob.join()
}
}
val closedWithinHandoff =
withTimeoutOrNull(retireCallbackTimeoutMillis) {
runCatching { close(transport) }.isSuccess
} == true
if (!closedWithinHandoff) {
// Keep transport cleanup alive after the bounded owner handoff returns.
cleanupScope.launch {
runCatching { close(transport) }
}
}
} finally {
retirementComplete.complete(Unit)
}
}
} else {
withContext(NonCancellable) {
withTimeoutOrNull(RETIRE_COMPLETION_TIMEOUT_MILLIS) {
retirementComplete.await()
}
}
}
}
private class Connection(
val channel: ChannelClient.Channel,
val input: InputStream,
val output: OutputStream,
val resources: WearRealtimeChannelResources,
val generation: Long,
val stopTalk: suspend (owner: WearRealtimeAttemptOwner) -> Unit,
) {
val key = ChannelKey(channel.nodeId, channel.path)
private val writeMutex = Mutex()
private val closeMutex = Mutex()
private var closed = false
val retirementStarted = AtomicBoolean()
val retirementComplete = CompletableDeferred<Unit>()
val activation = CompletableDeferred<Boolean>()
@Volatile var owner: WearRealtimeAttemptOwner? = null
@Volatile var claimSequence: Long = 0L
@Volatile var ready: Boolean = false
suspend fun write(
type: WearRealtimeAudioFrameType,
@@ -101,21 +562,30 @@ internal class WearRealtimeChannelRegistry(
) {
writeMutex.withLock {
withContext(Dispatchers.IO) {
WearRealtimeAudioFraming.write(output, type, payload)
WearRealtimeAudioFraming.write(resources.output, type, payload)
}
}
}
suspend fun close(client: ChannelClient) {
input.closeQuietly()
output.closeQuietly()
runCatching { client.close(channel).awaitWearTask() }
suspend fun close(transport: WearRealtimeChannelTransport) {
// Retirement must not close the stream beneath a frame already selected for this connection.
writeMutex.withLock {
closeMutex.withLock {
if (closed) return
transport.close(channel, resources)
closed = true
}
}
}
}
private companion object {
const val CONNECTION_POLL_MILLIS = 25L
const val CONNECTION_READY_TIMEOUT_MILLIS = 3_000L
const val DEFAULT_CONNECTION_READY_TIMEOUT_MILLIS = 3_000L
const val DEFAULT_RETIRE_CALLBACK_TIMEOUT_MILLIS = 1_000L
const val DEFAULT_MAX_STAGED_CONNECTIONS_PER_NODE = 4
const val DEFAULT_MAX_STAGED_CONNECTIONS = 16
const val RETIRE_COMPLETION_TIMEOUT_MILLIS = 2_500L
}
}
@@ -18,6 +18,8 @@ import android.util.Log
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.delay
@@ -97,9 +99,14 @@ internal class WearRealtimeTalkController(
timeoutMs: Long,
onError: (String) -> Unit,
) -> Unit,
private val sendWatchFrame: suspend (nodeId: String, type: WearRealtimeAudioFrameType, payload: ByteArray) -> Unit,
private val sendWatchFrame:
suspend (
owner: WearRealtimeAttemptOwner,
type: WearRealtimeAudioFrameType,
payload: ByteArray,
) -> Unit,
private val onSnapshot: (WearRealtimeTalkSnapshot) -> Unit = {},
private val onForceCloseWatchChannel: (String) -> Unit = {},
private val onForceCloseWatchChannel: (WearRealtimeAttemptOwner) -> Unit = {},
) {
private val json = Json { ignoreUnknownKeys = true }
private val lifecycleMutex = Mutex()
@@ -111,18 +118,18 @@ internal class WearRealtimeTalkController(
@Volatile private var sessionId: String? = null
@Volatile private var ownerNodeId: String? = null
@Volatile private var activeOwner: WearRealtimeAttemptOwner? = null
@Volatile private var ownerSessionKey: String? = null
@Volatile private var ownerAttemptId: String? = null
private var audioFrames: Channel<ByteArray>? = null
private var appendJob: Job? = null
private val outputQueueLock = Any()
private var outputQueue: WearRealtimeOutputQueue? = null
private var outputJob: Job? = null
private var playbackIdleJob: Job? = null
private var eventDispatchScope: CoroutineScope? = null
private var playbackEndsAtMillis = 0L
private var userEntryId: String? = null
private var assistantEntryId: String? = null
@@ -147,26 +154,29 @@ internal class WearRealtimeTalkController(
)
suspend fun start(
nodeId: String,
owner: WearRealtimeAttemptOwner,
sessionKey: String,
attemptId: String,
language: String?,
onSessionActivated: () -> Unit = {},
): Boolean =
lifecycleMutex.withLock {
var existingSession = false
val startGeneration =
synchronized(lifecycleStateLock) {
if (!isConnected()) return@withLock false
if (WearRealtimeAttemptKey(nodeId, attemptId) in canceledAttempts) return@withLock false
if (WearRealtimeAttemptKey(owner.nodeId, owner.attemptId) in canceledAttempts) {
return@withLock false
}
if (sessionId != null) {
if (ownerNodeId != nodeId || ownerSessionKey != sessionKey || ownerAttemptId != attemptId) {
if (activeOwner != owner || ownerSessionKey != sessionKey) {
return@withLock false
}
return@withLock true
existingSession = true
return@synchronized lifecycleGeneration.get()
}
ownerNodeId = nodeId
activeOwner = owner
ownerSessionKey = sessionKey
ownerAttemptId = attemptId
val generation = lifecycleGeneration.get()
updateState(
active = true,
@@ -177,20 +187,23 @@ internal class WearRealtimeTalkController(
)
generation
}
if (existingSession) {
onSessionActivated()
return@withLock true
}
fun startIsStale(): Boolean =
startGeneration != lifecycleGeneration.get() ||
!isConnected() ||
ownerNodeId != nodeId ||
ownerSessionKey != sessionKey ||
ownerAttemptId != attemptId
activeOwner != owner ||
ownerSessionKey != sessionKey
val payload =
try {
requestRealtimeSession(sessionKey, language)
} catch (err: Throwable) {
synchronized(lifecycleStateLock) {
if (!startIsStale()) fail(err.message ?: "Unable to start Real-Time Talk")
if (!startIsStale()) fail(err.message ?: "Unable to start Real-Time Talk", expectedOwner = owner)
}
return@withLock false
}
@@ -209,10 +222,10 @@ internal class WearRealtimeTalkController(
val activated =
synchronized(lifecycleStateLock) {
if (startIsStale()) {
if (ownerAttemptId == attemptId) resetLocked()
if (activeOwner == owner) resetLocked()
false
} else if (createdSessionId.isNullOrBlank()) {
fail("Real-Time Talk returned no session")
fail("Real-Time Talk returned no session", expectedOwner = owner)
false
} else {
realtimeAgentCoordinator.beginSession(
@@ -222,8 +235,10 @@ internal class WearRealtimeTalkController(
),
)
sessionId = createdSessionId
startOutputLoop(createdSessionId)
startAppendLoop(createdSessionId)
eventDispatchScope?.cancel()
eventDispatchScope = CoroutineScope(scope.coroutineContext + SupervisorJob(scope.coroutineContext[Job]))
startOutputLoop(owner, createdSessionId)
startAppendLoop(owner, createdSessionId)
updateState(
active = true,
listening = true,
@@ -243,6 +258,7 @@ internal class WearRealtimeTalkController(
}
return@withLock false
}
onSessionActivated()
true
}
@@ -290,24 +306,39 @@ internal class WearRealtimeTalkController(
val closingSession =
synchronized(lifecycleStateLock) {
if (nodeId != null && attemptId != null) rememberCanceledAttemptLocked(nodeId, attemptId)
if (
(nodeId != null && ownerNodeId != null && ownerNodeId != nodeId) ||
(attemptId != null && ownerAttemptId != null && ownerAttemptId != attemptId)
) {
val owner = activeOwner
val identityMatches =
when {
owner != null ->
(nodeId == null || owner.nodeId == nodeId) &&
(attemptId == null || owner.attemptId == attemptId)
nodeId != null && attemptId != null -> true
nodeId == null && attemptId == null -> true
else -> false
}
if (!identityMatches) {
null
} else {
accepted = true
sessionId.also { resetLocked() }
}
}
if (!accepted) {
return@withLock false
}
if (!accepted) return@withLock false
if (!closingSession.isNullOrBlank()) {
runCatching {
val params = buildJsonObject { put("sessionId", JsonPrimitive(closingSession)) }
requestGateway("talk.session.close", params.toString(), 5_000L)
runCatching { closeGatewaySession(closingSession) }
}
true
}
suspend fun stop(owner: WearRealtimeAttemptOwner): Boolean =
lifecycleMutex.withLock {
val closingSession =
synchronized(lifecycleStateLock) {
if (activeOwner != owner) return@withLock false
sessionId.also { resetLocked() }
}
if (!closingSession.isNullOrBlank()) {
scope.launch { runCatching { closeGatewaySession(closingSession) } }
}
true
}
@@ -325,24 +356,40 @@ internal class WearRealtimeTalkController(
}
fun abort() {
val closingNodeId =
val closingOwner =
synchronized(lifecycleStateLock) {
lifecycleGeneration.incrementAndGet()
val nodeId = ownerNodeId
val owner = activeOwner
resetLocked()
nodeId
owner
}
closingNodeId?.let(onForceCloseWatchChannel)
closingOwner?.let(onForceCloseWatchChannel)
}
private fun abort(
expectedOwner: WearRealtimeAttemptOwner?,
expectedSessionId: String?,
) {
val closingOwner =
synchronized(lifecycleStateLock) {
if (expectedOwner != null && activeOwner != expectedOwner) return
if (expectedSessionId != null && sessionId != expectedSessionId) return
lifecycleGeneration.incrementAndGet()
val owner = activeOwner
resetLocked()
owner
}
closingOwner?.let(onForceCloseWatchChannel)
}
fun appendAudio(
nodeId: String,
owner: WearRealtimeAttemptOwner,
payload: ByteArray,
) {
if (
payload.isEmpty() ||
payload.size > WearProtocol.MAX_REALTIME_AUDIO_FRAME_BYTES ||
ownerNodeId != nodeId ||
activeOwner != owner ||
sessionId == null ||
_snapshot.value.speaking
) {
@@ -350,7 +397,11 @@ internal class WearRealtimeTalkController(
}
val activeSessionId = sessionId ?: return
if (audioFrames?.trySend(payload.copyOf())?.isSuccess != true) {
fail("Watch audio input is unavailable", expectedSessionId = activeSessionId)
fail(
"Watch audio input is unavailable",
expectedOwner = owner,
expectedSessionId = activeSessionId,
)
}
}
@@ -371,12 +422,21 @@ internal class WearRealtimeTalkController(
val eventSessionId =
obj["relaySessionId"].asStringOrNull()
?: obj["sessionId"].asStringOrNull()
val currentSessionId = sessionId
if (currentSessionId == null || eventSessionId != currentSessionId) return
// The gateway creates every relaySessionId with randomUUID(); it is the canonical
// correlation token for rejecting events from a retired session.
val (owner, currentSessionId) =
synchronized(lifecycleStateLock) {
val currentSessionId = sessionId
if (currentSessionId == null || eventSessionId != currentSessionId) return
val owner = activeOwner ?: return
owner to currentSessionId
}
when (obj["type"].asStringOrNull()) {
"ready", "inputAudio" ->
updateState(
updateStateIfCurrent(
owner = owner,
sessionId = currentSessionId,
active = true,
listening = true,
speaking = false,
@@ -386,7 +446,11 @@ internal class WearRealtimeTalkController(
"audio" -> {
val encoded = obj["audioBase64"].asStringOrNull() ?: return
if (encoded.length > OUTPUT_QUEUE_BASE64_CHAR_CAPACITY) {
fail("Watch audio output exceeds the relay buffer")
fail(
"Watch audio output exceeds the relay buffer",
expectedOwner = owner,
expectedSessionId = currentSessionId,
)
return
}
val bytes =
@@ -395,13 +459,15 @@ internal class WearRealtimeTalkController(
?.takeIf(ByteArray::isNotEmpty)
?: return
if (bytes.size % PCM_16_BYTES != 0) {
fail("Invalid Watch audio frame")
fail("Invalid Watch audio frame", expectedOwner = owner, expectedSessionId = currentSessionId)
return
}
if (!enqueueOutput(WearRealtimeAudioFrameType.OUTPUT_PCM, bytes)) {
if (!enqueueOutput(owner, currentSessionId, WearRealtimeAudioFrameType.OUTPUT_PCM, bytes)) {
return
}
updateState(
updateStateIfCurrent(
owner = owner,
sessionId = currentSessionId,
active = true,
listening = false,
speaking = true,
@@ -410,53 +476,91 @@ internal class WearRealtimeTalkController(
)
}
"clear" -> {
enqueueOutput(WearRealtimeAudioFrameType.CLEAR_OUTPUT, byteArrayOf())
enqueueOutput(owner, currentSessionId, WearRealtimeAudioFrameType.CLEAR_OUTPUT, byteArrayOf())
}
"mark" -> {
val markName = obj["markName"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: return
scope.launch {
runCatching {
val params =
buildJsonObject {
put("sessionId", JsonPrimitive(currentSessionId))
put("markName", JsonPrimitive(markName))
}
requestGateway("talk.session.acknowledgeMark", params.toString(), 8_000L)
}
}
}
"transcript" -> {
val text = obj["text"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: return
val final = obj["final"].asBooleanOrNull() == true
when (obj["role"].asStringOrNull()) {
"user" -> {
upsertConversation(WearRealtimeTalkRole.USER, text, final)
if (final) {
updateState(
active = true,
listening = false,
speaking = false,
status = WearRealtimeTalkStatus.THINKING,
statusText = "Agent working",
)
}
}
"assistant" -> upsertConversation(WearRealtimeTalkRole.ASSISTANT, text, final)
}
}
"toolCall" -> {
val callId = obj["callId"].asStringOrNull() ?: return
val name = obj["name"].asStringOrNull() ?: return
realtimeAgentCoordinator.handleToolCall(
callId = callId,
name = name,
args = obj["args"],
forced = obj["forced"].asBooleanOrNull() == true,
)
acknowledgeMark(owner, currentSessionId, markName)
}
"transcript" -> handleTranscriptEvent(owner, currentSessionId, obj)
"toolCall" -> handleToolCallEvent(owner, currentSessionId, obj)
"toolResult" -> Unit
"error" -> fail(obj["message"].asStringOrNull() ?: "Real-Time Talk failed")
"close" -> abort()
"error" ->
fail(
obj["message"].asStringOrNull() ?: "Real-Time Talk failed",
expectedOwner = owner,
expectedSessionId = currentSessionId,
)
"close" -> abort(owner, currentSessionId)
}
}
private fun acknowledgeMark(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
markName: String,
) {
val ownerScope =
synchronized(lifecycleStateLock) {
eventDispatchScope.takeIf { isCurrent(owner, activeSessionId) }
} ?: return
ownerScope.launch {
synchronized(lifecycleStateLock) {
if (!isCurrent(owner, activeSessionId)) return@launch
}
runCatching {
val params =
buildJsonObject {
put("sessionId", JsonPrimitive(activeSessionId))
put("markName", JsonPrimitive(markName))
}
requestGateway("talk.session.acknowledgeMark", params.toString(), 8_000L)
}
}
}
private fun handleTranscriptEvent(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
obj: JsonObject,
) {
synchronized(lifecycleStateLock) {
if (!isCurrent(owner, activeSessionId)) return
val text = obj["text"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: return
val final = obj["final"].asBooleanOrNull() == true
when (obj["role"].asStringOrNull()) {
"user" -> {
upsertConversation(WearRealtimeTalkRole.USER, text, final)
if (final) {
updateState(
active = true,
listening = false,
speaking = false,
status = WearRealtimeTalkStatus.THINKING,
statusText = "Agent working",
)
}
}
"assistant" -> upsertConversation(WearRealtimeTalkRole.ASSISTANT, text, final)
}
}
}
private fun handleToolCallEvent(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
obj: JsonObject,
) {
synchronized(lifecycleStateLock) {
if (!isCurrent(owner, activeSessionId)) return
val callId = obj["callId"].asStringOrNull() ?: return
val name = obj["name"].asStringOrNull() ?: return
realtimeAgentCoordinator.handleToolCall(
callId = callId,
name = name,
args = obj["args"],
forced = obj["forced"].asBooleanOrNull() == true,
)
}
}
@@ -472,7 +576,10 @@ internal class WearRealtimeTalkController(
)
}
private fun startOutputLoop(activeSessionId: String) {
private fun startOutputLoop(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
) {
val messages = Channel<WearRealtimeOutputMessage>(capacity = OUTPUT_QUEUE_CAPACITY)
val queue = WearRealtimeOutputQueue(messages)
synchronized(outputQueueLock) { outputQueue.also { outputQueue = queue } }
@@ -484,18 +591,17 @@ internal class WearRealtimeTalkController(
for (message in messages) {
var delivered = false
try {
if (sessionId != activeSessionId) continue
val nodeId = ownerNodeId ?: continue
if (!isCurrent(owner, activeSessionId)) continue
when (message.type) {
WearRealtimeAudioFrameType.OUTPUT_PCM -> {
delivered = true
for (chunk in chunkWearRealtimeOutput(message.payload)) {
if (!isCurrentOutput(activeSessionId, nodeId)) {
if (!isCurrentOutput(owner, activeSessionId)) {
delivered = false
break
}
sendWatchFrame(nodeId, message.type, chunk)
if (!isCurrentOutput(activeSessionId, nodeId)) {
sendWatchFrame(owner, message.type, chunk)
if (!isCurrentOutput(owner, activeSessionId)) {
delivered = false
break
}
@@ -506,13 +612,13 @@ internal class WearRealtimeTalkController(
audioByteCount = chunk.size,
)
}
if (!isCurrentOutput(activeSessionId, nodeId)) {
if (!isCurrentOutput(owner, activeSessionId)) {
delivered = false
}
}
WearRealtimeAudioFrameType.CLEAR_OUTPUT -> {
sendWatchFrame(nodeId, message.type, message.payload)
delivered = isCurrentOutput(activeSessionId, nodeId)
sendWatchFrame(owner, message.type, message.payload)
delivered = isCurrentOutput(owner, activeSessionId)
}
WearRealtimeAudioFrameType.INPUT_PCM -> error("Phone cannot emit Watch input audio")
}
@@ -520,6 +626,7 @@ internal class WearRealtimeTalkController(
if (err is CancellationException) throw err
fail(
"Unable to send audio to Watch",
expectedOwner = owner,
expectedSessionId = activeSessionId,
)
break
@@ -534,12 +641,14 @@ internal class WearRealtimeTalkController(
if (!delivered) continue
when (message.type) {
WearRealtimeAudioFrameType.OUTPUT_PCM -> {
schedulePlaybackIdle()
schedulePlaybackIdle(owner, activeSessionId)
}
WearRealtimeAudioFrameType.CLEAR_OUTPUT -> {
playbackEndsAtMillis = 0L
playbackIdleJob?.cancel()
updateState(
updateStateIfCurrent(
owner = owner,
sessionId = activeSessionId,
active = true,
listening = true,
speaking = false,
@@ -554,19 +663,22 @@ internal class WearRealtimeTalkController(
}
private suspend fun isCurrentOutput(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
nodeId: String,
): Boolean =
currentCoroutineContext().isActive &&
sessionId == activeSessionId &&
ownerNodeId == nodeId
activeOwner == owner
private fun enqueueOutput(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
type: WearRealtimeAudioFrameType,
payload: ByteArray,
): Boolean {
val accepted =
synchronized(outputQueueLock) {
if (!isCurrent(owner, activeSessionId)) return@synchronized false
val queue = outputQueue ?: return@synchronized false
val audioBytes = payload.size.takeIf { type == WearRealtimeAudioFrameType.OUTPUT_PCM } ?: 0
if (audioBytes > OUTPUT_QUEUE_BYTE_CAPACITY - queue.retainedAudioBytes) {
@@ -578,13 +690,20 @@ internal class WearRealtimeTalkController(
}
}
if (!accepted) {
fail("Watch audio link is unavailable")
fail(
"Watch audio link is unavailable",
expectedOwner = owner,
expectedSessionId = activeSessionId,
)
return false
}
return true
}
private fun startAppendLoop(activeSessionId: String) {
private fun startAppendLoop(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
) {
audioFrames?.close()
appendJob?.cancel()
val frames = Channel<ByteArray>(capacity = INPUT_QUEUE_CAPACITY)
@@ -592,7 +711,7 @@ internal class WearRealtimeTalkController(
appendJob =
scope.launch {
for (frame in frames) {
if (sessionId != activeSessionId) continue
if (!isCurrent(owner, activeSessionId)) continue
val params =
buildJsonObject {
put("sessionId", JsonPrimitive(activeSessionId))
@@ -607,11 +726,18 @@ internal class WearRealtimeTalkController(
"talk.session.appendAudio",
params.toString(),
8_000L,
) { message -> fail(message, expectedSessionId = activeSessionId) }
) { message ->
fail(
message,
expectedOwner = owner,
expectedSessionId = activeSessionId,
)
}
} catch (err: Throwable) {
if (err is CancellationException) throw err
fail(
err.message ?: "Unable to send Watch audio",
expectedOwner = owner,
expectedSessionId = activeSessionId,
)
}
@@ -619,15 +745,20 @@ internal class WearRealtimeTalkController(
}
}
private fun schedulePlaybackIdle() {
private fun schedulePlaybackIdle(
owner: WearRealtimeAttemptOwner,
activeSessionId: String,
) {
playbackIdleJob?.cancel()
playbackIdleJob =
scope.launch {
while (SystemClock.elapsedRealtime() < playbackEndsAtMillis) {
delay(20L)
}
if (sessionId != null) {
updateState(
if (isCurrent(owner, activeSessionId)) {
updateStateIfCurrent(
owner = owner,
sessionId = activeSessionId,
active = true,
listening = true,
speaking = false,
@@ -688,24 +819,48 @@ internal class WearRealtimeTalkController(
speaking = speaking,
status = status,
statusText = statusText,
attemptId = ownerAttemptId,
attemptId = activeOwner?.attemptId,
),
)
}
private fun updateStateIfCurrent(
owner: WearRealtimeAttemptOwner,
sessionId: String,
active: Boolean,
listening: Boolean,
speaking: Boolean,
status: WearRealtimeTalkStatus,
statusText: String,
) {
synchronized(lifecycleStateLock) {
if (!isCurrent(owner, sessionId)) return
updateState(active, listening, speaking, status, statusText)
}
}
private fun isCurrent(
owner: WearRealtimeAttemptOwner,
expectedSessionId: String,
): Boolean = activeOwner == owner && sessionId == expectedSessionId
private fun fail(
message: String,
expectedOwner: WearRealtimeAttemptOwner? = null,
expectedSessionId: String? = null,
) {
val (closingSession, closingNodeId) =
var closingSession: String? = null
val closingOwner =
synchronized(lifecycleStateLock) {
// Transport callbacks and non-cancellable I/O can outlive their relay.
// Only that relay may own teardown, or a late error can stop its replacement.
if (expectedOwner != null && activeOwner != expectedOwner) return
if (expectedSessionId != null && sessionId != expectedSessionId) return
Log.w(TAG, message)
val currentSession = sessionId
val currentNodeId = ownerNodeId
realtimeAgentCoordinator.endSession(currentSession)
val currentOwner = activeOwner
closingSession = currentSession
realtimeAgentCoordinator.resetTransport()
setSnapshot(
_snapshot.value.copy(
active = false,
@@ -716,9 +871,8 @@ internal class WearRealtimeTalkController(
),
)
sessionId = null
ownerNodeId = null
activeOwner = null
ownerSessionKey = null
ownerAttemptId = null
audioFrames?.close()
audioFrames = null
appendJob?.cancel()
@@ -728,29 +882,30 @@ internal class WearRealtimeTalkController(
?.close()
outputJob?.cancel()
outputJob = null
eventDispatchScope?.cancel()
eventDispatchScope = null
playbackIdleJob?.cancel()
playbackIdleJob = null
playbackEndsAtMillis = 0L
currentSession to currentNodeId
}
if (!closingSession.isNullOrBlank()) {
scope.launch {
runCatching {
val params = buildJsonObject { put("sessionId", JsonPrimitive(closingSession)) }
requestGateway("talk.session.close", params.toString(), 5_000L)
}
currentOwner
}
closingSession?.takeIf(String::isNotBlank)?.let { session ->
scope.launch { runCatching { closeGatewaySession(session) } }
}
closingNodeId?.let(onForceCloseWatchChannel)
closingOwner?.let(onForceCloseWatchChannel)
}
private suspend fun closeGatewaySession(closingSession: String) {
val params = buildJsonObject { put("sessionId", JsonPrimitive(closingSession)) }
requestGateway("talk.session.close", params.toString(), 5_000L)
}
private fun resetLocked() {
val closingAttemptId = ownerAttemptId
realtimeAgentCoordinator.endSession(sessionId)
val closingAttemptId = activeOwner?.attemptId
realtimeAgentCoordinator.resetTransport()
sessionId = null
ownerNodeId = null
activeOwner = null
ownerSessionKey = null
ownerAttemptId = null
audioFrames?.close()
audioFrames = null
appendJob?.cancel()
@@ -760,6 +915,8 @@ internal class WearRealtimeTalkController(
?.close()
outputJob?.cancel()
outputJob = null
eventDispatchScope?.cancel()
eventDispatchScope = null
playbackIdleJob?.cancel()
playbackIdleJob = null
playbackEndsAtMillis = 0L
@@ -737,6 +737,32 @@ class GatewayBootstrapAuthTest {
assertTrue(locationOptions.commands.contains(OpenClawLocationCommand.Get.rawValue))
}
@Test
fun permissionSurfaceReconnectsOnlyAfterAndroidAuthorityChanges() {
val app: android.app.Application = RuntimeEnvironment.getApplication()
shadowOf(app).denyPermissions(Manifest.permission.CAMERA)
val (runtime, prefs) = createNeutralizedRuntime()
armSavedActiveManualGateway(prefs)
writeField(
runtime,
"connectedEndpoint",
GatewayEndpoint.manual(host = "127.0.0.1", port = 18789),
)
runtime.refreshNodePermissionSurface()
assertNull(desiredConnection(runtime, "nodeSession"))
shadowOf(app).grantPermissions(Manifest.permission.CAMERA)
runtime.refreshNodePermissionSurface()
val options =
readField<GatewayConnectOptions>(
waitForDesiredConnection(runtime, "nodeSession"),
"options",
)
assertTrue(options.permissions.getValue("camera"))
}
@Test
fun connect_showsSecureEndpointGuidanceWhenTlsProbeFails() {
val app = RuntimeEnvironment.getApplication()
@@ -1,16 +1,28 @@
package ai.openclaw.app
import android.content.ContentProvider
import android.content.ContentValues
import android.content.Context
import android.content.Intent
import android.database.Cursor
import android.net.Uri
import android.os.Looper
import androidx.lifecycle.SavedStateHandle
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.Robolectric
import org.robolectric.RobolectricTestRunner
import org.robolectric.RuntimeEnvironment
import org.robolectric.Shadows.shadowOf
import org.robolectric.annotation.Config
import org.robolectric.shadows.ShadowContentResolver
import java.util.UUID
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
@@ -84,6 +96,32 @@ class MainActivityLifecycleTest {
assertEquals(listOf(replacement), routed)
}
@Test
fun pendingIntentRouterRetainsShareOverflowUntilViewModelActivation() {
val router = MainActivityPendingIntentRouter()
val routed = mutableListOf<Intent>()
repeat(MAX_PENDING_CHAT_SHARES) { index ->
val share = Intent(Intent.ACTION_SEND).setType("text/plain").putExtra(Intent.EXTRA_TEXT, "share-$index")
if (index == 0) {
router.setInitialIntent(share)
} else {
assertTrue(router.onNewIntent(share, routed::add))
}
}
assertFalse(
router.onNewIntent(
Intent(Intent.ACTION_SEND).setType("text/plain").putExtra(Intent.EXTRA_TEXT, "overflow"),
routed::add,
),
)
assertTrue(router.activate(routed::add))
assertEquals(MAX_PENDING_CHAT_SHARES, routed.size)
assertEquals(1, router.takeShareOverflowCount())
assertEquals(0, router.takeShareOverflowCount())
}
@Test
fun initialIntentGateDistinguishesRecreationFromProcessRestoration() {
val retainedGate = MainActivityInitialIntentGate()
@@ -93,12 +131,84 @@ class MainActivityLifecycleTest {
assertTrue(MainActivityInitialIntentGate().claim())
}
@Test
fun blockedShareMimeResolutionSurvivesActivityRecreation() {
val app = RuntimeEnvironment.getApplication() as NodeApp
app.chatShareDraftQueue.clear()
val resolverEntered = CountDownLatch(1)
val releaseResolver = CountDownLatch(1)
ShadowContentResolver.registerProviderInternal(
"blocked-share",
BlockingMimeProvider(resolverEntered, releaseResolver),
)
val sharedUri = Uri.parse("content://blocked-share/document")
val shareIntent =
Intent(Intent.ACTION_SEND)
.setType("*/*")
.putExtra(Intent.EXTRA_STREAM, sharedUri)
val controller =
Robolectric
.buildActivity(MainActivity::class.java)
.create()
.start()
.resume()
val activity = controller.get()
val prefs =
SecurePrefs(
app,
securePrefsOverride =
app.getSharedPreferences(
"share-recreation-test-${UUID.randomUUID()}",
Context.MODE_PRIVATE,
),
)
val viewModel = MainViewModel(app, prefs, SavedStateHandle())
val expectedOwner = viewModel.captureChatShareOwner()
assertTrue(viewModel.claimInitialIntentRouting())
val handleLaunchIntent =
MainActivity::class.java
.getDeclaredMethod("handleLaunchIntent", MainViewModel::class.java, Intent::class.java)
.apply { isAccessible = true }
handleLaunchIntent.invoke(activity, viewModel, shareIntent)
assertTrue(resolverEntered.await(5, TimeUnit.SECONDS))
controller.pause().stop().destroy()
assertFalse(viewModel.claimInitialIntentRouting())
releaseResolver.countDown()
assertTrue(waitUntil { app.chatShareDraftQueue.size() == 1 })
val draft = requireNotNull(app.chatShareDraftQueue.head.value)
assertEquals(listOf(sharedUri), draft.attachments.map(SharedAttachment::uri))
assertEquals(expectedOwner, app.chatShareDraftQueue.ownerOf(draft.id))
}
@Test
fun runtimeStaysForegroundAcrossConfigurationRecreation() {
assertFalse(shouldNotifyRuntimeBackgrounded(isChangingConfigurations = true))
assertTrue(shouldNotifyRuntimeBackgrounded(isChangingConfigurations = false))
}
@Test
fun topResumedPermissionHostRefreshesAuthorityAfterActivation() {
val events = mutableListOf<String>()
updateTopResumedPermissionHost(
isTopResumedActivity = true,
activate = { events += "activate" },
deactivate = { events += "deactivate" },
refreshPermissionSurface = { events += "refresh" },
)
updateTopResumedPermissionHost(
isTopResumedActivity = false,
activate = { events += "activate" },
deactivate = { events += "deactivate" },
refreshPermissionSurface = { events += "refresh" },
)
assertEquals(listOf("activate", "refresh", "deactivate"), events)
}
@Test
fun runtimeUiStarterWaitsForReadinessAndStartsOnce() {
val starter = MainActivityRuntimeUiStarter()
@@ -178,4 +288,57 @@ class MainActivityLifecycleTest {
NodeForegroundService.resume(app, startNow = false)
}
}
private fun waitUntil(
timeoutMillis: Long = 2_000,
predicate: () -> Boolean,
): Boolean {
val deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis)
while (System.nanoTime() < deadline) {
shadowOf(Looper.getMainLooper()).idle()
if (predicate()) return true
Thread.sleep(10)
}
shadowOf(Looper.getMainLooper()).idle()
return predicate()
}
private class BlockingMimeProvider(
private val entered: CountDownLatch,
private val release: CountDownLatch,
) : ContentProvider() {
override fun onCreate(): Boolean = true
override fun getType(uri: Uri): String {
entered.countDown()
check(release.await(5, TimeUnit.SECONDS))
return "application/pdf"
}
override fun query(
uri: Uri,
projection: Array<out String>?,
selection: String?,
selectionArgs: Array<out String>?,
sortOrder: String?,
): Cursor? = null
override fun insert(
uri: Uri,
values: ContentValues?,
): Uri? = null
override fun delete(
uri: Uri,
selection: String?,
selectionArgs: Array<out String>?,
): Int = 0
override fun update(
uri: Uri,
values: ContentValues?,
selection: String?,
selectionArgs: Array<out String>?,
): Int = 0
}
}
@@ -7,6 +7,8 @@ import ai.openclaw.app.ui.chat.ChatComposerStateStore
import ai.openclaw.app.ui.chat.PendingAttachment
import android.content.Context
import android.content.Intent
import android.net.Uri
import android.os.Looper
import androidx.lifecycle.SavedStateHandle
import kotlinx.coroutines.runBlocking
import org.junit.After
@@ -22,6 +24,8 @@ import org.robolectric.RuntimeEnvironment
import org.robolectric.Shadows.shadowOf
import org.robolectric.annotation.Config
import java.util.UUID
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
@@ -30,6 +34,7 @@ class MainViewModelTest {
fun resetNodeServiceStartSuppression() {
val app = RuntimeEnvironment.getApplication()
NodeForegroundService.resume(app, startNow = false)
(app as NodeApp).chatShareDraftQueue.clear()
val appShadow = shadowOf(app)
while (appShadow.nextStartedService != null) {
// Drain queued service intents so each test owns its lifecycle assertions.
@@ -231,6 +236,66 @@ class MainViewModelTest {
assertNotNull(state.tryBeginTrackedSend(owner))
}
@Test
fun warmShareIntentsQueueOnceInArrivalOrderWithCapturedOwner() {
val (viewModel, _) = createViewModel(resolveShareMimeType = { "application/pdf" })
val firstUri = Uri.parse("content://share/first")
val secondUri = Uri.parse("content://share/second")
val owner = viewModel.captureChatShareOwner()
assertTrue(viewModel.handleShareLaunchIntent(shareIntent(firstUri, "first")))
assertTrue(viewModel.handleShareLaunchIntent(shareIntent(secondUri, "second")))
assertTrue(waitUntil { viewModel.chatShareDrafts.value.size == 2 })
val drafts = viewModel.chatShareDrafts.value
assertEquals(listOf("first", "second"), drafts.map(ChatShareDraft::text))
assertEquals(listOf(firstUri, secondUri), drafts.map { draft -> draft.attachments.single().uri })
assertTrue(drafts.all { draft -> viewModel.chatShareDraftTargetsOwner(draft.id, owner, owner.sessionKey) })
}
@Test
fun blockedShareReportsRetainedOverflowAndReleasesItsSlot() {
val resolverEntered = CountDownLatch(1)
val releaseResolver = CountDownLatch(1)
val blockedUri = Uri.parse("content://share/blocked")
val nextUri = Uri.parse("content://share/next")
val (viewModel, _) =
createViewModel(
resolveShareMimeType = { uri ->
if (uri == blockedUri) {
resolverEntered.countDown()
check(releaseResolver.await(5, TimeUnit.SECONDS))
}
"application/pdf"
},
shareLaunchCapacity = 1,
)
assertTrue(viewModel.handleShareLaunchIntent(shareIntent(blockedUri, "blocked")))
assertTrue(resolverEntered.await(5, TimeUnit.SECONDS))
assertFalse(viewModel.handleShareLaunchIntent(shareIntent(nextUri, "overflow")))
assertEquals(1L, viewModel.shareLaunchOverflowRevision.value)
releaseResolver.countDown()
assertTrue(waitUntil { viewModel.chatShareDrafts.value.size == 1 })
assertEquals(1, viewModel.takeShareLaunchOverflowCount())
assertEquals(0, viewModel.takeShareLaunchOverflowCount())
viewModel.reportShareLaunchOverflow(2)
viewModel.reportShareLaunchOverflow()
assertEquals(3L, viewModel.shareLaunchOverflowRevision.value)
assertEquals(3, viewModel.takeShareLaunchOverflowCount())
(RuntimeEnvironment.getApplication() as NodeApp).chatShareDraftQueue.clear()
assertTrue(viewModel.handleShareLaunchIntent(shareIntent(nextUri, "next")))
assertTrue(
waitUntil {
viewModel.chatShareDrafts.value
.singleOrNull()
?.text == "next"
},
)
}
@Test
fun gatewayAuthResetCleanupPurgesOnlyThatGatewaysComposerState() =
runBlocking {
@@ -380,7 +445,10 @@ class MainViewModelTest {
assertEquals("ai.openclaw.app.action.RESUME", intent?.action)
}
private fun createViewModel(): Pair<MainViewModel, SecurePrefs> {
private fun createViewModel(
resolveShareMimeType: (Uri) -> String? = { null },
shareLaunchCapacity: Int = MAX_PENDING_CHAT_SHARES,
): Pair<MainViewModel, SecurePrefs> {
val app = RuntimeEnvironment.getApplication() as NodeApp
val prefs =
SecurePrefs(
@@ -391,7 +459,39 @@ class MainViewModelTest {
Context.MODE_PRIVATE,
),
)
return MainViewModel(app, prefs, SavedStateHandle()) to prefs
return (
MainViewModel(
app = app,
prefs = prefs,
savedStateHandle = SavedStateHandle(),
resolveShareMimeType = resolveShareMimeType,
shareLaunchCapacity = shareLaunchCapacity,
) to
prefs
)
}
private fun shareIntent(
uri: Uri,
text: String,
): Intent =
Intent(Intent.ACTION_SEND)
.setType("*/*")
.putExtra(Intent.EXTRA_TEXT, text)
.putExtra(Intent.EXTRA_STREAM, uri)
private fun waitUntil(
timeoutMillis: Long = 2_000,
predicate: () -> Boolean,
): Boolean {
val deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis)
while (System.nanoTime() < deadline) {
shadowOf(Looper.getMainLooper()).idle()
if (predicate()) return true
Thread.sleep(10)
}
shadowOf(Looper.getMainLooper()).idle()
return predicate()
}
private fun draft(name: String): CronEditorDraftState {
@@ -0,0 +1,72 @@
package ai.openclaw.app
import ai.openclaw.app.wear.WearRealtimeAttemptOwner
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitCancellation
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.test.runTest
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Test
class NodeRuntimeWearRealtimeTalkTest {
@Test
fun `replacement during relay creation stops the stale session and rejects start`() =
runTest {
val owner = WearRealtimeAttemptOwner("watch-a", "attempt-a", 1L)
val startEntered = CompletableDeferred<Unit>()
val releaseStart = CompletableDeferred<Unit>()
val stoppedOwners = mutableListOf<WearRealtimeAttemptOwner>()
var currentOwner: WearRealtimeAttemptOwner? = owner
val result =
async {
startWearRealtimeTalkWhileCurrent(
owner = owner,
isCurrent = { candidate -> currentOwner == candidate },
start = { onSessionActivated ->
startEntered.complete(Unit)
releaseStart.await()
onSessionActivated()
true
},
stop = { staleOwner -> stoppedOwners += staleOwner },
)
}
startEntered.await()
currentOwner = WearRealtimeAttemptOwner("watch-a", "attempt-b", 2L)
releaseStart.complete(Unit)
assertFalse(result.await())
assertEquals(listOf(owner), stoppedOwners)
}
@Test
fun `cancellation after relay activation still stops the uncommitted session`() =
runTest {
val owner = WearRealtimeAttemptOwner("watch-a", "attempt-a", 1L)
val relayActivated = CompletableDeferred<Unit>()
val stoppedOwners = mutableListOf<WearRealtimeAttemptOwner>()
val result =
async {
startWearRealtimeTalkWhileCurrent(
owner = owner,
isCurrent = { true },
start = { onSessionActivated ->
onSessionActivated()
relayActivated.complete(Unit)
awaitCancellation()
},
stop = { staleOwner -> stoppedOwners += staleOwner },
)
}
relayActivated.await()
result.cancelAndJoin()
assertEquals(listOf(owner), stoppedOwners)
}
}
@@ -1,6 +1,7 @@
package ai.openclaw.app
import android.Manifest
import android.app.Dialog
import android.content.pm.PackageManager
import androidx.activity.ComponentActivity
import kotlinx.coroutines.Dispatchers
@@ -21,7 +22,9 @@ import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.Robolectric
import org.robolectric.RobolectricTestRunner
import org.robolectric.Shadows.shadowOf
import org.robolectric.annotation.Config
import org.robolectric.shadows.ShadowDialog
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
@@ -32,7 +35,7 @@ class PermissionRequesterTest {
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val requests = FakePermissionRequests()
val requester = PermissionRequester(activity(), requests::request)
val requester = requester(activity(), requests)
try {
val first = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 10) }
@@ -68,7 +71,7 @@ class PermissionRequesterTest {
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val requests = FakePermissionRequests()
val requester = PermissionRequester(activity(), requests::request)
val requester = requester(activity(), requests)
try {
repeat(4) { index ->
@@ -103,7 +106,7 @@ class PermissionRequesterTest {
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val requests = FakePermissionRequests()
val requester = PermissionRequester(activity(), requests::request)
val requester = requester(activity(), requests)
try {
val cancelled = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
@@ -132,7 +135,7 @@ class PermissionRequesterTest {
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val requests = FakePermissionRequests()
val requester = PermissionRequester(activity(), requests::request)
val requester = requester(activity(), requests)
try {
val pending = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
@@ -147,12 +150,218 @@ class PermissionRequesterTest {
)
runCurrent()
cancelDialog(checkNotNull(ShadowDialog.getLatestDialog()))
runCurrent()
assertEquals(mapOf(Manifest.permission.CAMERA to false), pending.await())
} finally {
Dispatchers.resetMain()
}
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun replacementActivityCompletesPendingRequestAndOwnsLaterPrompts() =
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val originalActivity = activity()
val originalRequests = FakePermissionRequests()
val requester = requester(originalActivity, originalRequests)
try {
val pending = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
runCurrent()
assertEquals(1, originalRequests.size)
val replacementActivity = activity()
val replacementRequests = FakePermissionRequests()
requester.attach(replacementActivity, replacementRequests::request)
requester.activate(replacementActivity)
requester.deactivate(originalActivity)
requester.detach(originalActivity)
assertTrue(originalRequests.deliver(requester, 0, mapOf(Manifest.permission.CAMERA to true)))
runCurrent()
assertEquals(mapOf(Manifest.permission.CAMERA to true), pending.await())
val replacementPrompt =
async { requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO), timeoutMs = 1_000) }
runCurrent()
assertEquals(1, replacementRequests.size)
replacementPrompt.cancelAndJoin()
} finally {
Dispatchers.resetMain()
}
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun requestWaitsForReplacementActivityAcrossRecreationGap() =
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val originalActivity = activity()
val originalRequests = FakePermissionRequests()
val requester = requester(originalActivity, originalRequests)
try {
requester.deactivate(originalActivity)
requester.detach(originalActivity)
val pending = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
runCurrent()
assertEquals(0, originalRequests.size)
assertFalse(pending.isCompleted)
val replacementActivity = activity()
val replacementRequests = FakePermissionRequests()
requester.attach(replacementActivity, replacementRequests::request)
requester.activate(replacementActivity)
runCurrent()
assertEquals(1, replacementRequests.size)
assertFalse(pending.isCompleted)
assertTrue(replacementRequests.deliver(requester, 0, mapOf(Manifest.permission.CAMERA to true)))
runCurrent()
assertEquals(mapOf(Manifest.permission.CAMERA to true), pending.await())
} finally {
Dispatchers.resetMain()
}
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun resumedEarlierTaskReclaimsPermissionPromptOwnership() =
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val firstActivity = activity()
val firstRequests = FakePermissionRequests()
val requester = requester(firstActivity, firstRequests)
val secondActivity = activity()
val secondRequests = FakePermissionRequests()
try {
requester.deactivate(firstActivity)
requester.attach(secondActivity, secondRequests::request)
requester.activate(secondActivity)
requester.deactivate(secondActivity)
requester.activate(firstActivity)
val pending = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
runCurrent()
assertEquals(1, firstRequests.size)
assertEquals(0, secondRequests.size)
pending.cancelAndJoin()
} finally {
Dispatchers.resetMain()
}
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun permanentDenialWaitsForReplacementActivity() =
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val originalActivity = activity()
val originalRequests = FakePermissionRequests()
val requester = requester(originalActivity, originalRequests)
try {
val pending = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
runCurrent()
assertTrue(originalRequests.deliver(requester, 0, mapOf(Manifest.permission.CAMERA to false)))
requester.deactivate(originalActivity)
requester.detach(originalActivity)
runCurrent()
assertFalse(pending.isCompleted)
val replacementActivity = activity()
val replacementRequests = FakePermissionRequests()
requester.attach(replacementActivity, replacementRequests::request)
requester.activate(replacementActivity)
runCurrent()
val settingsDialog = checkNotNull(ShadowDialog.getLatestDialog())
assertTrue(settingsDialog.isShowing)
assertFalse(pending.isCompleted)
cancelDialog(settingsDialog)
runCurrent()
assertEquals(mapOf(Manifest.permission.CAMERA to false), pending.await())
} finally {
Dispatchers.resetMain()
}
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun permanentDenialPromptMovesToNewActiveActivity() =
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val originalActivity = activity()
val originalRequests = FakePermissionRequests()
val requester = requester(originalActivity, originalRequests)
try {
val pending = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
runCurrent()
assertTrue(originalRequests.deliver(requester, 0, mapOf(Manifest.permission.CAMERA to false)))
runCurrent()
val originalDialog = checkNotNull(ShadowDialog.getLatestDialog())
assertTrue(originalDialog.isShowing)
assertFalse(pending.isCompleted)
val replacementActivity = activity()
val replacementRequests = FakePermissionRequests()
requester.attach(replacementActivity, replacementRequests::request)
requester.activate(replacementActivity)
runCurrent()
val replacementDialog = checkNotNull(ShadowDialog.getLatestDialog())
assertFalse(originalDialog.isShowing)
assertTrue(replacementDialog !== originalDialog)
assertTrue(replacementDialog.isShowing)
assertFalse(pending.isCompleted)
cancelDialog(replacementDialog)
runCurrent()
assertEquals(mapOf(Manifest.permission.CAMERA to false), pending.await())
} finally {
Dispatchers.resetMain()
}
}
@Test
@OptIn(ExperimentalCoroutinesApi::class)
fun rationaleHostLossRetriesOnReplacementActivity() =
runTest {
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
val rationaleActivity = rationaleActivity()
val rationaleRequests = FakePermissionRequests()
val requester = requester(rationaleActivity, rationaleRequests)
try {
val pending = async { requester.requestIfMissing(listOf(Manifest.permission.CAMERA), timeoutMs = 1_000) }
runCurrent()
assertEquals(0, rationaleRequests.size)
assertFalse(pending.isCompleted)
val replacementActivity = activity()
val replacementRequests = FakePermissionRequests()
requester.attach(replacementActivity, replacementRequests::request)
requester.activate(replacementActivity)
runCurrent()
assertEquals(0, rationaleRequests.size)
assertEquals(1, replacementRequests.size)
assertTrue(replacementRequests.deliver(requester, 0, mapOf(Manifest.permission.CAMERA to true)))
runCurrent()
assertEquals(mapOf(Manifest.permission.CAMERA to true), pending.await())
} finally {
Dispatchers.resetMain()
}
}
@Test
fun requestCodeAllocatorWrapsWithinLegacyRangeAndSkipsLiveCodes() {
val allocator =
@@ -172,6 +381,29 @@ class PermissionRequesterTest {
.buildActivity(ComponentActivity::class.java)
.setup()
.get()
private fun rationaleActivity(): ComponentActivity =
Robolectric
.buildActivity(PermissionRationaleActivity::class.java)
.setup()
.get()
private fun cancelDialog(dialog: Dialog) {
checkNotNull(shadowOf(dialog).onCancelListener).onCancel(dialog)
}
private fun requester(
activity: ComponentActivity,
requests: FakePermissionRequests,
): PermissionRequester =
PermissionRequester(activity.applicationContext).also { requester ->
requester.attach(activity, requests::request)
requester.activate(activity)
}
}
class PermissionRationaleActivity : ComponentActivity() {
override fun shouldShowRequestPermissionRationale(permission: String): Boolean = true
}
private class FakePermissionRequest(
@@ -0,0 +1,150 @@
package ai.openclaw.app.node
import android.Manifest
import android.app.Application
import android.content.pm.PackageManager
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.RuntimeEnvironment
import org.robolectric.Shadows.shadowOf
import org.robolectric.annotation.Config
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
class AndroidPermissionSnapshotTest {
@Test
fun gatewayPermissions_keepIndependentlyGrantableAuthoritySeparate() {
val app = appContext()
shadowOf(app.packageManager).setSystemFeature(PackageManager.FEATURE_TELEPHONY, true)
shadowOf(app).grantPermissions(
Manifest.permission.CAMERA,
Manifest.permission.ACCESS_COARSE_LOCATION,
Manifest.permission.SEND_SMS,
Manifest.permission.READ_CONTACTS,
Manifest.permission.WRITE_CALENDAR,
)
val permissions =
readAndroidPermissionSnapshot(
context = app,
smsEnabled = true,
callLogEnabled = true,
photosEnabled = true,
backgroundLocationEnabled = true,
).gatewayPermissions()
assertTrue(permissions.getValue("camera"))
assertTrue(permissions.getValue("location"))
assertFalse(permissions.getValue("locationPrecise"))
assertFalse(permissions.getValue("locationBackground"))
assertTrue(permissions.getValue("smsSend"))
assertFalse(permissions.getValue("smsRead"))
assertTrue(permissions.getValue("contactsRead"))
assertFalse(permissions.getValue("contactsWrite"))
assertFalse(permissions.getValue("calendarRead"))
assertTrue(permissions.getValue("calendarWrite"))
}
@Test
fun snapshotGatesVariantSpecificPermissionsByAvailableFeature() {
val app = appContext()
shadowOf(app.packageManager).setSystemFeature(PackageManager.FEATURE_TELEPHONY, true)
shadowOf(app).grantPermissions(
Manifest.permission.SEND_SMS,
Manifest.permission.READ_SMS,
Manifest.permission.READ_CALL_LOG,
Manifest.permission.READ_MEDIA_IMAGES,
)
val snapshot =
readAndroidPermissionSnapshot(
context = app,
smsEnabled = false,
callLogEnabled = false,
photosEnabled = false,
backgroundLocationEnabled = false,
)
assertFalse(snapshot.smsSend)
assertFalse(snapshot.smsRead)
assertFalse(snapshot.callLog)
assertFalse(snapshot.photos)
}
@Test
fun backgroundLocationRequiresFeatureAndForegroundAuthority() {
val app = appContext()
shadowOf(app).grantPermissions(Manifest.permission.ACCESS_BACKGROUND_LOCATION)
val withoutForeground =
readAndroidPermissionSnapshot(
context = app,
smsEnabled = false,
callLogEnabled = false,
photosEnabled = false,
backgroundLocationEnabled = true,
)
assertFalse(withoutForeground.locationBackground)
shadowOf(app).grantPermissions(Manifest.permission.ACCESS_COARSE_LOCATION)
val featureDisabled =
readAndroidPermissionSnapshot(
context = app,
smsEnabled = false,
callLogEnabled = false,
photosEnabled = false,
backgroundLocationEnabled = false,
)
assertFalse(featureDisabled.locationBackground)
val available =
readAndroidPermissionSnapshot(
context = app,
smsEnabled = false,
callLogEnabled = false,
photosEnabled = false,
backgroundLocationEnabled = true,
)
assertTrue(available.locationBackground)
}
@Test
fun gatewayPermissionOrderIsStable() {
val permissions =
readAndroidPermissionSnapshot(
context = appContext(),
smsEnabled = false,
callLogEnabled = false,
photosEnabled = false,
backgroundLocationEnabled = false,
).gatewayPermissions()
assertEquals(
listOf(
"camera",
"microphone",
"location",
"locationPrecise",
"locationBackground",
"smsSend",
"smsRead",
"notificationListener",
"notifications",
"photos",
"contactsRead",
"contactsWrite",
"calendarRead",
"calendarWrite",
"callLog",
"motion",
),
permissions.keys.toList(),
)
}
private fun appContext(): Application = RuntimeEnvironment.getApplication()
}
@@ -635,6 +635,21 @@ class ConnectionManagerTest {
assertFalse(options.caps.contains(OpenClawCapability.Motion.rawValue))
}
@Test
fun buildNodeConnectOptions_advertisesCurrentPermissionSnapshot() {
val permissionSnapshot =
emptyPermissionSnapshot().copy(
camera = true,
location = true,
locationPrecise = false,
smsSend = true,
)
val options = newManager(permissionSnapshot = permissionSnapshot).buildNodeConnectOptions()
assertEquals(permissionSnapshot.gatewayPermissions(), options.permissions)
}
private fun newManager(
cameraEnabled: Boolean = false,
locationMode: LocationMode = LocationMode.Off,
@@ -650,6 +665,7 @@ class ConnectionManagerTest {
voiceWakeAvailable: Boolean = true,
mobileUiAvailable: Boolean = false,
inlineWidgetsAvailable: Boolean = true,
permissionSnapshot: AndroidPermissionSnapshot = emptyPermissionSnapshot(),
): ConnectionManager {
val context = RuntimeEnvironment.getApplication()
context
@@ -679,7 +695,28 @@ class ConnectionManagerTest {
voiceWakeAvailable = { voiceWakeAvailable },
mobileUiAvailable = { mobileUiAvailable },
inlineWidgetsAvailable = { inlineWidgetsAvailable },
permissionSnapshot = { permissionSnapshot },
manualTls = { false },
)
}
private fun emptyPermissionSnapshot(): AndroidPermissionSnapshot =
AndroidPermissionSnapshot(
camera = false,
microphone = false,
location = false,
locationPrecise = false,
locationBackground = false,
smsSend = false,
smsRead = false,
notificationListener = false,
notifications = false,
photos = false,
contactsRead = false,
contactsWrite = false,
calendarRead = false,
calendarWrite = false,
callLog = false,
motion = false,
)
}
@@ -3,6 +3,7 @@ package ai.openclaw.app.node
import android.Manifest
import android.app.Application
import android.content.pm.ApplicationInfo
import android.content.pm.PackageManager
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.boolean
@@ -158,6 +159,56 @@ class DeviceHandlerTest {
}
}
@Test
fun handleDevicePermissions_derivesCompositeStatesFromCanonicalSnapshot() {
val app = appContext()
shadowOf(app.packageManager).setSystemFeature(PackageManager.FEATURE_TELEPHONY, true)
val snapshot =
emptyPermissionSnapshot().copy(
smsSend = true,
contactsRead = true,
calendarRead = true,
calendarWrite = true,
)
val handler =
DeviceHandler.forTesting(
appContext = app,
appSource = FakeDeviceAppSource(emptyList()),
smsEnabled = true,
permissionSnapshot = { snapshot },
)
val payload = handler.handleDevicePermissions(null).payloadJson
assertEquals("granted", permissionStatus(payload, "sms"))
assertEquals("denied", permissionStatus(payload, "contacts"))
assertEquals("granted", permissionStatus(payload, "calendar"))
val smsCapabilities =
parsePayload(payload)
.getValue("permissions")
.jsonObject
.getValue("sms")
.jsonObject
.getValue("capabilities")
.jsonObject
assertEquals(
"granted",
smsCapabilities
.getValue("send")
.jsonObject
.getValue("status")
.jsonPrimitive.content,
)
assertEquals(
"denied",
smsCapabilities
.getValue("read")
.jsonObject
.getValue("status")
.jsonPrimitive.content,
)
}
@Test
fun smsTopLevelStatusTreatsSendOnlyPartialGrantAsGranted() {
assertTrue(
@@ -301,6 +352,26 @@ class DeviceHandlerTest {
}
}
private fun emptyPermissionSnapshot(): AndroidPermissionSnapshot =
AndroidPermissionSnapshot(
camera = false,
microphone = false,
location = false,
locationPrecise = false,
locationBackground = false,
smsSend = false,
smsRead = false,
notificationListener = false,
notifications = false,
photos = false,
contactsRead = false,
contactsWrite = false,
calendarRead = false,
calendarWrite = false,
callLog = false,
motion = false,
)
@Test
fun handleDeviceHealth_returnsExpectedShape() {
val handler = DeviceHandler(appContext())
@@ -5,6 +5,7 @@ import ai.openclaw.app.PendingAssistantAutoSend
import ai.openclaw.app.chat.ChatComposerOwner
import ai.openclaw.app.chat.ChatMessageContent
import ai.openclaw.app.chat.SessionBranch
import androidx.compose.ui.unit.dp
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
@@ -12,6 +13,12 @@ import org.junit.Assert.assertTrue
import org.junit.Test
class ChatScreenTest {
@Test
fun jumpToLatestReservesItsTouchTargetBelowMessages() {
assertEquals(0.dp, chatReaderListBottomInset(showJumpToLatest = false))
assertEquals(56.dp, chatReaderListBottomInset(showJumpToLatest = true))
}
@Test
fun branchMessageCountUsesCountNeutralCopy() {
assertEquals("Messages: 1", branchMessageCountText(1))
@@ -406,8 +406,8 @@ class WearProxyControllerTest {
requestGateway = { _, _ -> buildJsonObject {} },
isGatewayConnected = { true },
gatewayStatusText = { "Connected" },
startRealtimeTalk = { nodeId, sessionKey, attemptId, language ->
startArgs = listOf(nodeId, sessionKey, attemptId, language)
startRealtimeTalk = { nodeId, sessionKey, attemptId, language, attemptScopedAudio ->
startArgs = listOf(nodeId, sessionKey, attemptId, language, attemptScopedAudio.toString())
WearRealtimeTalkSnapshot(attemptId = attemptId, active = true)
},
)
@@ -420,13 +420,14 @@ class WearProxyControllerTest {
put("sessionKey", "agent:main:thread-7")
put("attemptId", "attempt-7")
put("language", "DE")
put("attemptScopedAudio", true)
},
),
sourceNodeId = "watch-a",
)
assertTrue(response.ok)
assertEquals(listOf("watch-a", "agent:main:thread-7", "attempt-7", "de"), startArgs)
assertEquals(listOf("watch-a", "agent:main:thread-7", "attempt-7", "de", "true"), startArgs)
assertTrue(
checkNotNull(response.result)
.jsonObject
@@ -446,7 +447,7 @@ class WearProxyControllerTest {
requestGateway = { _, _ -> buildJsonObject {} },
isGatewayConnected = { true },
gatewayStatusText = { "Connected" },
startRealtimeTalk = { _, _, _, _ ->
startRealtimeTalk = { _, _, _, _, _ ->
starts += 1
WearRealtimeTalkSnapshot(active = true)
},
@@ -459,6 +460,37 @@ class WearProxyControllerTest {
assertEquals(0, starts)
}
@Test
fun legacyTalkStartDefaultsToTheFixedAudioChannel() =
runTest {
var attemptScopedAudio: Boolean? = null
val controller =
WearProxyController(
requestGateway = { _, _ -> buildJsonObject {} },
isGatewayConnected = { true },
gatewayStatusText = { "Connected" },
startRealtimeTalk = { _, _, attemptId, _, scoped ->
attemptScopedAudio = scoped
WearRealtimeTalkSnapshot(attemptId = attemptId, active = true)
},
)
val response =
controller.handle(
request(
WearRpcMethod.TalkStart,
buildJsonObject {
put("sessionKey", "agent:main:thread-7")
put("attemptId", "attempt-7")
},
),
sourceNodeId = "watch-a",
)
assertTrue(response.ok)
assertEquals(false, attemptScopedAudio)
}
@Test
fun talkStopBindsTheWatchNodeAndAttempt() =
runTest {
@@ -32,7 +32,19 @@ class WearProxyListenerManifestTest {
assertTrue(
resolvesToBridgeService(
action = ChannelClient.ACTION_CHANNEL_EVENT,
path = WearProtocol.REALTIME_AUDIO_CHANNEL_PATH,
path = WearProtocol.realtimeAudioChannelPath("attempt-7"),
),
)
assertTrue(
resolvesToBridgeService(
action = ChannelClient.ACTION_CHANNEL_EVENT,
path = WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH,
),
)
assertFalse(
resolvesToBridgeService(
action = ChannelClient.ACTION_CHANNEL_EVENT,
path = "${WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH}-invalid",
),
)
}
@@ -13,6 +13,8 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.test.runCurrent
import kotlinx.coroutines.test.runTest
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
@@ -63,6 +65,24 @@ class WearRealtimeTalkControllerTest {
assertEquals(0, gatewayCalls)
}
@Test
fun `partial scoped stop rejects when no active owner can match`() =
runTest {
val controller =
WearRealtimeTalkController(
scope = this,
isConnected = { true },
requestGateway = { _, _, _ -> """{"relaySessionId":"relay-late"}""" },
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> },
)
assertFalse(controller.stop(nodeId = "watch-a"))
assertFalse(controller.stop(attemptId = "attempt-a"))
assertTrue(controller.start("watch-b", "session-b", "attempt-b", "de"))
assertTrue(controller.stop("watch-b", "attempt-b"))
}
@Test
fun `abort during connecting keeps a missing late session off`() =
runTest {
@@ -78,7 +98,7 @@ class WearRealtimeTalkControllerTest {
onSnapshot = { snapshot ->
if (snapshot.status == WearRealtimeTalkStatus.CONNECTING) controller.abort()
},
onForceCloseWatchChannel = { nodeId -> forcedChannelCloses += nodeId },
onForceCloseWatchChannel = { owner -> forcedChannelCloses += owner.nodeId },
)
assertFalse(
@@ -117,7 +137,7 @@ class WearRealtimeTalkControllerTest {
},
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> },
onForceCloseWatchChannel = { nodeId -> forcedChannelCloses += nodeId },
onForceCloseWatchChannel = { owner -> forcedChannelCloses += owner.nodeId },
)
val startResult =
@@ -158,7 +178,7 @@ class WearRealtimeTalkControllerTest {
},
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> },
onForceCloseWatchChannel = { nodeId -> forcedChannelCloses += nodeId },
onForceCloseWatchChannel = { owner -> forcedChannelCloses += owner.nodeId },
)
assertTrue(
@@ -195,16 +215,16 @@ class WearRealtimeTalkControllerTest {
@Test
fun `late append error from a stopped session does not fail its replacement`() =
runTest {
var relaySequence = 0
var staleAppendError: ((String) -> Unit)? = null
var createCount = 0
val controller =
WearRealtimeTalkController(
scope = this,
isConnected = { true },
requestGateway = { method, _, _ ->
if (method == "talk.session.create") {
relaySequence += 1
"""{"relaySessionId":"relay-$relaySequence"}"""
createCount += 1
"""{"relaySessionId":"relay-$createCount"}"""
} else {
"""{"ok":true}"""
}
@@ -232,17 +252,17 @@ class WearRealtimeTalkControllerTest {
@Test
fun `late Watch output error from a stopped session does not fail its replacement`() =
runTest {
var relaySequence = 0
val outputStarted = CompletableDeferred<Unit>()
val releaseOutput = CompletableDeferred<Unit>()
var createCount = 0
val controller =
WearRealtimeTalkController(
scope = this,
isConnected = { true },
requestGateway = { method, _, _ ->
if (method == "talk.session.create") {
relaySequence += 1
"""{"relaySessionId":"relay-$relaySequence"}"""
createCount += 1
"""{"relaySessionId":"relay-$createCount"}"""
} else {
"""{"ok":true}"""
}
@@ -280,6 +300,176 @@ class WearRealtimeTalkControllerTest {
assertTrue(controller.stop("watch-a", "attempt-b"))
}
@Test
fun `stale close callback cannot abort replacement`() =
runTest {
var createCount = 0
val controller =
WearRealtimeTalkController(
scope = this,
isConnected = { true },
requestGateway = { method, _, _ ->
if (method == "talk.session.create") {
createCount += 1
"""{"relaySessionId":"relay-$createCount"}"""
} else {
"""{"ok":true}"""
}
},
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> },
)
val staleOwner = WearRealtimeAttemptOwner("watch-a", "attempt-a", 1L)
val replacementOwner = WearRealtimeAttemptOwner("watch-a", "attempt-b", 2L)
assertTrue(controller.start(staleOwner, "session-a", "de"))
controller.abort()
assertTrue(controller.start(replacementOwner, "session-b", "de"))
WearRealtimeTalkController::class.java
.getDeclaredMethod(
"abort",
WearRealtimeAttemptOwner::class.java,
String::class.java,
).apply { isAccessible = true }
.invoke(controller, staleOwner, "relay-1")
assertEquals(WearRealtimeTalkStatus.LISTENING, controller.snapshot.value.status)
assertEquals("attempt-b", controller.snapshot.value.attemptId)
assertTrue(controller.stop(replacementOwner))
}
@Test
fun `stale relay events cannot mutate or dispatch into replacement`() =
runTest {
val gatewayMethods = mutableListOf<String>()
var createCount = 0
val controller =
WearRealtimeTalkController(
scope = this,
isConnected = { true },
requestGateway = { method, _, _ ->
gatewayMethods += method
if (method == "talk.session.create") {
createCount += 1
"""{"relaySessionId":"relay-$createCount"}"""
} else {
"""{"ok":true}"""
}
},
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> },
)
val staleOwner = WearRealtimeAttemptOwner("watch-a", "attempt-a", 1L)
val replacementOwner = WearRealtimeAttemptOwner("watch-a", "attempt-b", 2L)
assertTrue(controller.start(staleOwner, "session-a", "de"))
controller.handleGatewayEvent(
"talk.event",
"""{"relaySessionId":"relay-1","type":"mark","markName":"stale-mark"}""",
)
controller.abort()
assertTrue(controller.start(replacementOwner, "session-b", "de"))
controller.invokePrivate(
"handleTranscriptEvent",
staleOwner,
"relay-1",
buildJsonObject {
put("role", JsonPrimitive("user"))
put("text", JsonPrimitive("stale transcript"))
put("final", JsonPrimitive(true))
},
)
controller.invokePrivate(
"handleToolCallEvent",
staleOwner,
"relay-1",
buildJsonObject {
put("callId", JsonPrimitive("stale-call"))
put("name", JsonPrimitive("stale-tool"))
},
)
runCurrent()
val snapshot = controller.snapshot.value
assertTrue(snapshot.conversation.isEmpty())
assertEquals(WearRealtimeTalkStatus.LISTENING, snapshot.status)
assertEquals("attempt-b", snapshot.attemptId)
assertFalse("talk.session.acknowledgeMark" in gatewayMethods)
assertFalse("talk.client.toolCall" in gatewayMethods)
assertTrue(controller.stop(replacementOwner))
}
@Test
fun `replacement cancels delayed tool correlation when the session key is reused`() =
runTest {
val gatewayMethods = mutableListOf<String>()
val oldToolStarted = CompletableDeferred<Unit>()
val oldToolResponse = CompletableDeferred<String>()
var createCount = 0
val controller =
WearRealtimeTalkController(
scope = this,
isConnected = { true },
requestGateway = { method, _, _ ->
gatewayMethods += method
when (method) {
"talk.session.create" -> {
createCount += 1
"""{"relaySessionId":"relay-$createCount"}"""
}
"talk.client.toolCall" -> {
oldToolStarted.complete(Unit)
oldToolResponse.await()
}
else -> """{"ok":true}"""
}
},
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> },
)
val staleOwner = WearRealtimeAttemptOwner("watch-a", "attempt-a", 1L)
val replacementOwner = WearRealtimeAttemptOwner("watch-a", "attempt-b", 2L)
assertTrue(controller.start(staleOwner, "session-main", "de"))
controller.handleGatewayEvent(
"talk.event",
"""
{
"relaySessionId":"relay-1",
"type":"toolCall",
"callId":"old-call",
"name":"openclaw_agent_consult"
}
""".trimIndent(),
)
runCurrent()
oldToolStarted.await()
assertTrue(controller.stop(staleOwner))
assertTrue(controller.start(replacementOwner, "session-main", "de"))
oldToolResponse.complete("""{"runId":"old-run"}""")
runCurrent()
controller.handleGatewayEvent(
"chat",
"""
{
"sessionKey":"session-main",
"runId":"old-run",
"state":"final",
"message":{"role":"assistant","content":"stale"}
}
""".trimIndent(),
)
runCurrent()
assertEquals(0, gatewayMethods.count { it == "talk.session.submitToolResult" })
assertEquals(WearRealtimeTalkStatus.LISTENING, controller.snapshot.value.status)
assertEquals("attempt-b", controller.snapshot.value.attemptId)
assertTrue(controller.stop(replacementOwner))
}
@Test
fun `retries without language when an older gateway rejects only that field`() =
runTest {
@@ -729,7 +919,7 @@ class WearRealtimeTalkControllerTest {
releaseOutput.await()
}
},
onForceCloseWatchChannel = { forcedChannelCloses += it },
onForceCloseWatchChannel = { forcedChannelCloses += it.nodeId },
)
assertTrue(controller.start("watch-a", "session-a", "attempt-a", "de"))
val audio = ByteArray(WearProtocol.MAX_REALTIME_AUDIO_FRAME_BYTES * 65)
@@ -769,7 +959,7 @@ class WearRealtimeTalkControllerTest {
},
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> },
onForceCloseWatchChannel = forcedChannelCloses::add,
onForceCloseWatchChannel = { forcedChannelCloses += it.nodeId },
)
assertTrue(controller.start("watch-a", "session-a", "attempt-a", "de"))
@@ -801,7 +991,7 @@ class WearRealtimeTalkControllerTest {
},
sendGatewayFrame = { _, _, _, _ -> },
sendWatchFrame = { _, _, _ -> error("wear link down") },
onForceCloseWatchChannel = { nodeId -> forcedChannelCloses += nodeId },
onForceCloseWatchChannel = { owner -> forcedChannelCloses += owner.nodeId },
)
assertTrue(
controller.start(
@@ -830,3 +1020,46 @@ class WearRealtimeTalkControllerTest {
assertEquals(listOf("watch-a"), forcedChannelCloses)
}
}
private suspend fun WearRealtimeTalkController.start(
nodeId: String,
sessionKey: String,
attemptId: String,
language: String?,
): Boolean =
start(
owner = testWearRealtimeOwner(nodeId, attemptId),
sessionKey = sessionKey,
language = language,
)
private fun WearRealtimeTalkController.appendAudio(
nodeId: String,
payload: ByteArray,
) {
val attemptId = snapshot.value.attemptId ?: return
appendAudio(testWearRealtimeOwner(nodeId, attemptId), payload)
}
private fun testWearRealtimeOwner(
nodeId: String,
attemptId: String,
): WearRealtimeAttemptOwner =
WearRealtimeAttemptOwner(
nodeId = nodeId,
attemptId = attemptId,
channelGeneration = attemptId.hashCode().toLong(),
)
private fun WearRealtimeTalkController.invokePrivate(
name: String,
vararg args: Any,
) {
javaClass.declaredMethods
.single { method ->
method.name == name &&
method.parameterTypes.size == args.size &&
method.parameterTypes.zip(args).all { (type, arg) -> type.isAssignableFrom(arg.javaClass) }
}.apply { isAccessible = true }
.invoke(this, *args)
}
@@ -0,0 +1,91 @@
package ai.openclaw.app
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
class VoiceE2eReceiverTest {
@Test
fun terminalAuthFailureStopsWaiting() {
val problem =
problem(
code = "AUTH_TOKEN_MISSING",
message = "unauthorized: gateway token missing",
pauseReconnect = true,
retryable = false,
)
assertEquals(
"unauthorized: gateway token missing",
voiceE2eTerminalGatewayFailure(problem),
)
}
@Test
fun pairingApprovalKeepsWaiting() {
val problem =
problem(
code = "PAIRING_REQUIRED",
message = "pairing approval required",
pauseReconnect = true,
retryable = true,
)
assertNull(voiceE2eTerminalGatewayFailure(problem))
}
@Test
fun retryableTransportFailureKeepsWaiting() {
val problem =
problem(
code = "UNAVAILABLE",
message = "gateway unavailable",
pauseReconnect = false,
retryable = true,
)
assertNull(voiceE2eTerminalGatewayFailure(problem))
}
@Test
fun timeoutIncludesLatestConnectionDetail() {
assertEquals(
"Gateway connection timed out after 12000 ms: pairing approval required",
voiceE2eGatewayTimeoutMessage(
timeoutMs = 12_000L,
statusText = "Reconnecting...",
problem =
problem(
code = "PAIRING_REQUIRED",
message = "pairing approval required",
pauseReconnect = true,
retryable = true,
),
),
)
assertEquals(
"Gateway connection timed out after 12000 ms: Connecting...",
voiceE2eGatewayTimeoutMessage(
timeoutMs = 12_000L,
statusText = "Connecting...",
problem = null,
),
)
}
private fun problem(
code: String,
message: String,
pauseReconnect: Boolean,
retryable: Boolean,
): GatewayConnectionProblem =
GatewayConnectionProblem(
code = code,
message = message,
reason = null,
requestId = null,
recommendedNextStep = null,
pauseReconnect = pauseReconnect,
retryable = retryable,
)
}
+1
View File
@@ -11,6 +11,7 @@ android {
defaultConfig {
minSdk = 31
targetSdk = 36
missingDimensionStrategy("store", "play")
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments["androidx.benchmark.suppressErrors"] = "DEBUGGABLE,EMULATOR"
}
@@ -39,10 +39,9 @@ class CronJobNavigationTest {
.firstOrNull { it.isClickable },
) { "Cron fixture row must expose a click action" }
assertTrue("Cron fixture row must expose a click action", cronJobRow.isClickable)
assertFalse(device.hasObject(By.text("Inspect scheduled gateway work.")))
assertFalse(device.hasObject(By.text("Run Now")))
cronJobRow.click()
assertNotNull(device.wait(Until.findObject(By.text("Inspect scheduled gateway work.")), waitTimeoutMs))
assertNotNull(findTextAfterScrolling("Run Now"))
assertNotNull(findTextAfterScrolling("Recent Runs"))
assertNotNull(findTextAfterScrolling("Release checklist ready", exact = false))
@@ -29,7 +29,7 @@ Options:
--device <serial> adb device serial
--package <pkg> package name (default: ai.openclaw.app)
--activity <activity> launch activity (default: .MainActivity)
--skip-install skip :app:installDebug
--skip-install skip :app:installPlayDebug
--launch-runs <n> launch-to-connected runs (default: 4)
--screen-loops <n> screen benchmark loops (default: 6)
--chat-loops <n> chat benchmark loops (default: 8)
@@ -145,7 +145,7 @@ trap cleanup EXIT
if [[ "$INSTALL_APP" == "1" ]]; then
(
cd "$ANDROID_DIR"
./gradlew :app:installDebug --console=plain >"$run_dir/install.log" 2>&1
./gradlew :app:installPlayDebug --console=plain >"$run_dir/install.log" 2>&1
)
fi
@@ -99,7 +99,7 @@ symbols_csv="$tmp_dir/symbols.csv"
children_txt="$tmp_dir/children.txt"
cd "$ANDROID_DIR"
./gradlew :app:installDebug --console=plain >"$tmp_dir/install.log" 2>&1
./gradlew :app:installPlayDebug --console=plain >"$tmp_dir/install.log" 2>&1
if ! uv run --no-project python3 "$app_profiler" \
-p "$PACKAGE" \
@@ -12,13 +12,15 @@ import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.intOrNull
import kotlinx.serialization.json.jsonObject
import java.nio.charset.CharacterCodingException
import java.security.MessageDigest
object WearProtocol {
const val VERSION = 1
const val REQUEST_PATH = "/openclaw/wear/v1/request"
const val RESPONSE_PATH = "/openclaw/wear/v1/response"
const val EVENT_PATH = "/openclaw/wear/v1/event"
const val REALTIME_AUDIO_CHANNEL_PATH = "/openclaw/wear/v1/realtime/audio"
const val LEGACY_REALTIME_AUDIO_CHANNEL_PATH = "/openclaw/wear/v1/realtime/audio"
const val REALTIME_AUDIO_CHANNEL_PATH_PREFIX = "/openclaw/wear/v1/realtime/audio/"
const val PHONE_CAPABILITY = "openclaw_phone_proxy_v1"
const val WATCH_CAPABILITY = "openclaw_wear_companion_v1"
@@ -28,9 +30,42 @@ object WearProtocol {
const val MAX_REALTIME_AUDIO_FRAME_BYTES = 8 * 1024
const val REALTIME_AUDIO_SAMPLE_RATE_HZ = 24_000
const val REALTIME_AUDIO_FRAME_MILLIS = 20
const val RPC_REQUEST_TIMEOUT_MILLIS = 10_000L
// The Watch opens the audio channel before sending talk.start. Keep the
// pending phone-side channel through that RPC deadline plus setup margin.
const val REALTIME_AUDIO_PENDING_CHANNEL_TIMEOUT_MILLIS = RPC_REQUEST_TIMEOUT_MILLIS + 5_000L
// Bound recursive JSON parsing at the untrusted Data Layer boundary.
const val MAX_JSON_DEPTH = 32
fun realtimeAudioChannelPath(attemptId: String): String {
require(attemptId.isNotBlank())
val digest = MessageDigest.getInstance("SHA-256").digest(attemptId.encodeToByteArray())
return buildString(REALTIME_AUDIO_CHANNEL_PATH_PREFIX.length + digest.size * 2) {
append(REALTIME_AUDIO_CHANNEL_PATH_PREFIX)
digest.forEach { byte ->
val value = byte.toInt() and 0xff
append(LOWER_HEX[value ushr 4])
append(LOWER_HEX[value and 0x0f])
}
}
}
fun isRealtimeAudioChannelPath(path: String): Boolean {
if (path == LEGACY_REALTIME_AUDIO_CHANNEL_PATH) return true
return isAttemptScopedRealtimeAudioChannelPath(path)
}
fun isAttemptScopedRealtimeAudioChannelPath(path: String): Boolean {
if (!path.startsWith(REALTIME_AUDIO_CHANNEL_PATH_PREFIX)) return false
val token = path.substring(REALTIME_AUDIO_CHANNEL_PATH_PREFIX.length)
return token.length == REALTIME_AUDIO_ATTEMPT_TOKEN_CHARS &&
token.all { char -> char in '0'..'9' || char in 'a'..'f' }
}
private const val REALTIME_AUDIO_ATTEMPT_TOKEN_CHARS = 64
private const val LOWER_HEX = "0123456789abcdef"
}
enum class WearProxyCapability(
@@ -40,6 +75,7 @@ enum class WearProxyCapability(
GatewayControls(wireValue = "gateway-controls"),
ModelControls(wireValue = "model-controls"),
SessionSelectionLookup(wireValue = "session-selection-lookup"),
AttemptScopedRealtimeAudio(wireValue = "attempt-scoped-realtime-audio"),
;
companion object {
@@ -10,7 +10,9 @@ import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertThrows
import org.junit.Assert.assertTrue
import org.junit.Test
class WearProtocolTest {
@@ -103,7 +105,31 @@ class WearProtocolTest {
assertEquals("/openclaw/wear/v1/request", WearProtocol.REQUEST_PATH)
assertEquals("/openclaw/wear/v1/response", WearProtocol.RESPONSE_PATH)
assertEquals("/openclaw/wear/v1/event", WearProtocol.EVENT_PATH)
assertEquals("/openclaw/wear/v1/realtime/audio", WearProtocol.REALTIME_AUDIO_CHANNEL_PATH)
assertEquals(10_000L, WearProtocol.RPC_REQUEST_TIMEOUT_MILLIS)
assertEquals(15_000L, WearProtocol.REALTIME_AUDIO_PENDING_CHANNEL_TIMEOUT_MILLIS)
assertTrue(
WearProtocol.REALTIME_AUDIO_PENDING_CHANNEL_TIMEOUT_MILLIS >
WearProtocol.RPC_REQUEST_TIMEOUT_MILLIS,
)
val realtimePath = WearProtocol.realtimeAudioChannelPath("attempt-7")
assertEquals(
"/openclaw/wear/v1/realtime/audio",
WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH,
)
assertEquals(
"/openclaw/wear/v1/realtime/audio/9804dc90c374fd8e83c9b95a75611f9bec6e0c6ecdcbed5319d6491208417521",
realtimePath,
)
assertEquals(realtimePath, WearProtocol.realtimeAudioChannelPath("attempt-7"))
assertTrue(WearProtocol.isRealtimeAudioChannelPath(realtimePath))
assertTrue(WearProtocol.isAttemptScopedRealtimeAudioChannelPath(realtimePath))
assertTrue(WearProtocol.isRealtimeAudioChannelPath(WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH))
assertFalse(
WearProtocol.isAttemptScopedRealtimeAudioChannelPath(
WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH,
),
)
assertFalse(WearProtocol.isRealtimeAudioChannelPath("$realtimePath/extra"))
assertEquals("openclaw_phone_proxy_v1", WearProtocol.PHONE_CAPABILITY)
assertEquals("openclaw_wear_companion_v1", WearProtocol.WATCH_CAPABILITY)
assertEquals("gateway_offline", WearConnectionFailure.GatewayOffline.wireValue)
@@ -117,6 +143,10 @@ class WearProtocolTest {
assertEquals("gateway-controls", WearProxyCapability.GatewayControls.wireValue)
assertEquals("model-controls", WearProxyCapability.ModelControls.wireValue)
assertEquals("session-selection-lookup", WearProxyCapability.SessionSelectionLookup.wireValue)
assertEquals(
"attempt-scoped-realtime-audio",
WearProxyCapability.AttemptScopedRealtimeAudio.wireValue,
)
assertEquals(WearProxyCapability.AgentControls, WearProxyCapability.fromWireValue("agent-controls"))
assertEquals(null, WearProxyCapability.fromWireValue("future-capability"))
}
@@ -377,6 +377,7 @@ internal class WearGatewayRepository(
attemptId: String,
language: String?,
phoneNodeId: String,
attemptScopedAudio: Boolean,
): WearRealtimeTalkSnapshot {
val response =
requester.request(
@@ -385,6 +386,7 @@ internal class WearGatewayRepository(
put("sessionKey", sessionKey)
put("attemptId", attemptId)
language?.let { put("language", it) }
if (attemptScopedAudio) put("attemptScopedAudio", true)
},
phoneNodeId,
requirePreferredNode = true,
@@ -98,7 +98,7 @@ internal class WearProxyClient private constructor(
): WearRpcResult {
var attemptedPreferredPhone: PreferredPhoneRegistration? = null
val result =
withTimeoutOrNull(REQUEST_TIMEOUT_MS) {
withTimeoutOrNull(WearProtocol.RPC_REQUEST_TIMEOUT_MILLIS) {
requestBeforeDeadline(method, params, expectedNodeId, requirePreferredNode) { registration ->
attemptedPreferredPhone = registration
}
@@ -337,7 +337,6 @@ internal class WearProxyClient private constructor(
)
companion object {
private const val REQUEST_TIMEOUT_MS = 10_000L
private const val MAX_BUFFERED_EVENTS = 64
fun create(context: Context): WearProxyClient {
@@ -1,6 +1,7 @@
package ai.openclaw.wear
import ai.openclaw.wear.shared.WearProtocol
import ai.openclaw.wear.shared.WearProxyCapability
import ai.openclaw.wear.shared.WearRealtimeAudioFrameType
import ai.openclaw.wear.shared.WearRealtimeAudioFraming
import ai.openclaw.wear.shared.WearRealtimeTalkSnapshot
@@ -16,6 +17,7 @@ import com.google.android.gms.wearable.ChannelClient
import com.google.android.gms.wearable.Wearable
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.NonCancellable
@@ -37,6 +39,7 @@ import kotlinx.coroutines.yield
import java.io.InputStream
import java.io.OutputStream
import java.util.Locale
import java.util.concurrent.atomic.AtomicLong
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
import kotlin.math.ceil
@@ -51,7 +54,10 @@ internal class WearRealtimeTalkClient(
private val lifecycleLock = Mutex()
private val channelLock = Mutex()
private val audioLock = Any()
private val audioFocus = WearAudioFocusController(context) { scope.launch { clearOutput(resumeCapture = true) } }
private val audioFocus =
WearAudioFocusController(context) {
activeAttempt?.let { attempt -> scope.launch { clearOutput(attempt, resumeCapture = true) } }
}
private val _isCapturing = MutableStateFlow(false)
val isCapturing: StateFlow<Boolean> = _isCapturing
private val _isPlaying = MutableStateFlow(false)
@@ -61,14 +67,11 @@ internal class WearRealtimeTalkClient(
private val _channelFailed = MutableStateFlow(false)
val channelFailed: StateFlow<Boolean> = _channelFailed
@Volatile private var activeNodeId: String? = null
private val attemptGeneration = AtomicLong()
@Volatile private var activeAttemptId: String? = null
@Volatile private var activeAttempt: ActiveAttempt? = null
@Volatile private var audioRecord: AudioRecord? = null
private var channel: ChannelClient.Channel? = null
private var channelInput: InputStream? = null
private var channelOutput: OutputStream? = null
private var captureJob: Job? = null
private var readJob: Job? = null
private var playbackIdleJob: Job? = null
@@ -78,22 +81,32 @@ internal class WearRealtimeTalkClient(
private var audioTrack: AudioTrack? = null
private var playbackEndsAtMillis = 0L
private data class ChannelResources(
internal data class ChannelResources(
val channel: ChannelClient.Channel,
val input: InputStream,
val output: OutputStream,
)
internal data class ActiveAttempt(
val nodeId: String,
val attemptId: String,
val generation: Long,
val resources: ChannelResources,
)
suspend fun start(
session: WearSession,
attemptId: String,
capabilities: Set<WearProxyCapability>,
): WearRealtimeTalkSnapshot =
lifecycleLock.withLock {
_channelFailed.value = false
val nodeId = session.phoneNodeId
val attemptScopedAudio = WearProxyCapability.AttemptScopedRealtimeAudio in capabilities
var resources: ChannelResources? = null
var channelOpened = false
var activatedAttempt: ActiveAttempt? = null
try {
openChannel(nodeId)
resources = openChannel(nodeId, attemptId, attemptScopedAudio)
channelOpened = true
val language =
Locale
@@ -101,14 +114,30 @@ internal class WearRealtimeTalkClient(
.language
.lowercase(Locale.ROOT)
.takeIf { value -> value.length == ISO_639_1_LANGUAGE_LENGTH }
val snapshot = repository.startRealtimeTalk(session.key, attemptId, language, nodeId)
activeNodeId = nodeId
activeAttemptId = attemptId
startReader(nodeId, attemptId)
startCapture(nodeId)
val snapshot =
repository.startRealtimeTalk(
sessionKey = session.key,
attemptId = attemptId,
language = language,
phoneNodeId = nodeId,
attemptScopedAudio = attemptScopedAudio,
)
val attempt =
ActiveAttempt(
nodeId = nodeId,
attemptId = attemptId,
generation = attemptGeneration.incrementAndGet(),
resources = checkNotNull(resources),
)
activate(attempt)
activatedAttempt = attempt
resources = null
startReader(attempt)
startCapture(attempt)
snapshot
} catch (err: Throwable) {
closeLocal()
closeChannel(resources)
activatedAttempt?.let(::closeLocal)
if (channelOpened) {
// Finish ambiguous-start cleanup before another attempt can acquire
// the lifecycle lock and create a replacement relay for this Watch.
@@ -120,16 +149,15 @@ internal class WearRealtimeTalkClient(
suspend fun stop(): WearRealtimeTalkSnapshot =
lifecycleLock.withLock {
val nodeId = activeNodeId
val attemptId = activeAttemptId
val attempt = activeAttempt
try {
if (nodeId == null || attemptId == null) {
if (attempt == null) {
WearRealtimeTalkSnapshot()
} else {
repository.stopRealtimeTalk(nodeId, attemptId)
repository.stopRealtimeTalk(attempt.nodeId, attempt.attemptId)
}
} finally {
closeLocal()
closeLocal(attempt)
}
}
@@ -142,65 +170,79 @@ internal class WearRealtimeTalkClient(
closeLocal()
}
private suspend fun openChannel(nodeId: String) {
private suspend fun openChannel(
nodeId: String,
attemptId: String,
attemptScopedAudio: Boolean,
): ChannelResources {
var lastError: Throwable? = null
repeat(CHANNEL_OPEN_ATTEMPTS) { attempt ->
var opened: ChannelClient.Channel? = null
var input: InputStream? = null
var output: OutputStream? = null
try {
opened = channelClient.openChannel(nodeId, WearProtocol.REALTIME_AUDIO_CHANNEL_PATH).awaitRealtimeTask()
opened =
channelClient
.openChannel(nodeId, wearRealtimeAudioChannelPath(attemptId, attemptScopedAudio))
.awaitRealtimeTask()
input = channelClient.getInputStream(opened).awaitRealtimeTask()
output = channelClient.getOutputStream(opened).awaitRealtimeTask()
installChannel(opened, input, output)
return
return ChannelResources(opened, input, output)
} catch (err: Throwable) {
withContext(NonCancellable) {
input.closeQuietly()
output.closeQuietly()
opened?.let { channel -> runCatching { channelClient.close(channel).awaitRealtimeTask() } }
}
if (err is CancellationException) throw err
lastError = err
input.closeQuietly()
output.closeQuietly()
opened?.let { channel -> runCatching { channelClient.close(channel).awaitRealtimeTask() } }
if (attempt + 1 < CHANNEL_OPEN_ATTEMPTS) delay(CHANNEL_RETRY_DELAY_MILLIS)
}
}
throw WearProxyException("phone_unavailable", lastError?.message ?: "Unable to open Watch audio channel")
}
private fun startReader(
nodeId: String,
attemptId: String,
) {
val input = checkNotNull(channelInput)
readJob?.cancel()
readJob =
scope.launch {
private fun startReader(attempt: ActiveAttempt) {
val reader =
scope.launch(start = CoroutineStart.LAZY) {
try {
while (activeNodeId == nodeId && activeAttemptId == attemptId) {
val frame = WearRealtimeAudioFraming.read(input) ?: break
if (activeNodeId != nodeId || activeAttemptId != attemptId) break
while (isCurrent(attempt)) {
val frame = WearRealtimeAudioFraming.read(attempt.resources.input) ?: break
if (!isCurrent(attempt)) break
when (frame.type) {
WearRealtimeAudioFrameType.OUTPUT_PCM -> writeOutput(frame.payload)
WearRealtimeAudioFrameType.CLEAR_OUTPUT -> clearOutput(resumeCapture = true)
WearRealtimeAudioFrameType.OUTPUT_PCM -> writeOutput(attempt, frame.payload)
WearRealtimeAudioFrameType.CLEAR_OUTPUT -> clearOutput(attempt, resumeCapture = true)
WearRealtimeAudioFrameType.INPUT_PCM -> error("Phone sent an invalid Watch audio frame")
}
}
if (activeNodeId == nodeId) handleChannelFailure(nodeId)
handleChannelFailure(attempt)
} catch (err: CancellationException) {
throw err
} catch (_: Throwable) {
if (activeNodeId == nodeId) handleChannelFailure(nodeId)
handleChannelFailure(attempt)
}
}
val installed =
synchronized(audioLock) {
if (!isCurrent(attempt)) {
false
} else {
readJob?.cancel()
readJob = reader
true
}
}
if (installed) reader.start() else reader.cancel()
}
@SuppressLint("MissingPermission")
private fun startCapture(nodeId: String) {
synchronized(audioLock) { startCaptureLocked(nodeId) }
private fun startCapture(attempt: ActiveAttempt) {
synchronized(audioLock) { startCaptureLocked(attempt) }
}
@SuppressLint("MissingPermission")
private fun startCaptureLocked(nodeId: String) {
if (_isCapturing.value || _isPlaying.value || activeNodeId != nodeId) return
private fun startCaptureLocked(attempt: ActiveAttempt) {
if (_isCapturing.value || _isPlaying.value || !isCurrent(attempt)) return
val frameBytes =
WearProtocol.REALTIME_AUDIO_SAMPLE_RATE_HZ * PCM_16_BYTES *
WearProtocol.REALTIME_AUDIO_FRAME_MILLIS / 1_000
@@ -236,7 +278,7 @@ internal class WearRealtimeTalkClient(
currentCoroutineContext().isActive &&
_isCapturing.value &&
audioRecord === recorder &&
activeNodeId == nodeId
isCurrent(attempt)
) {
val read = recorder.read(buffer, 0, buffer.size)
val evenBytes = read - (read and 1)
@@ -246,12 +288,12 @@ internal class WearRealtimeTalkClient(
yield()
continue
}
sendInputFrame(buffer.copyOf(evenBytes))
sendInputFrame(attempt, buffer.copyOf(evenBytes))
}
} catch (err: CancellationException) {
throw err
} catch (_: Throwable) {
if (activeNodeId == nodeId) handleChannelFailure(nodeId)
handleChannelFailure(attempt)
} finally {
runCatching { recorder.stop() }
runCatching { recorder.release() }
@@ -260,18 +302,24 @@ internal class WearRealtimeTalkClient(
}
}
private suspend fun sendInputFrame(payload: ByteArray) {
private suspend fun sendInputFrame(
attempt: ActiveAttempt,
payload: ByteArray,
) {
channelLock.withLock {
val output = channelOutput ?: error("Wear audio channel is closed")
if (!isCurrent(attempt)) return
withContext(Dispatchers.IO) {
WearRealtimeAudioFraming.write(output, WearRealtimeAudioFrameType.INPUT_PCM, payload)
WearRealtimeAudioFraming.write(attempt.resources.output, WearRealtimeAudioFrameType.INPUT_PCM, payload)
}
}
}
private fun writeOutput(bytes: ByteArray) {
private fun writeOutput(
attempt: ActiveAttempt,
bytes: ByteArray,
) {
synchronized(audioLock) {
if (activeNodeId == null) return
if (!isCurrent(attempt)) return
if (!_isPlaying.value) {
pauseCaptureLocked()
check(audioFocus.request())
@@ -296,7 +344,7 @@ internal class WearRealtimeTalkClient(
.toLong()
.coerceAtLeast(1L)
playbackEndsAtMillis = maxOf(SystemClock.elapsedRealtime(), playbackEndsAtMillis) + durationMillis
schedulePlaybackIdle()
schedulePlaybackIdle(attempt)
}
}
@@ -346,7 +394,7 @@ internal class WearRealtimeTalkClient(
.also { check(it.state == AudioTrack.STATE_INITIALIZED) }
}
private fun schedulePlaybackIdle() {
private fun schedulePlaybackIdle(attempt: ActiveAttempt) {
playbackIdleJob?.cancel()
val scheduledPlaybackEndMillis = playbackEndsAtMillis
val finalFrameDurationMillis = mouthLevelAccumulator.pendingFrameDurationMillis()
@@ -358,7 +406,7 @@ internal class WearRealtimeTalkClient(
// discards it only when the matching AudioTrack tail is also discarded.
while (SystemClock.elapsedRealtime() < finalFrameStartsAtMillis) delay(MOUTH_FRAME_MILLIS.toLong())
synchronized(audioLock) {
if (playbackEndsAtMillis == scheduledPlaybackEndMillis) {
if (isCurrent(attempt) && playbackEndsAtMillis == scheduledPlaybackEndMillis) {
mouthLevelAccumulator.flush().forEach { level -> mouthTimelineLocked().trySend(level) }
}
}
@@ -367,20 +415,29 @@ internal class WearRealtimeTalkClient(
delay(PLAYBACK_DRAIN_GRACE_MILLIS)
synchronized(audioLock) {
if (
isCurrent(attempt) &&
playbackEndsAtMillis == scheduledPlaybackEndMillis &&
SystemClock.elapsedRealtime() >= scheduledPlaybackEndMillis
) {
clearOutputLocked(resumeCapture = true)
clearOutputLocked(attempt, resumeCapture = true)
}
}
}
}
private fun clearOutput(resumeCapture: Boolean) {
synchronized(audioLock) { clearOutputLocked(resumeCapture) }
private fun clearOutput(
attempt: ActiveAttempt,
resumeCapture: Boolean,
) {
synchronized(audioLock) {
if (isCurrent(attempt)) clearOutputLocked(attempt, resumeCapture)
}
}
private fun clearOutputLocked(resumeCapture: Boolean) {
private fun clearOutputLocked(
attempt: ActiveAttempt?,
resumeCapture: Boolean,
) {
playbackIdleJob?.cancel()
playbackIdleJob = null
playbackEndsAtMillis = 0L
@@ -400,7 +457,9 @@ internal class WearRealtimeTalkClient(
audioTrack = null
_isPlaying.value = false
audioFocus.abandon()
if (resumeCapture) activeNodeId?.let { nodeId -> runCatching { startCaptureLocked(nodeId) } }
if (resumeCapture && attempt != null && isCurrent(attempt)) {
runCatching { startCaptureLocked(attempt) }
}
}
private fun pauseCaptureLocked() {
@@ -413,56 +472,37 @@ internal class WearRealtimeTalkClient(
runCatching { recorder?.release() }
}
private fun handleChannelFailure(nodeId: String) {
val attemptId = activeAttemptId ?: return
_channelFailed.value = true
closeLocal()
scope.launch { runCatching { repository.stopRealtimeTalk(nodeId, attemptId) } }
private fun handleChannelFailure(attempt: ActiveAttempt) {
if (!closeLocal(attempt, failed = true)) return
scope.launch { runCatching { repository.stopRealtimeTalk(attempt.nodeId, attempt.attemptId) } }
}
private fun closeLocal() {
activeNodeId = null
activeAttemptId = null
private fun activate(attempt: ActiveAttempt) {
synchronized(audioLock) {
pauseCaptureLocked()
clearOutputLocked(resumeCapture = false)
check(activeAttempt == null)
_channelFailed.value = false
activeAttempt = attempt
}
readJob?.cancel()
readJob = null
val resources = detachChannel()
scope.launch { closeChannel(resources) }
}
private suspend fun closeChannel() {
closeChannel(detachChannel())
}
@Synchronized
private fun installChannel(
opened: ChannelClient.Channel,
input: InputStream,
output: OutputStream,
) {
check(channel == null)
channel = opened
channelInput = input
channelOutput = output
}
@Synchronized
private fun detachChannel(): ChannelResources? {
val current = channel ?: return null
val input = channelInput
val output = channelOutput
channel = null
channelInput = null
channelOutput = null
if (input == null || output == null) {
input.closeQuietly()
output.closeQuietly()
return null
}
return ChannelResources(current, input, output)
private fun closeLocal(
expected: ActiveAttempt? = activeAttempt,
failed: Boolean = false,
): Boolean {
val attempt =
synchronized(audioLock) {
val current = activeAttempt ?: return false
if (expected != null && current.generation != expected.generation) return false
activeAttempt = null
if (failed) _channelFailed.value = true
readJob?.cancel()
readJob = null
pauseCaptureLocked()
clearOutputLocked(attempt = null, resumeCapture = false)
current
}
scope.launch { closeChannel(attempt.resources) }
return true
}
private suspend fun closeChannel(resources: ChannelResources?) {
@@ -472,6 +512,8 @@ internal class WearRealtimeTalkClient(
runCatching { channelClient.close(resources.channel).awaitRealtimeTask() }
}
private fun isCurrent(attempt: ActiveAttempt): Boolean = activeAttempt?.generation == attempt.generation
private companion object {
const val CHANNEL_OPEN_ATTEMPTS = 2
const val CHANNEL_RETRY_DELAY_MILLIS = 250L
@@ -482,6 +524,17 @@ internal class WearRealtimeTalkClient(
}
}
internal fun wearRealtimeAudioChannelPath(
attemptId: String,
attemptScopedAudio: Boolean,
): String =
if (attemptScopedAudio) {
WearProtocol.realtimeAudioChannelPath(attemptId)
} else {
// v2026.7.2 shipped the fixed path. Keep it for staggered phone/Watch updates.
WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH
}
internal fun pcm16LeMouthLevels(
pcm: ByteArray,
sampleRateHz: Int = WearProtocol.REALTIME_AUDIO_SAMPLE_RATE_HZ,
@@ -184,6 +184,7 @@ internal class WearViewModel(
private val resyncEventBuffer = WearEventResyncBuffer()
private val historyLoadTracker = WearHistoryLoadTracker()
private val sendAttemptTracker = WearSendAttemptTracker()
private val controlBusyOwner = WearControlBusyOwner()
private var loadJob: Job? = null
private var phoneRouteGeneration = 0L
@@ -290,15 +291,22 @@ internal class WearViewModel(
}
fun startRealtimeTalk() {
val selectedSession = mutableState.value.selectedSession ?: return
if (mutableState.value.talkBusy || mutableState.value.realtimeTalk.active) return
val current = mutableState.value
val selectedSession = current.selectedSession ?: return
if (current.talkBusy || current.realtimeTalk.active) return
val capabilities = current.proxyCapabilities
val attemptId = "wear-${UUID.randomUUID()}"
talkAttemptId = attemptId
val startJob =
viewModelScope.launch(start = CoroutineStart.LAZY) {
mutableState.update { it.copy(talkBusy = true, failure = null) }
try {
val snapshot = realtimeTalkClient.start(selectedSession, attemptId)
val snapshot =
realtimeTalkClient.start(
selectedSession,
attemptId,
capabilities,
)
if (talkAttemptId != attemptId) return@launch
mutableState.update { it.copy(realtimeTalk = snapshot, talkBusy = false) }
} catch (err: CancellationException) {
@@ -397,6 +405,7 @@ internal class WearViewModel(
fun selectAgent(agentId: String) {
val current = mutableState.value
val phoneNodeId = current.phoneNodeId ?: return
val routeGeneration = phoneRouteGeneration
if (
current.controlBusy ||
current.talkBusy ||
@@ -408,18 +417,26 @@ internal class WearViewModel(
) {
return
}
val controlAction = beginControlAction(phoneNodeId, routeGeneration) ?: return
viewModelScope.launch {
mutableState.update { it.copy(controlBusy = true, failure = null) }
try {
if (!isCurrentControlRoute(phoneNodeId, routeGeneration)) return@launch
repository.selectAgent(agentId, phoneNodeId, current.proxyCapabilities)
mutableState.update { it.switchAgentContext(agentId) }
if (!isCurrentControlRoute(phoneNodeId, routeGeneration)) return@launch
mutableState.update { state ->
if (isCurrentControlRoute(phoneNodeId, routeGeneration, state)) {
state.switchAgentContext(agentId)
} else {
state
}
}
refresh()
} catch (err: CancellationException) {
throw err
} catch (err: Throwable) {
recordFailure(err, loading = false)
recordFailureForControlRoute(err, phoneNodeId, routeGeneration, loading = false)
} finally {
mutableState.update { it.copy(controlBusy = false) }
finishControlAction(controlAction)
}
}
}
@@ -428,6 +445,7 @@ internal class WearViewModel(
val current = mutableState.value
val phoneNodeId = current.phoneNodeId ?: return
val session = current.selectedSession ?: return
val routeGeneration = phoneRouteGeneration
if (
current.controlBusy ||
current.talkBusy ||
@@ -440,9 +458,10 @@ internal class WearViewModel(
) {
return
}
val controlAction = beginControlAction(phoneNodeId, routeGeneration) ?: return
viewModelScope.launch {
mutableState.update { it.copy(controlBusy = true, failure = null) }
try {
if (!isCurrentControlRoute(phoneNodeId, routeGeneration)) return@launch
cancelModelLoad()
val responseRequest = eventSequenceTracker.beginResponseRequest()
val selection =
@@ -452,6 +471,7 @@ internal class WearViewModel(
phoneNodeId = phoneNodeId,
capabilities = current.proxyCapabilities,
)
if (!isCurrentControlRoute(phoneNodeId, routeGeneration)) return@launch
val currentSession = mutableState.value.selectedSession ?: return@launch
if (!wearSessionRequestIsCurrent(session, currentSession, selection.phoneNodeId)) return@launch
if (
@@ -468,17 +488,20 @@ internal class WearViewModel(
val acceptedModelRef = selection.selectedModelRef
val updatedSession = currentSession.copy(modelRef = acceptedModelRef)
mutableState.update { state ->
if (!isCurrentControlRoute(phoneNodeId, routeGeneration, state)) return@update state
val selectedSession = state.selectedSession ?: return@update state
if (!wearSessionRequestIsCurrent(session, selectedSession, selection.phoneNodeId)) return@update state
state.switchModelContext(acceptedModelRef)
}
loadModels(updatedSession)
if (isCurrentControlRoute(phoneNodeId, routeGeneration)) {
loadModels(updatedSession)
}
} catch (err: CancellationException) {
throw err
} catch (err: Throwable) {
recordFailure(err)
recordFailureForControlRoute(err, phoneNodeId, routeGeneration)
} finally {
mutableState.update { it.copy(controlBusy = false) }
finishControlAction(controlAction)
}
}
}
@@ -486,6 +509,7 @@ internal class WearViewModel(
fun setGatewayEnabled(enabled: Boolean) {
val current = mutableState.value
val phoneNodeId = current.phoneNodeId ?: return
val routeGeneration = phoneRouteGeneration
if (
current.controlBusy ||
current.connected == enabled ||
@@ -493,9 +517,10 @@ internal class WearViewModel(
) {
return
}
val controlAction = beginControlAction(phoneNodeId, routeGeneration) ?: return
viewModelScope.launch {
mutableState.update { it.copy(controlBusy = true, failure = null) }
try {
if (!isCurrentControlRoute(phoneNodeId, routeGeneration)) return@launch
if (!enabled) {
talkStartJob?.cancel()
talkStartJob = null
@@ -503,28 +528,21 @@ internal class WearViewModel(
realtimeTalkClient.disconnectLocal()
}
val status = repository.setGatewayEnabled(enabled, phoneNodeId, current.proxyCapabilities)
mutableState.update {
it.copy(
connected = status.connected,
phoneNodeId = status.phoneNodeId,
activeAgentId = status.activeAgentId ?: it.activeAgentId,
selectedModelRef =
wearSelectedModelRef(
it.selectedSession?.key,
status.activeSessionKey,
status.selectedModelRef ?: it.selectedModelRef,
),
proxyCapabilities = status.capabilities,
realtimeTalk = if (enabled) it.realtimeTalk else WearRealtimeTalkSnapshot(),
)
if (!isCurrentControlRoute(phoneNodeId, routeGeneration)) return@launch
mutableState.update { state ->
if (!isCurrentControlRoute(phoneNodeId, routeGeneration, state)) {
state
} else {
applyWearGatewayControlStatus(state, status, enabled)
}
}
refresh()
} catch (err: CancellationException) {
throw err
} catch (err: Throwable) {
recordFailure(err, loading = false)
recordFailureForControlRoute(err, phoneNodeId, routeGeneration, loading = false)
} finally {
mutableState.update { it.copy(controlBusy = false) }
finishControlAction(controlAction)
}
}
}
@@ -915,6 +933,7 @@ internal class WearViewModel(
private fun resetForPhoneRouteChange() {
phoneRouteGeneration += 1
controlBusyOwner.reset()
mutableState.update(WearUiState::resetForPhoneChange)
}
@@ -1031,6 +1050,40 @@ internal class WearViewModel(
currentRouteGeneration = phoneRouteGeneration,
)
private fun isCurrentControlRoute(
phoneNodeId: String,
routeGeneration: Long,
state: WearUiState = mutableState.value,
): Boolean =
wearControlRouteIsCurrent(
requestedPhoneNodeId = phoneNodeId,
currentState = state,
requestedRouteGeneration = routeGeneration,
currentRouteGeneration = phoneRouteGeneration,
)
private fun beginControlAction(
phoneNodeId: String,
routeGeneration: Long,
): Long? {
val owner = controlBusyOwner.claim() ?: return null
while (true) {
val state = mutableState.value
if (state.controlBusy || !isCurrentControlRoute(phoneNodeId, routeGeneration, state)) {
controlBusyOwner.release(owner)
return null
}
if (mutableState.compareAndSet(state, state.copy(controlBusy = true, failure = null))) {
return owner
}
}
}
private fun finishControlAction(owner: Long) {
if (!controlBusyOwner.release(owner)) return
mutableState.update { state -> state.copy(controlBusy = false) }
}
private fun recordFailure(
error: Throwable,
loading: Boolean = mutableState.value.loading,
@@ -1067,6 +1120,16 @@ internal class WearViewModel(
}
}
private fun recordFailureForControlRoute(
error: Throwable,
phoneNodeId: String,
routeGeneration: Long,
loading: Boolean = mutableState.value.loading,
) {
if (!isCurrentControlRoute(phoneNodeId, routeGeneration)) return
recordFailure(error, loading)
}
override fun onCleared() {
modelLoadJob?.cancel()
talkStartJob?.cancel()
@@ -1130,6 +1193,55 @@ internal fun wearSessionActionIsCurrent(
requestedSession.phoneNodeId,
)
internal fun wearControlRouteIsCurrent(
requestedPhoneNodeId: String,
currentState: WearUiState,
requestedRouteGeneration: Long,
currentRouteGeneration: Long,
): Boolean =
requestedRouteGeneration == currentRouteGeneration &&
currentState.phoneNodeId == requestedPhoneNodeId
internal class WearControlBusyOwner {
private var nextOwner = 0L
private var activeOwner: Long? = null
fun claim(): Long? {
if (activeOwner != null) return null
nextOwner += 1
return nextOwner.also { owner -> activeOwner = owner }
}
fun release(owner: Long): Boolean {
if (activeOwner != owner) return false
activeOwner = null
return true
}
fun reset() {
activeOwner = null
}
}
internal fun applyWearGatewayControlStatus(
state: WearUiState,
status: WearProxyStatus,
enabled: Boolean,
): WearUiState =
state.copy(
connected = status.connected,
phoneNodeId = status.phoneNodeId,
activeAgentId = status.activeAgentId ?: state.activeAgentId,
selectedModelRef =
wearSelectedModelRef(
state.selectedSession?.key,
status.activeSessionKey,
status.selectedModelRef ?: state.selectedModelRef,
),
proxyCapabilities = status.capabilities,
realtimeTalk = if (enabled) state.realtimeTalk else WearRealtimeTalkSnapshot(),
)
internal fun wearSnapshotSourcesMatch(
firstPhoneNodeId: String,
firstStreamId: String?,
@@ -86,7 +86,7 @@ class WearGatewayRepositoryTest {
WearRpcMethod.AgentsSelect -> JsonObject(emptyMap())
WearRpcMethod.GatewayDisconnect ->
json.parseToJsonElement(
"""{"connected":false,"status":"Offline","activeAgentId":"main","selectedModelRef":"openai/gpt-test","capabilities":["agent-controls","gateway-controls","model-controls","session-selection-lookup"]}""",
"""{"connected":false,"status":"Offline","activeAgentId":"main","selectedModelRef":"openai/gpt-test","capabilities":["agent-controls","gateway-controls","model-controls","session-selection-lookup","attempt-scoped-realtime-audio"]}""",
)
else -> error("unexpected $method")
}
@@ -154,7 +154,7 @@ class WearGatewayRepositoryTest {
val requester =
RecordingRequester { _, _ ->
json.parseToJsonElement(
"""{"connected":true,"status":"Connected","capabilities":["agent-controls","future-capability","gateway-controls","model-controls","session-selection-lookup"]}""",
"""{"connected":true,"status":"Connected","capabilities":["agent-controls","future-capability","gateway-controls","model-controls","session-selection-lookup","attempt-scoped-realtime-audio"]}""",
)
}
@@ -292,13 +292,14 @@ class WearGatewayRepositoryTest {
attemptId = "attempt-7",
language = "de",
phoneNodeId = "phone-a",
attemptScopedAudio = true,
)
assertTrue(snapshot.active)
assertEquals(
json
.parseToJsonElement(
"""{"sessionKey":"agent:main:thread-7","attemptId":"attempt-7","language":"de"}""",
"""{"sessionKey":"agent:main:thread-7","attemptId":"attempt-7","language":"de","attemptScopedAudio":true}""",
).jsonObject,
requester.calls.single().second,
)
@@ -306,6 +307,31 @@ class WearGatewayRepositoryTest {
assertTrue(requester.requirePreferredNodes.single())
}
@Test
fun realtimeTalkStartOmitsAttemptScopedAudioForLegacyPhones() =
runTest {
val requester =
RecordingRequester { _, _ ->
json.parseToJsonElement("""{"active":true}""")
}
WearGatewayRepository(requester).startRealtimeTalk(
sessionKey = "agent:main:thread-7",
attemptId = "attempt-7",
language = null,
phoneNodeId = "phone-a",
attemptScopedAudio = false,
)
assertEquals(
json
.parseToJsonElement(
"""{"sessionKey":"agent:main:thread-7","attemptId":"attempt-7"}""",
).jsonObject,
requester.calls.single().second,
)
}
@Test
fun observedFinalMessageSurvivesAnOlderSnapshotWithoutDuplication() {
val older = WearChatMessage(id = "m1", role = "assistant", text = "older", timestamp = 1)
@@ -1,5 +1,6 @@
package ai.openclaw.wear
import ai.openclaw.wear.shared.WearProxyCapability
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
@@ -153,6 +154,86 @@ class WearSessionScopeTest {
)
}
@Test
fun delayedControlsRequireTheOriginalPhoneRouteGeneration() {
val phoneA = WearUiState(phoneNodeId = "phone-a", controlBusy = true)
assertTrue(
wearControlRouteIsCurrent(
requestedPhoneNodeId = "phone-a",
currentState = phoneA,
requestedRouteGeneration = 3,
currentRouteGeneration = 3,
),
)
assertFalse(
wearControlRouteIsCurrent(
requestedPhoneNodeId = "phone-a",
currentState = WearUiState(phoneNodeId = "phone-b", controlBusy = true),
requestedRouteGeneration = 3,
currentRouteGeneration = 4,
),
)
assertFalse(
wearControlRouteIsCurrent(
requestedPhoneNodeId = "phone-a",
currentState = phoneA,
requestedRouteGeneration = 3,
currentRouteGeneration = 5,
),
)
}
@Test
fun staleControlCompletionCannotClearReplacementBusyOwner() {
val owners = WearControlBusyOwner()
val staleOwner = checkNotNull(owners.claim())
owners.reset()
val replacementOwner = checkNotNull(owners.claim())
assertFalse(owners.release(staleOwner))
assertTrue(owners.release(replacementOwner))
}
@Test
fun abandonedControlActionReleasesItsOwnBusyOwner() {
val owners = WearControlBusyOwner()
val owner = checkNotNull(owners.claim())
assertTrue(owners.release(owner))
assertTrue(owners.claim() != null)
}
@Test
fun gatewayControlResponseKeepsBusyUntilItsOwnerFinalizes() {
val updated =
applyWearGatewayControlStatus(
state =
WearUiState(
phoneNodeId = "phone-a",
controlBusy = true,
activeAgentId = "agent-a",
),
status =
WearProxyStatus(
connected = true,
activeAgentId = "agent-b",
activeSessionKey = null,
selectedModelRef = null,
capabilities = setOf(WearProxyCapability.GatewayControls),
eventStreamId = null,
eventSequence = null,
phoneNodeId = "phone-b",
),
enabled = true,
)
assertTrue(updated.controlBusy)
assertEquals("phone-b", updated.phoneNodeId)
assertEquals("agent-b", updated.activeAgentId)
}
@Test
fun snapshotResponsesRequireTheSamePhoneAndEventStream() {
assertEquals(true, wearSnapshotSourcesMatch("phone-a", "stream-a", "phone-a", "stream-a"))
@@ -5,6 +5,7 @@ import ai.openclaw.wear.shared.WearRpcMethod
import android.animation.ValueAnimator
import android.content.Intent
import android.os.Looper
import android.os.Parcel
import android.os.PowerManager
import android.provider.Settings
import androidx.activity.ComponentActivity
@@ -17,9 +18,13 @@ import androidx.compose.ui.graphics.Color
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.LifecycleRegistry
import com.google.android.gms.wearable.ChannelClient
import kotlinx.coroutines.Job
import kotlinx.coroutines.channels.Channel
import kotlinx.serialization.json.JsonObject
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertSame
import org.junit.Assert.assertTrue
import org.junit.Test
import org.junit.runner.RunWith
@@ -30,6 +35,8 @@ import org.robolectric.Shadows.shadowOf
import org.robolectric.annotation.Config
import org.robolectric.shadows.ShadowSystemClock
import org.robolectric.shadows.ShadowValueAnimator
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.time.Duration
@RunWith(RobolectricTestRunner::class)
@@ -73,13 +80,19 @@ class WearTalkAvatarTest {
)
val client = realtimeTalkClient()
val queuedLevels = Channel<Float>(Channel.UNLIMITED)
client.setPrivateField("activeNodeId", "watch-a")
val attempt = realtimeAttempt(generation = 1L)
client.setPrivateField("activeAttempt", attempt)
client.setPrivateField("mouthFrames", queuedLevels)
try {
val writeOutput = WearRealtimeTalkClient::class.java.getDeclaredMethod("writeOutput", ByteArray::class.java)
val writeOutput =
WearRealtimeTalkClient::class.java.getDeclaredMethod(
"writeOutput",
WearRealtimeTalkClient.ActiveAttempt::class.java,
ByteArray::class.java,
)
writeOutput.isAccessible = true
chunks.forEach { chunk -> writeOutput.invoke(client, chunk) }
chunks.forEach { chunk -> writeOutput.invoke(client, attempt, chunk) }
awaitPlaybackTeardown(client)
val actualLevels =
@@ -92,6 +105,47 @@ class WearTalkAvatarTest {
}
}
@Test
fun staleAttemptCallbacksCannotMutateReplacement() {
val client = realtimeTalkClient()
val stale = realtimeAttempt(generation = 1L)
val replacement = realtimeAttempt(generation = 2L)
try {
client.invokePrivate("activate", stale)
client.invokePrivate("handleChannelFailure", stale)
assertTrue(client.channelFailed.value)
assertEquals(null, client.privateField("activeAttempt"))
client.invokePrivate("activate", replacement)
val replacementReader = Job()
client.setPrivateField("readJob", replacementReader)
assertFalse(client.channelFailed.value)
client.invokePrivate("writeOutput", stale, pcm16Le(samplesForFrames(1), sample = 20_000))
client.invokePrivate("handleChannelFailure", stale)
client.invokePrivate("closeLocal", stale, false)
assertFalse(client.isPlaying.value)
assertFalse(client.channelFailed.value)
assertTrue(replacementReader.isActive)
assertSame(replacement, client.privateField("activeAttempt"))
} finally {
client.shutdown()
}
}
@Test
fun realtimeAudioPathUsesNegotiatedAttemptScope() {
assertEquals(
WearProtocol.LEGACY_REALTIME_AUDIO_CHANNEL_PATH,
wearRealtimeAudioChannelPath("attempt-7", attemptScopedAudio = false),
)
assertEquals(
WearProtocol.realtimeAudioChannelPath("attempt-7"),
wearRealtimeAudioChannelPath("attempt-7", attemptScopedAudio = true),
)
}
@Test
fun mouthEnvelopeUsesFastAttackAndSoftReleaseWithoutOvershoot() {
val attack = smoothAvatarMouth(current = 0f, target = 1f, deltaSeconds = 0.02f)
@@ -360,6 +414,19 @@ class WearTalkAvatarTest {
return WearRealtimeTalkClient(RuntimeEnvironment.getApplication(), WearGatewayRepository(requester))
}
private fun realtimeAttempt(generation: Long): WearRealtimeTalkClient.ActiveAttempt =
WearRealtimeTalkClient.ActiveAttempt(
nodeId = "watch-a",
attemptId = "attempt-$generation",
generation = generation,
resources =
WearRealtimeTalkClient.ChannelResources(
channel = FakeRealtimeChannel("watch-a", "channel-$generation"),
input = ByteArrayInputStream(byteArrayOf()),
output = ByteArrayOutputStream(),
),
)
private fun awaitPlaybackTeardown(client: WearRealtimeTalkClient) {
ShadowSystemClock.advanceBy(Duration.ofSeconds(1L))
val deadlineNanos = System.nanoTime() + 2_000_000_000L
@@ -388,6 +455,28 @@ class WearTalkAvatarTest {
}
}
private fun Any.privateField(name: String): Any? =
javaClass.getDeclaredField(name).run {
isAccessible = true
get(this@privateField)
}
private fun WearRealtimeTalkClient.invokePrivate(
name: String,
vararg args: Any,
) {
javaClass.declaredMethods
.single { method ->
method.name == name &&
method.parameterTypes.size == args.size &&
method.parameterTypes.zip(args).all { (type, arg) ->
type.isAssignableFrom(arg.javaClass) ||
(type == Boolean::class.javaPrimitiveType && arg is Boolean)
}
}.apply { isAccessible = true }
.invoke(this, *args)
}
private companion object {
const val WEAR_REALTIME_SAMPLE_RATE_HZ = 24_000
}
@@ -447,3 +536,21 @@ class WearTalkAvatarTest {
override val lifecycle: Lifecycle = registry
}
}
private data class FakeRealtimeChannel(
private val nodeId: String,
private val label: String,
) : ChannelClient.Channel {
override fun getNodeId(): String = nodeId
override fun getPath(): String = WearProtocol.realtimeAudioChannelPath("attempt-$label")
override fun describeContents(): Int = 0
override fun writeToParcel(
dest: Parcel,
flags: Int,
) {
dest.writeString(label)
}
}
+4 -4
View File
@@ -292,10 +292,6 @@ avdmanager_bin() {
printf '%s\n' "$AVDMANAGER"
return
fi
if command -v avdmanager >/dev/null 2>&1; then
command -v avdmanager
return
fi
for sdk_root in "${ANDROID_HOME:-}" "${ANDROID_SDK_ROOT:-}" "$HOME/Library/Android/sdk"; do
for relative_path in cmdline-tools/latest/bin/avdmanager cmdline-tools/bin/avdmanager tools/bin/avdmanager; do
if [[ -n "$sdk_root" && -x "$sdk_root/$relative_path" ]]; then
@@ -304,6 +300,10 @@ avdmanager_bin() {
fi
done
done
if command -v avdmanager >/dev/null 2>&1; then
command -v avdmanager
return
fi
echo "avdmanager not found. Install Android SDK command-line tools or set AVDMANAGER." >&2
return 127
}
@@ -22,7 +22,13 @@ import { resolveNodeCommandAllowlist } from "./node-command-policy.js";
const LIVE = isLiveTestEnabled();
const LIVE_ANDROID_NODE = isTruthyEnvValue(process.env.OPENCLAW_LIVE_ANDROID_NODE);
const describeLive = LIVE && LIVE_ANDROID_NODE ? describe : describe.skip;
const SKIPPED_INTERACTIVE_COMMANDS = new Set<string>();
const SKIPPED_INTERACTIVE_COMMANDS = new Set([
"screen.record",
"talk.ptt.start",
"talk.ptt.stop",
"talk.ptt.cancel",
"talk.ptt.once",
]);
type CommandOutcome = "success" | "error";
@@ -296,12 +302,44 @@ const COMMAND_PROFILES: Record<string, CommandProfile> = {
buildParams: () => ({}),
timeoutMs: 20_000,
outcome: "success",
allowedErrorCodes: ["SMS_PERMISSION_REQUIRED"],
onSuccess: (payload) => {
const obj = assertObjectPayload("sms.search", payload);
expect(["number", "string"]).toContain(typeof obj.count);
expect(Array.isArray(obj.messages)).toBe(true);
},
},
"system.notify": {
buildParams: () => ({
title: "OpenClaw Android E2E",
body: "Live node integration check",
sound: "none",
priority: "passive",
}),
timeoutMs: 20_000,
outcome: "success",
allowedErrorCodes: ["NOT_AUTHORIZED"],
},
"contacts.search": {
buildParams: () => ({ query: "__openclaw_live_no_match__", limit: 1 }),
timeoutMs: 20_000,
outcome: "success",
allowedErrorCodes: ["CONTACTS_PERMISSION_REQUIRED"],
onSuccess: (payload) => {
const obj = assertObjectPayload("contacts.search", payload);
expect(Array.isArray(obj.contacts)).toBe(true);
},
},
"calendar.events": {
buildParams: () => ({ limit: 1 }),
timeoutMs: 20_000,
outcome: "success",
allowedErrorCodes: ["CALENDAR_PERMISSION_REQUIRED"],
onSuccess: (payload) => {
const obj = assertObjectPayload("calendar.events", payload);
expect(Array.isArray(obj.events)).toBe(true);
},
},
"debug.logs": {
buildParams: () => ({}),
timeoutMs: 20_000,
@@ -546,16 +584,60 @@ function evaluateCommandResult(params: {
}
const code = result.errorCode ?? "UNKNOWN";
if (profile.outcome === "success") {
return `expected success, got ${code}: ${result.errorMessage ?? "unknown error"}`;
}
const allowed = new Set(profile.allowedErrorCodes ?? []);
if (allowed.has(code)) {
return null;
}
if (profile.outcome === "success") {
return `expected success, got ${code}: ${result.errorMessage ?? "unknown error"}`;
}
return `unexpected error ${code}: ${result.errorMessage ?? "unknown error"}`;
}
describe("android node command profiles", () => {
it("accepts declared environment errors for success profiles", () => {
const profile = expectDefined(COMMAND_PROFILES["contacts.search"], "contacts.search profile");
expect(
evaluateCommandResult({
result: {
command: "contacts.search",
ok: false,
errorCode: "CONTACTS_PERMISSION_REQUIRED",
errorMessage: "grant Contacts permission",
durationMs: 1,
},
profile,
ctx: { notifications: [] },
}),
).toBeNull();
});
it("still rejects undeclared errors for success profiles", () => {
const profile = expectDefined(COMMAND_PROFILES["contacts.search"], "contacts.search profile");
expect(
evaluateCommandResult({
result: {
command: "contacts.search",
ok: false,
errorCode: "INVALID_REQUEST",
errorMessage: "invalid request",
durationMs: 1,
},
profile,
ctx: { notifications: [] },
}),
).toContain("expected success");
});
it("keeps microphone capture commands out of the non-interactive matrix", () => {
expect(
["talk.ptt.start", "talk.ptt.stop", "talk.ptt.cancel", "talk.ptt.once"].every((command) =>
SKIPPED_INTERACTIVE_COMMANDS.has(command),
),
).toBe(true);
});
});
describeLive("android node capability integration (preconditioned)", () => {
let client: GatewayClient | null = null;
let nodeId = "";
+16
View File
@@ -0,0 +1,16 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
const PERF_SCRIPTS = [
"apps/android/scripts/perf-online-benchmark.sh",
"apps/android/scripts/perf-startup-hotspots.sh",
];
describe("Android performance scripts", () => {
it.each(PERF_SCRIPTS)("installs the Play debug variant in %s", (scriptPath) => {
const script = readFileSync(scriptPath, "utf8");
expect(script).toContain(":app:installPlayDebug");
expect(script).not.toContain(":app:installDebug");
});
});
+14
View File
@@ -113,6 +113,20 @@ describe("android screenshots script", () => {
expect(script).toContain("is not the screenshot AVD");
});
it("prefers avdmanager from the configured SDK over an unrelated PATH install", () => {
const script = readFileSync(SCRIPT, "utf8");
const functionStart = script.indexOf("avdmanager_bin() {");
const sdkLookup = script.indexOf(
'for sdk_root in "${ANDROID_HOME:-}" "${ANDROID_SDK_ROOT:-}" "$HOME/Library/Android/sdk"; do',
functionStart,
);
const pathLookup = script.indexOf("if command -v avdmanager", functionStart);
expect(functionStart).toBeGreaterThan(-1);
expect(sdkLookup).toBeGreaterThan(functionStart);
expect(pathLookup).toBeGreaterThan(sdkLookup);
});
it.each(["../escape", "en/US", ".hidden", "en..US", ""])(
"rejects locale path escapes before dry-run output: %j",
(locale) => {