diff --git a/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt b/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt index 84dceee9ac46..16230d76dd40 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt @@ -184,6 +184,8 @@ private const val CRON_JOBS_PAGE_SIZE = 200 private const val CRON_JOBS_MAX_PAGES = 100 private const val CRON_JOBS_MAX_COUNT = CRON_JOBS_PAGE_SIZE * CRON_JOBS_MAX_PAGES private const val CRON_JOBS_SNAPSHOT_MAX_ATTEMPTS = 3 +private const val USAGE_INCOMPLETE_RETRY_DELAY_MS = 5_000L +private const val USAGE_INCOMPLETE_RETRY_LIMIT = 3 private const val OperatorAdminScope = "operator.admin" private const val OperatorPairingScope = "operator.pairing" @@ -1282,6 +1284,8 @@ class NodeRuntime private constructor( val usageRefreshing: StateFlow = _usageRefreshing.asStateFlow() private val _usageErrorText = MutableStateFlow(null) val usageErrorText: StateFlow = _usageErrorText.resolveOptionalNativeText() + private val usageRefreshGuard = LatestGatewayRefreshGuard() + private var usageIncompleteRetryJob: Job? = null private val _skillsSummary = MutableStateFlow(GatewaySkillsSummary(skills = emptyList())) val skillsSummary: StateFlow = _skillsSummary.asStateFlow() private val _skillsRefreshing = MutableStateFlow(false) @@ -1361,6 +1365,9 @@ class NodeRuntime private constructor( @Volatile internal var gatewayDataRequestTimeoutObserverForTests: ((method: String, timeoutMs: Long) -> Unit)? = null @Volatile internal var clawHubSkillInstallBeforeClaimObserverForTests: (() -> Unit)? = null + + @Volatile internal var usageIncompleteRetryDelayMsForTests: Long? = null + private val _channelsSummary = MutableStateFlow(GatewayChannelsSummary(channels = emptyList())) val channelsSummary: StateFlow = _channelsSummary.asStateFlow() private val _channelsRefreshing = MutableStateFlow(false) @@ -1732,6 +1739,8 @@ class NodeRuntime private constructor( if (retirePendingCronRuns) { pendingCronRunRegistry.clear { _pendingCronRunJobIds.value = it } } + usageIncompleteRetryJob?.cancel() + usageRefreshGuard.invalidate() _usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList()) _usageRefreshing.value = false _usageErrorText.value = null @@ -6331,20 +6340,85 @@ class NodeRuntime private constructor( } } - private suspend fun refreshUsageFromGateway() = - refreshGatewaySummary( - summary = _usageSummary, - refreshing = _usageRefreshing, - errorText = _usageErrorText, - disconnectedSummary = GatewayUsageSummary(updatedAtMs = null, providers = emptyList()), - failureText = nativeText("Could not load usage."), - ) { gatewayScope -> - val root = json.parseToJsonElement(requestGatewayData(gatewayScope, "usage.status", "{}")).asObjectOrNull() - GatewayUsageSummary( - updatedAtMs = root.long("updatedAt"), - providers = parseUsageProviders(root?.get("providers") as? JsonArray), - ) + private suspend fun refreshUsageFromGateway() { + usageIncompleteRetryJob?.cancel() + val gatewayScope = captureGatewayDataScope() ?: return + val refreshGeneration = usageRefreshGuard.begin() + if (refreshUsageOnceFromGateway(gatewayScope, refreshGeneration) && _usageSummary.value.refreshing) { + scheduleIncompleteUsageRetry(gatewayScope, refreshGeneration) } + } + + private fun publishUsageRefresh( + gatewayScope: GatewayDataScope, + refreshGeneration: Long, + publish: () -> Unit, + ): Boolean { + var refreshCurrent = false + val scopeCurrent = + publishGatewayData(gatewayScope) { + refreshCurrent = usageRefreshGuard.publishIfCurrent(refreshGeneration, publish) + } + return scopeCurrent && refreshCurrent + } + + private suspend fun refreshUsageOnceFromGateway( + gatewayScope: GatewayDataScope, + refreshGeneration: Long, + ): Boolean { + publishUsageRefresh(gatewayScope, refreshGeneration) { + _usageRefreshing.value = true + _usageErrorText.value = null + } + if (!operatorConnected) { + return publishUsageRefresh(gatewayScope, refreshGeneration) { + _usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList()) + _usageRefreshing.value = false + } + } + return try { + val root = json.parseToJsonElement(requestGatewayData(gatewayScope, "usage.status", "{}")).asObjectOrNull() + val nextSummary = + GatewayUsageSummary( + updatedAtMs = root.long("updatedAt"), + providers = parseUsageProviders(root?.get("providers") as? JsonArray), + refreshing = root.boolean("refreshing"), + ) + publishUsageRefresh(gatewayScope, refreshGeneration) { _usageSummary.value = nextSummary } + } catch (_: Throwable) { + publishUsageRefresh(gatewayScope, refreshGeneration) { + // Preserve same-identity provider rows across a transient refresh failure. + _usageSummary.value = _usageSummary.value.copy(refreshing = false) + _usageErrorText.value = nativeText("Could not load usage.") + } + } finally { + publishUsageRefresh(gatewayScope, refreshGeneration) { _usageRefreshing.value = false } + } + } + + private fun scheduleIncompleteUsageRetry( + gatewayScope: GatewayDataScope, + refreshGeneration: Long, + ) { + // Mirrors the shared clients: three delayed retries, cancelled by a new cycle or gateway. + usageIncompleteRetryJob?.cancel() + usageIncompleteRetryJob = + scope.launch { + repeat(USAGE_INCOMPLETE_RETRY_LIMIT) { + delay(usageIncompleteRetryDelayMsForTests ?: USAGE_INCOMPLETE_RETRY_DELAY_MS) + if (!isGatewayDataScopeCurrent(gatewayScope)) return@launch + if (!refreshUsageOnceFromGateway(gatewayScope, refreshGeneration)) return@launch + if (!_usageSummary.value.refreshing) return@launch + } + publishUsageRefresh(gatewayScope, refreshGeneration) { + // Clearing the marker alone renders as "No usage data yet.", which claims + // the operator has no providers. Reuse the transient-failure text so a + // spent retry budget reads as the load failure it is. + _usageSummary.value = _usageSummary.value.copy(refreshing = false) + _usageErrorText.value = nativeText("Could not load usage.") + } + } + } private suspend fun refreshSkillsFromGateway(): Boolean = refreshGatewaySummary( @@ -8776,6 +8850,7 @@ data class GatewayCronJobSummary( data class GatewayUsageSummary( val updatedAtMs: Long?, val providers: List, + val refreshing: Boolean = false, ) data class GatewayUsageProviderSummary( diff --git a/apps/android/app/src/main/java/ai/openclaw/app/node/ConnectionManager.kt b/apps/android/app/src/main/java/ai/openclaw/app/node/ConnectionManager.kt index 2e2ee1f3247d..b49e65c34a09 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/node/ConnectionManager.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/node/ConnectionManager.kt @@ -54,6 +54,7 @@ class ConnectionManager internal constructor( internal const val AGENT_KIND_CLIENT_CAPABILITY = "agent-kind" internal const val INLINE_WIDGETS_CLIENT_CAPABILITY = "inline-widgets" + internal const val USAGE_REFRESHING_CLIENT_CAPABILITY = "usage-refreshing" internal fun operatorScopesForStoredDeviceToken(storedScopes: List): List { val normalized = @@ -233,6 +234,7 @@ class ConnectionManager internal constructor( buildList { add(AGENT_KIND_CLIENT_CAPABILITY) if (inlineWidgetsAvailable()) add(INLINE_WIDGETS_CLIENT_CAPABILITY) + add(USAGE_REFRESHING_CLIENT_CAPABILITY) }, commands = emptyList(), permissions = emptyMap(), diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt index 29f799732225..d3acefe95371 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt @@ -247,6 +247,7 @@ private fun UsageSettingsScreen( val isConnected by viewModel.isConnected.collectAsState() val providerCount = usageSummary.providers.size val issueCount = usageSummary.providers.count { it.error != null } + val usageConverging = usageRefreshVisible(usageRefreshing, usageSummary.refreshing) LaunchedEffect(isConnected) { if (isConnected) { @@ -264,7 +265,7 @@ private fun UsageSettingsScreen( ), ) Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) { - ClawSecondaryButton(text = if (usageRefreshing) nativeString("Refreshing") else nativeString("Refresh"), onClick = viewModel::refreshUsage, enabled = isConnected && !usageRefreshing, modifier = Modifier.weight(1f)) + ClawSecondaryButton(text = if (usageConverging) nativeString("Refreshing") else nativeString("Refresh"), onClick = viewModel::refreshUsage, enabled = isConnected && !usageConverging, modifier = Modifier.weight(1f)) } usageErrorText?.let { errorText -> ClawPanel { @@ -276,14 +277,17 @@ private fun UsageSettingsScreen( ClawPanel { Text(text = nativeString("Connect the gateway to load usage."), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted) } - usageSummary.providers.isEmpty() -> + usageSummary.providers.isNotEmpty() -> UsageProvidersPanel(providers = usageSummary.providers) + // The warning panel above already reports a failed load; adding + // "No usage data yet." beside it claims the operator has no providers. + usageErrorText != null -> Unit + else -> ClawPanel { Column(verticalArrangement = Arrangement.spacedBy(3.dp)) { - Text(text = nativeString("No usage data yet."), style = ClawTheme.type.section, color = ClawTheme.colors.text) + Text(text = if (usageConverging) nativeString("Refreshing") else nativeString("No usage data yet."), style = ClawTheme.type.section, color = ClawTheme.colors.text) Text(text = nativeString("Provider limits will appear here when your gateway reports them."), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted) } } - else -> UsageProvidersPanel(providers = usageSummary.providers) } } } @@ -2553,6 +2557,11 @@ private fun UsageProvidersPanel(providers: List) { } } +internal fun usageRefreshVisible( + requestRefreshing: Boolean, + summaryRefreshing: Boolean, +): Boolean = requestRefreshing || summaryRefreshing + @Composable private fun UsageProviderListRow(provider: GatewayUsageProviderSummary) { val hasIssue = provider.error != null diff --git a/apps/android/app/src/test/java/ai/openclaw/app/UsageStatusRuntimeTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/UsageStatusRuntimeTest.kt new file mode 100644 index 000000000000..b575a7f665e1 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/app/UsageStatusRuntimeTest.kt @@ -0,0 +1,137 @@ +package ai.openclaw.app + +import ai.openclaw.app.gateway.GatewayEndpoint +import ai.openclaw.app.ui.usageRefreshVisible +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import java.lang.reflect.Field +import java.util.UUID +import java.util.concurrent.atomic.AtomicInteger + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class UsageStatusRuntimeTest { + @Test + fun incompleteUsageConvergesOnACompletedPayload() { + val runtime = createRuntime() + connect(runtime) + runtime.usageIncompleteRetryDelayMsForTests = 10L + val calls = AtomicInteger() + runtime.gatewayDataRequestOverrideForTests = { _, method, _ -> + check(method == "usage.status") + if (calls.incrementAndGet() == 1) { + """{"updatedAt":1,"providers":[],"refreshing":true}""" + } else { + """{"updatedAt":2,"providers":[{"displayName":"Claude","plan":"Pro","windows":[]}]}""" + } + } + + runtime.refreshUsage() + waitUntil { + runtime.usageSummary.value.providers + .isNotEmpty() + } + assertEquals(2, calls.get()) + assertFalse(runtime.usageSummary.value.refreshing) + } + + @Test + fun incompleteUsageRetriesStayBounded() { + val runtime = createRuntime() + connect(runtime) + runtime.usageIncompleteRetryDelayMsForTests = 10L + val calls = AtomicInteger() + runtime.gatewayDataRequestOverrideForTests = { _, _, _ -> + calls.incrementAndGet() + """{"updatedAt":1,"providers":[],"refreshing":true}""" + } + + runtime.refreshUsage() + waitUntil { calls.get() == 4 } + Thread.sleep(100) + assertEquals(4, calls.get()) + assertFalse(runtime.usageSummary.value.refreshing) + assertFalse( + usageRefreshVisible( + requestRefreshing = runtime.usageRefreshing.value, + summaryRefreshing = runtime.usageSummary.value.refreshing, + ), + ) + // Clearing the marker without this would render "No usage data yet.", + // claiming the operator has no providers instead of a failed load. + assertNotNull(runtime.usageErrorText.value) + } + + @Test + fun aTransientFailurePreservesRowsAndStopsTheRetryChain() { + val runtime = createRuntime() + connect(runtime) + runtime.usageIncompleteRetryDelayMsForTests = 10L + val calls = AtomicInteger() + runtime.gatewayDataRequestOverrideForTests = { _, _, _ -> + if (calls.incrementAndGet() == 1) { + """{"updatedAt":1,"providers":[{"displayName":"Claude","plan":"Pro","windows":[]}],"refreshing":true}""" + } else { + error("usage unavailable") + } + } + + runtime.refreshUsage() + waitUntil { + runtime.usageErrorText.value != null + } + Thread.sleep(100) + assertEquals(2, calls.get()) + assertEquals( + "Claude", + runtime.usageSummary.value.providers + .single() + .displayName, + ) + assertFalse(runtime.usageSummary.value.refreshing) + } + + private fun createRuntime(): NodeRuntime { + val app = RuntimeEnvironment.getApplication() + val prefs = + app.getSharedPreferences( + "usage.${UUID.randomUUID()}", + android.content.Context.MODE_PRIVATE, + ) + return NodeRuntime(app, SecurePrefs(app, securePrefsOverride = prefs)) + } + + private fun connect(runtime: NodeRuntime) { + field(runtime, "connectedEndpoint").set(runtime, GatewayEndpoint.manual("127.0.0.1", 18789)) + field(runtime, "operatorConnected").set(runtime, true) + } + + private fun waitUntil(condition: () -> Boolean) { + repeat(200) { + if (condition()) return + Thread.sleep(10) + } + error("condition did not become true") + } + + private fun field( + target: Any, + name: String, + ): Field { + var type: Class<*>? = target.javaClass + while (type != null) { + try { + return type.getDeclaredField(name).apply { isAccessible = true } + } catch (_: NoSuchFieldException) { + type = type.superclass + } + } + error("field $name not found") + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/app/node/ConnectionManagerTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/node/ConnectionManagerTest.kt index 0077eaea9fd1..65899a4d2066 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/node/ConnectionManagerTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/node/ConnectionManagerTest.kt @@ -139,6 +139,7 @@ class ConnectionManagerTest { listOf( ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY, ConnectionManager.INLINE_WIDGETS_CLIENT_CAPABILITY, + ConnectionManager.USAGE_REFRESHING_CLIENT_CAPABILITY, ), options.caps, ) @@ -148,7 +149,13 @@ class ConnectionManagerTest { fun buildOperatorConnectOptions_omitsInlineWidgetsWithoutIsolatedWebViews() { val options = newManager(inlineWidgetsAvailable = false).buildOperatorConnectOptions() - assertEquals(listOf(ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY), options.caps) + assertEquals( + listOf( + ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY, + ConnectionManager.USAGE_REFRESHING_CLIENT_CAPABILITY, + ), + options.caps, + ) } @Test diff --git a/apps/android/app/src/test/java/ai/openclaw/app/ui/SettingsScreensTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/ui/SettingsScreensTest.kt index 32581ba95e51..3ca451c8ad35 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/ui/SettingsScreensTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/ui/SettingsScreensTest.kt @@ -349,6 +349,13 @@ class SettingsScreensTest { assertEquals("None", formatCronTimestamp(null)) } + @Test + fun usageRefreshStaysVisibleBetweenIncompleteRetries() { + assertTrue(usageRefreshVisible(requestRefreshing = true, summaryRefreshing = false)) + assertTrue(usageRefreshVisible(requestRefreshing = false, summaryRefreshing = true)) + assertFalse(usageRefreshVisible(requestRefreshing = false, summaryRefreshing = false)) + } + @Test fun approvalCardShowsTheWholeMonospacedCommandBeforeStackedActions() { val source = settingsScreensSource() diff --git a/apps/macos/Sources/OpenClaw/GatewayConnection.swift b/apps/macos/Sources/OpenClaw/GatewayConnection.swift index 5ce2882797e7..300b671056e5 100644 --- a/apps/macos/Sources/OpenClaw/GatewayConnection.swift +++ b/apps/macos/Sources/OpenClaw/GatewayConnection.swift @@ -14,8 +14,11 @@ private let gatewayConnectionLogger = Logger(subsystem: "ai.openclaw", category: actor GatewayConnection { static let shared = GatewayConnection( endpointProvider: GatewayConnection.defaultEndpointProvider) - nonisolated static let operatorClientCaps = - [OpenClawGatewayClientCapability.agentKind, OpenClawGatewayClientCapability.inlineWidgets] + nonisolated static let operatorClientCaps = [ + OpenClawGatewayClientCapability.agentKind, + OpenClawGatewayClientCapability.inlineWidgets, + OpenClawGatewayClientCapability.usageRefreshing, + ] typealias Config = (url: URL, token: String?, password: String?) diff --git a/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift b/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift index b2ad15d9e418..e5d47d9caa62 100644 --- a/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift +++ b/apps/macos/Sources/OpenClaw/MenuSessionsInjector.swift @@ -29,6 +29,11 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate { private var cachedUsageSummary: GatewayUsageSummary? private var usageCacheUpdatedAt: Date? private let usageRefreshIntervalSeconds: TimeInterval = 30 + private var usageRetryTask: Task? + private var usageRetryAttempts = 0 + private var usageLoadGeneration = 0 + private var usageRetryIntervalSeconds: TimeInterval = 5 + private let usageRetryLimit = 3 private var cachedCostSummary: GatewayCostUsageSummary? private var cachedCostErrorText: String? private var costCacheUpdatedAt: Date? @@ -36,6 +41,9 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate { private let nodesStore = NodesStore.shared #if DEBUG private var testControlChannelConnected: Bool? + private var testUsageLoad: (() async throws -> GatewayUsageSummary)? + private var testUsageLoadDidFinish: (@MainActor () -> Void)? + private var testUsageRetryDidExhaust: (@MainActor () -> Void)? #endif func install(into statusItem: NSStatusItem) { @@ -420,7 +428,8 @@ extension MenuSessionsInjector { private func insertUsageSection(into menu: NSMenu, at cursor: Int, width: CGFloat) -> Int { let rows = self.usageRows - if rows.isEmpty { + let stalled = self.isUsageStalled + if rows.isEmpty, !stalled { return cursor } @@ -444,6 +453,16 @@ extension MenuSessionsInjector { menu.insertItem(headerItem, at: cursor) cursor += 1 + if rows.isEmpty { + menu.insertItem( + self.makeMessageItem( + text: "Usage did not finish loading", + symbolName: "exclamationmark.triangle", + width: width), + at: cursor) + return cursor + 1 + } + if let selectedProvider = self.selectedUsageProviderId, let primary = rows.first(where: { $0.providerId.lowercased() == selectedProvider }), rows.count > 1 @@ -516,6 +535,17 @@ extension MenuSessionsInjector { return summary.primaryRows() } + /// Retry budget spent with the cold marker still set. The empty row list is + /// the Gateway saying "not loaded", so the menu must not hide the section + /// the way it does for an operator who genuinely has no usage providers. + /// Requires a live control channel: while disconnected the honest answer is + /// the disconnected state, not a usage-specific warning. + private var isUsageStalled: Bool { + self.isControlChannelConnected + && self.cachedUsageSummary?.refreshing == true + && self.usageRetryAttempts >= self.usageRetryLimit + } + private func buildUsageOverflowMenu(rows: [UsageRow], width: CGFloat) -> NSMenu { let menu = NSMenu() // Keep submenu delegate nil: reusing the status-menu delegate here causes @@ -776,19 +806,73 @@ extension MenuSessionsInjector { return } + self.usageLoadGeneration += 1 + let generation = self.usageLoadGeneration + guard self.isControlChannelConnected else { self.usageCacheUpdatedAt = Date() return } + self.usageRetryTask?.cancel() + self.usageRetryTask = nil + self.usageRetryAttempts = 0 + await self.loadUsageSummaryOnce(generation: generation) + } + + private func loadUsageSummaryOnce(generation: Int) async { + #if DEBUG + defer { self.testUsageLoadDidFinish?() } + #endif do { - self.cachedUsageSummary = try await UsageLoader.loadSummary() + let summary = try await self.loadUsageSummary() + guard generation == self.usageLoadGeneration else { return } + self.cachedUsageSummary = summary + if summary.refreshing == true { + // A cold Gateway marker is not a cacheable answer; converge with bounded retries. + self.usageCacheUpdatedAt = nil + self.scheduleUsageRetry(generation: generation) + return + } } catch { + guard generation == self.usageLoadGeneration else { return } self.cachedUsageSummary = nil } self.usageCacheUpdatedAt = Date() } + private func scheduleUsageRetry(generation: Int) { + guard self.usageRetryAttempts < self.usageRetryLimit else { + #if DEBUG + self.testUsageRetryDidExhaust?() + #endif + return + } + self.usageRetryAttempts += 1 + let interval = self.usageRetryIntervalSeconds + self.usageRetryTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + guard let self, + !Task.isCancelled, + self.isControlChannelConnected, + generation == self.usageLoadGeneration + else { return } + await self.loadUsageSummaryOnce(generation: generation) + // The final attempt leaves the marker set, so repaint on exhaustion + // too or an operator watching the open menu never sees the outcome. + if self.cachedUsageSummary?.refreshing != true || self.isUsageStalled { + await self.repaintOpenMenu(self.statusItem?.menu) + } + } + } + + private func loadUsageSummary() async throws -> GatewayUsageSummary { + #if DEBUG + if let load = self.testUsageLoad { return try await load() } + #endif + return try await UsageLoader.loadSummary() + } + private func refreshCostUsageCache(force: Bool) async { if !force, let updated = self.costCacheUpdatedAt, @@ -1295,6 +1379,34 @@ extension MenuSessionsInjector { self.usageCacheUpdatedAt = Date() } + func setTestingUsageLoader(_ load: (() async throws -> GatewayUsageSummary)?) { + self.testUsageLoad = load + } + + func setTestingUsageLoadDidFinish(_ didFinish: (@MainActor () -> Void)?) { + self.testUsageLoadDidFinish = didFinish + } + + func setTestingUsageRetryDidExhaust(_ didExhaust: (@MainActor () -> Void)?) { + self.testUsageRetryDidExhaust = didExhaust + } + + func setTestingUsageRetryInterval(_ seconds: TimeInterval) { + self.usageRetryIntervalSeconds = seconds + } + + func refreshUsageCacheForTesting(force: Bool) async { + await self.refreshUsageCache(force: force) + } + + var testingCachedUsageSummary: GatewayUsageSummary? { + self.cachedUsageSummary + } + + var testingUsageCacheUpdatedAt: Date? { + self.usageCacheUpdatedAt + } + func setTestingCostUsageSummary(_ summary: GatewayCostUsageSummary?, errorText: String? = nil) { self.cachedCostSummary = summary self.cachedCostErrorText = errorText diff --git a/apps/macos/Sources/OpenClaw/UsageData.swift b/apps/macos/Sources/OpenClaw/UsageData.swift index f97d0bd3d49a..e2a238230582 100644 --- a/apps/macos/Sources/OpenClaw/UsageData.swift +++ b/apps/macos/Sources/OpenClaw/UsageData.swift @@ -16,6 +16,13 @@ struct GatewayUsageProvider: Codable { struct GatewayUsageSummary: Codable { let updatedAt: Double let providers: [GatewayUsageProvider] + let refreshing: Bool? + + init(updatedAt: Double, providers: [GatewayUsageProvider], refreshing: Bool? = nil) { + self.updatedAt = updatedAt + self.providers = providers + self.refreshing = refreshing + } } struct UsageRow: Identifiable { diff --git a/apps/macos/Tests/OpenClawIPCTests/MacGatewayChatTransportMappingTests.swift b/apps/macos/Tests/OpenClawIPCTests/MacGatewayChatTransportMappingTests.swift index 5737d3fa72d8..3eced12e03e3 100644 --- a/apps/macos/Tests/OpenClawIPCTests/MacGatewayChatTransportMappingTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/MacGatewayChatTransportMappingTests.swift @@ -10,6 +10,7 @@ struct MacGatewayChatTransportMappingTests { #expect(GatewayConnection.operatorClientCaps == [ OpenClawGatewayClientCapability.agentKind, OpenClawGatewayClientCapability.inlineWidgets, + OpenClawGatewayClientCapability.usageRefreshing, ]) } diff --git a/apps/macos/Tests/OpenClawIPCTests/MenuSessionsInjectorTests.swift b/apps/macos/Tests/OpenClawIPCTests/MenuSessionsInjectorTests.swift index 9739e78b024e..beac3da19fe3 100644 --- a/apps/macos/Tests/OpenClawIPCTests/MenuSessionsInjectorTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/MenuSessionsInjectorTests.swift @@ -175,6 +175,125 @@ struct MenuSessionsInjectorTests { #expect(usageCostItem?.submenu?.delegate == nil) } + @Test func `cold incomplete usage converges without starting the cache ttl`() async { + let injector = MenuSessionsInjector() + injector.setTestingControlChannelConnected(true) + injector.setTestingUsageRetryInterval(0) + let events = UsageLoadEvents() + injector.setTestingUsageLoadDidFinish { events.finished() } + var calls = 0 + injector.setTestingUsageLoader { + calls += 1 + if calls == 1 { + return GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true) + } + return GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false) + } + + await injector.refreshUsageCacheForTesting(force: true) + #expect(injector.testingUsageCacheUpdatedAt == nil) + #expect(await events.waitFor(count: 2)) + #expect(calls == 2) + #expect(injector.testingUsageCacheUpdatedAt != nil) + } + + @Test func `incomplete usage retry is bounded`() async { + let injector = MenuSessionsInjector() + injector.setTestingControlChannelConnected(true) + injector.setTestingUsageRetryInterval(0) + let events = UsageLoadEvents() + injector.setTestingUsageLoadDidFinish { events.finished() } + injector.setTestingUsageRetryDidExhaust { events.exhausted() } + injector.setTestingUsageLoader { + GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true) + } + + await injector.refreshUsageCacheForTesting(force: true) + #expect(await events.waitFor(count: 4)) + #expect(await events.waitForExhaustion()) + #expect(injector.testingUsageCacheUpdatedAt == nil) + } + + @Test func `stalled usage keeps a visible menu section`() async { + let injector = MenuSessionsInjector() + injector.setTestingControlChannelConnected(true) + injector.setTestingUsageRetryInterval(0) + let events = UsageLoadEvents() + injector.setTestingUsageLoadDidFinish { events.finished() } + injector.setTestingUsageRetryDidExhaust { events.exhausted() } + + // An operator with no usage providers still gets no usage section. + injector.setTestingUsageSummary( + GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false)) + let quiet = Self.makeMenuShell() + injector.injectForTesting(into: quiet) + let quietItems = quiet.items.count(where: { $0.tag == 9_415_557 }) + + injector.setTestingUsageLoader { + GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true) + } + await injector.refreshUsageCacheForTesting(force: true) + #expect(await events.waitForExhaustion()) + + // Spent budget with the marker still set: separator, header, and the + // stalled row, never the silent menu an empty provider list produces. + let stalled = Self.makeMenuShell() + injector.injectForTesting(into: stalled) + #expect(stalled.items.count(where: { $0.tag == 9_415_557 }) == quietItems + 3) + } + + private static func makeMenuShell() -> NSMenu { + let menu = NSMenu() + menu.addItem(NSMenuItem(title: "Header", action: nil, keyEquivalent: "")) + menu.addItem(.separator()) + menu.addItem(NSMenuItem(title: "Send Heartbeats", action: nil, keyEquivalent: "")) + menu.addItem(.separator()) + menu.addItem(NSMenuItem(title: "Settings…", action: nil, keyEquivalent: "")) + return menu + } + + @Test func `late usage result from a replaced gateway is ignored`() async { + let injector = MenuSessionsInjector() + injector.setTestingControlChannelConnected(true) + let loads = DeferredUsageLoads() + injector.setTestingUsageLoader { try await loads.load() } + + let first = Task { await injector.refreshUsageCacheForTesting(force: true) } + #expect(await loads.waitForRequests(count: 1)) + let second = Task { await injector.refreshUsageCacheForTesting(force: true) } + #expect(await loads.waitForRequests(count: 2)) + + loads.complete( + at: 1, + with: GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false)) + await second.value + loads.complete( + at: 0, + with: GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false)) + await first.value + + #expect(injector.testingCachedUsageSummary?.updatedAt == 2) + } + + @Test func `fresh no-op does not invalidate a forced usage load`() async { + let injector = MenuSessionsInjector() + injector.setTestingControlChannelConnected(true) + injector.setTestingUsageSummary( + GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false)) + let loads = DeferredUsageLoads() + injector.setTestingUsageLoader { try await loads.load() } + + let forced = Task { await injector.refreshUsageCacheForTesting(force: true) } + #expect(await loads.waitForRequests(count: 1)) + await injector.refreshUsageCacheForTesting(force: false) + loads.complete( + at: 0, + with: GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false)) + await forced.value + + #expect(injector.testingCachedUsageSummary?.updatedAt == 2) + } + @Test func `status text keeps useful error detail`() { let injector = MenuSessionsInjector() let longError = """ @@ -237,3 +356,85 @@ struct MenuSessionsInjectorTests { connected: connected) } } + +@MainActor +private final class UsageLoadEvents { + private struct Snapshot: Sendable { + let completed: Int + let exhausted: Bool + } + + private var completed = 0 + private var didExhaust = false + private let stream: AsyncStream + private let continuation: AsyncStream.Continuation + + init() { + (self.stream, self.continuation) = AsyncStream.makeStream( + of: Snapshot.self, + bufferingPolicy: .bufferingNewest(1)) + } + + func finished() { + self.completed += 1 + self.publish() + } + + func exhausted() { + self.didExhaust = true + self.publish() + } + + func waitFor(count: Int) async -> Bool { + if self.completed >= count { return true } + return await self.wait { $0.completed >= count } + } + + func waitForExhaustion() async -> Bool { + if self.didExhaust { return true } + return await self.wait { $0.exhausted } + } + + private func publish() { + self.continuation.yield(Snapshot(completed: self.completed, exhausted: self.didExhaust)) + } + + private func wait(_ predicate: @escaping @Sendable (Snapshot) -> Bool) async -> Bool { + let stream = self.stream + return await withTaskGroup(of: Bool.self) { group in + group.addTask { + for await snapshot in stream where predicate(snapshot) { return true } + return false + } + group.addTask { + try? await Task.sleep(nanoseconds: 5_000_000_000) + return false + } + let result = await group.next() ?? false + group.cancelAll() + return result + } + } +} + +@MainActor +private final class DeferredUsageLoads { + private var continuations: [CheckedContinuation] = [] + + func load() async throws -> GatewayUsageSummary { + await withCheckedContinuation { continuation in + self.continuations.append(continuation) + } + } + + func waitForRequests(count: Int) async -> Bool { + for _ in 0..<100 where self.continuations.count < count { + await Task.yield() + } + return self.continuations.count >= count + } + + func complete(at index: Int, with summary: GatewayUsageSummary) { + self.continuations[index].resume(returning: summary) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift index 599e3a409023..db4a6e396094 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayConnectOptions.swift @@ -3,6 +3,7 @@ import OpenClawProtocol public enum OpenClawGatewayClientCapability { public static let agentKind = "agent-kind" public static let inlineWidgets = "inline-widgets" + public static let usageRefreshing = "usage-refreshing" } public struct GatewayConnectOptions: Sendable { diff --git a/docs/gateway/clients.md b/docs/gateway/clients.md index 8d5295494e40..7b45638f965b 100644 --- a/docs/gateway/clients.md +++ b/docs/gateway/clients.md @@ -97,9 +97,15 @@ const caps = [GATEWAY_CLIENT_CAPS.TOOL_EVENTS]; The current registry contains `approvals`, `exec-approvals`, `inline-widgets`, `run-tool-bindings`, `session-scoped-events`, `plugin-approvals`, -`task-suggestions`, `terminal-offset-seq`, `tool-events`, and `ui-commands`. +`task-suggestions`, `terminal-offset-seq`, `tool-events`, `ui-commands`, and +`usage-refreshing`. Advertise only capabilities the client actually implements. +`usage-refreshing` allows a cold `usage.status` request to return immediately +with `refreshing: true` and an empty provider list. A client advertising it must +keep that payload cache-cold and refetch on a short bounded schedule. Other +clients retain the blocking cold read. + `tool-events` gates live tool-execution streaming. The Gateway registers only connections that advertise this capability as recipients for a run's structured diff --git a/docs/gateway/protocol.md b/docs/gateway/protocol.md index 5357ac4d4a15..a7e6d37b0820 100644 --- a/docs/gateway/protocol.md +++ b/docs/gateway/protocol.md @@ -535,7 +535,7 @@ methods. Treat this as feature discovery, not a full enumeration of - `models.list` returns the runtime-allowed model catalog. See "`models.list` views" below. - - `usage.status` returns provider usage windows/remaining quota summaries. + - `usage.status` returns provider usage windows/remaining quota summaries. Clients advertising `usage-refreshing` receive an immediate `refreshing: true` placeholder on a cold cache and must refetch on a bounded schedule; other callers block for the cold provider read. - `usage.cost` returns aggregated cost usage summaries for a date range. Pass `agentId` for one agent, or `agentScope: "all"` to aggregate configured agents. - `doctor.memory.status` returns vector-memory / cached embedding readiness for the active default agent workspace. Pass `{ "probe": true }` or `{ "deep": true }` only for an explicit live embedding provider ping. Pass `{ "agentId": "agent-id" }` to scope Dreaming store stats to one agent workspace; omitting it aggregates configured Dreaming workspaces. - `doctor.memory.dreamDiary`, `doctor.memory.backfillDreamDiary`, `doctor.memory.resetDreamDiary`, `doctor.memory.resetGroundedShortTerm`, `doctor.memory.repairDreamingArtifacts`, and `doctor.memory.dedupeDreamDiary` accept optional `{ "agentId": "agent-id" }`; omitted, they operate on the configured default agent workspace. diff --git a/packages/gateway-protocol/src/client-info.ts b/packages/gateway-protocol/src/client-info.ts index c803ce129923..a2ba6f8f37eb 100644 --- a/packages/gateway-protocol/src/client-info.ts +++ b/packages/gateway-protocol/src/client-info.ts @@ -90,6 +90,7 @@ export const GATEWAY_CLIENT_CAPS = { TERMINAL_OFFSET_SEQ: "terminal-offset-seq", TOOL_EVENTS: "tool-events", UI_COMMANDS: "ui-commands", + USAGE_REFRESHING: "usage-refreshing", } as const; /** Optional capability advertised by clients during gateway handshake. */ diff --git a/src/gateway/board-host-tools.ts b/src/gateway/board-host-tools.ts index bfbccb41f8ef..9253ca907f8b 100644 --- a/src/gateway/board-host-tools.ts +++ b/src/gateway/board-host-tools.ts @@ -15,7 +15,8 @@ type GatewayHandlerInvocation = Parameters[0]; const BOARD_DATA_HANDLERS: Record = { "sessions.list": sessionReadHandlers["sessions.list"]!, - "usage.status": usageHandlers["usage.status"]!, + // Board reads are one-shot and cannot converge an incomplete marker. + "usage.status": (invocation) => usageHandlers["usage.status"]!({ ...invocation, client: null }), "usage.cost": usageHandlers["usage.cost"]!, "cron.list": cronHandlers["cron.list"]!, "cron.status": cronHandlers["cron.status"]!, diff --git a/src/gateway/server-methods/models-auth-status-usage-cache.ts b/src/gateway/server-methods/models-auth-status-usage-cache.ts index 2d8c8eefd897..a6f188291175 100644 --- a/src/gateway/server-methods/models-auth-status-usage-cache.ts +++ b/src/gateway/server-methods/models-auth-status-usage-cache.ts @@ -197,8 +197,9 @@ function scheduleProviderUsageRefresh(params: { return usage; }) .catch((err: unknown) => { - // Usage is auxiliary and stale data remains valid. Keep failures visible - // without delaying fresh auth-health responses. + // Usage is auxiliary and stale data remains valid. A failed refresh + // publishes nothing, so a capable client keeps seeing the incomplete + // marker and reports it once its retry budget is spent. log.debug( `usage refresh failed: providers=${params.providerIds.join(",")} error=${formatForLog(err)}`, ); @@ -225,26 +226,48 @@ type ProviderUsageCacheParams = { agentDir: string; configRef: OpenClawConfig; credentialKey: string; + coldRead?: "refresh-marker"; forceRefresh?: boolean; providerIds: UsageProviderId[]; now: number; }; +/** + * Credential identity without the selection bookkeeping. `usageStats` is stamped + * by every successful run (`markAuthProfileSuccess`), so treating it as an + * identity change discards a good snapshot several times a minute on a busy + * gateway — and a capable client would then receive a cold marker on every poll. + * It still belongs in the refresh trigger because it can reorder profiles. + */ +function providerUsageCredentialIdentity(credentialKey: string): string { + try { + const { usageStats: _selectionBookkeeping, ...identity } = JSON.parse(credentialKey) as Record< + string, + unknown + >; + return JSON.stringify(identity); + } catch { + return credentialKey; + } +} + function resolveProviderUsageCacheRead(params: ProviderUsageCacheParams) { const providerIds = params.providerIds.toSorted(); const providerKey = providerUsageCacheKey(providerIds); const credentialKey = scopeProviderUsageCredentialKey(params.credentialKey, providerIds); + const credentialIdentity = providerUsageCredentialIdentity(credentialKey); const cached = usageCacheByAgentId.get(params.agentId); const matching = cached?.agentDir === params.agentDir && cached.configRef === params.configRef && - cached.credentialKey === credentialKey && + providerUsageCredentialIdentity(cached.credentialKey) === credentialIdentity && cached.providerKey === providerKey ? cached : undefined; const needsRefresh = params.forceRefresh === true || !matching || + matching.credentialKey !== credentialKey || params.now - matching.refreshedAt >= USAGE_CACHE_TTL_MS; return { credentialKey, matching, needsRefresh, providerIds, providerKey }; } @@ -273,7 +296,7 @@ export function readProviderUsageStaleWhileRevalidate( return matching?.usageByProvider ?? new Map(); } -/** Returns cached provider usage, awaiting only a cold miss and refreshing stale data in place. */ +/** Returns cached provider usage while network refreshes run in the background for capable clients. */ async function loadProviderUsageSummaryStaleWhileRevalidate( params: ProviderUsageCacheParams, ): Promise { @@ -298,12 +321,17 @@ async function loadProviderUsageSummaryStaleWhileRevalidate( void refresh.catch(() => {}); return matching.summary; } - return await refresh; + if (params.coldRead !== "refresh-marker") { + return await refresh; + } + void refresh.catch(() => {}); + return { updatedAt: params.now, providers: [], refreshing: true }; } /** Shares the models.authStatus cache contract with the unscoped usage.status RPC. */ export async function loadUsageStatusStaleWhileRevalidate(params: { config: OpenClawConfig; + coldRead?: "refresh-marker"; now?: number; }): Promise { const agentId = resolveLegacyInheritedAuthAgentId(params.config); @@ -342,6 +370,7 @@ export async function loadUsageStatusStaleWhileRevalidate(params: { store, }), providerIds, + coldRead: params.coldRead, now: params.now ?? Date.now(), }); } diff --git a/src/gateway/server-methods/optional-model-catalog.test.ts b/src/gateway/server-methods/optional-model-catalog.test.ts index 2e2232a7d994..7383388d5cda 100644 --- a/src/gateway/server-methods/optional-model-catalog.test.ts +++ b/src/gateway/server-methods/optional-model-catalog.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from "vitest"; -import { readPreparedServerMethodModelCatalog } from "./optional-model-catalog.js"; +import { registerGatewayModelCatalogPrivateAccess } from "../server-model-catalog-auth.js"; +import { + loadOptionalServerMethodModelCatalogSnapshot, + readPreparedServerMethodModelCatalog, +} from "./optional-model-catalog.js"; import type { GatewayRequestContext } from "./types.js"; describe("readPreparedServerMethodModelCatalog", () => { @@ -20,3 +24,38 @@ describe("readPreparedServerMethodModelCatalog", () => { expect(loadGatewayModelCatalog).not.toHaveBeenCalled(); }); }); + +describe("loadOptionalServerMethodModelCatalogSnapshot", () => { + it("uses a prepared snapshot before cold discovery", async () => { + const snapshot = { + agentId: "work", + agentDir: "/tmp/work-agent", + catalogComplete: true, + workspaceDir: "/tmp/work", + config: {}, + entries: [], + routeVariants: [], + authModes: {}, + authStore: { version: 1, profiles: {} }, + authMaterializations: [], + metadataSnapshot: { index: { plugins: [] }, plugins: [] } as never, + }; + const loadGatewayModelCatalogSnapshot = vi.fn(); + const context = { + loadGatewayModelCatalogSnapshot, + logGateway: { debug: vi.fn() }, + } as unknown as GatewayRequestContext; + const readPrepared = vi.fn(async () => snapshot); + registerGatewayModelCatalogPrivateAccess(loadGatewayModelCatalogSnapshot, { + loadDeferred: vi.fn(), + readPrepared, + }); + + await expect( + loadOptionalServerMethodModelCatalogSnapshot(context, "chat.startup", { + loadParams: { agentId: "work" }, + }), + ).resolves.toBe(snapshot); + expect(loadGatewayModelCatalogSnapshot).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/server-methods/optional-model-catalog.ts b/src/gateway/server-methods/optional-model-catalog.ts index 355df2cfd2ca..bd09b57dd1b7 100644 --- a/src/gateway/server-methods/optional-model-catalog.ts +++ b/src/gateway/server-methods/optional-model-catalog.ts @@ -1,6 +1,7 @@ // Optional model-catalog access gives session/tool methods metadata when ready // while keeping provider discovery out of ordinary request hot paths. import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js"; +import { readPreparedCatalogSnapshot } from "../server-model-catalog-auth.js"; import type { GatewayModelCatalogSnapshot } from "../server-model-catalog.types.js"; import type { GatewayRequestContext } from "./types.js"; @@ -74,7 +75,11 @@ export function startOptionalServerMethodModelCatalogSnapshotLoad( loadParams?: Parameters[0], ): OptionalServerMethodModelCatalogLoad { return startOptionalServerMethodModelCatalogValueLoad({ - load: () => context.loadGatewayModelCatalogSnapshot(loadParams), + load: async () => + // Prepared startup facts avoid provider discovery. The cold loader remains + // the fallback because it publishes the generation used by capability checks. + (await readPreparedCatalogSnapshot(context, loadParams)) ?? + (await context.loadGatewayModelCatalogSnapshot(loadParams)), normalize: normalizeOptionalModelCatalogSnapshot, }); } diff --git a/src/gateway/server-methods/usage.status-cache.test.ts b/src/gateway/server-methods/usage.status-cache.test.ts index ff8c1f7c054d..4f72d484a4a0 100644 --- a/src/gateway/server-methods/usage.status-cache.test.ts +++ b/src/gateway/server-methods/usage.status-cache.test.ts @@ -46,6 +46,8 @@ const config = { agents: { list: [{ id: "main", default: true }] }, } as OpenClawConfig; +const refreshingCapableClient = { connect: { caps: ["usage-refreshing"] } }; + function createStore(access = "access-one") { return { version: 1, @@ -61,7 +63,7 @@ function createStore(access = "access-one") { }; } -async function runUsageStatus() { +async function runUsageStatus(client?: unknown, getRuntimeConfig = () => config) { const respond = vi.fn(); await expectDefined( usageHandlers["usage.status"], @@ -69,7 +71,8 @@ async function runUsageStatus() { )({ respond, params: {}, - context: { getRuntimeConfig: () => config }, + context: { getRuntimeConfig }, + client: client === undefined ? refreshingCapableClient : client, } as unknown as Parameters<(typeof usageHandlers)["usage.status"]>[0]); expect(respond).toHaveBeenCalledTimes(1); expect(respond.mock.calls[0]?.[0]).toBe(true); @@ -114,7 +117,7 @@ describe("usage.status provider usage cache", () => { vi.restoreAllMocks(); }); - it("loads the cached provider snapshot from the exact runtime config", async () => { + function mockExactConfigProviderUsage() { mocks.loadProviderUsageSummary.mockImplementation(async (options) => ({ updatedAt: now, providers: @@ -129,15 +132,108 @@ describe("usage.status provider usage cache", () => { ] : [], })); + } - const result = (await runUsageStatus()) as { + it("loads the cached provider snapshot from the exact runtime config", async () => { + mockExactConfigProviderUsage(); + + // Clientless internal reads stay blocking, so the first response already + // carries the snapshot the exact-config loader produced. + const result = (await runUsageStatus(null)) as { providers: Array<{ accountEmail?: string }>; }; expect(result.providers[0]?.accountEmail).toBe("configured@example.com"); }); + it("hands the exact runtime config to the background refresh", async () => { + mockExactConfigProviderUsage(); + + await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true }); + await vi.waitFor(async () => { + expect( + (await runUsageStatus()) as { providers: Array<{ accountEmail?: string }> }, + ).toMatchObject({ providers: [{ accountEmail: "configured@example.com" }] }); + }); + }); + + it("returns a cold marker only to clients that can converge it", async () => { + let finish: ((value: { updatedAt: number; providers: never[] }) => void) | undefined; + mocks.loadProviderUsageSummary.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + + await expect(runUsageStatus()).resolves.toEqual({ + updatedAt: now, + providers: [], + refreshing: true, + }); + + const legacy = runUsageStatus({ connect: { caps: [] } }); + const pending = Symbol("pending"); + await expect( + Promise.race([ + legacy, + new Promise((resolve) => { + setTimeout(() => resolve(pending), 25); + }), + ]), + ).resolves.toBe(pending); + finish?.({ updatedAt: now, providers: [] }); + await expect(legacy).resolves.toEqual({ updatedAt: now, providers: [] }); + }); + + it("keeps serving usage while run bookkeeping churns the profile store", async () => { + await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true }); + await vi.waitFor(async () => { + expect(await runUsageStatus()).toMatchObject({ providers: [{ provider: "openai" }] }); + }); + + // Every completed agent run stamps usage bookkeeping on the selected profile + // (markAuthProfileSuccess). That must not read as a credential change, or a + // busy gateway hands capable clients a cold marker on every poll and they + // report "did not finish loading" while every refresh is succeeding. + store = { ...store, usageStats: { "openai:default": { lastUsed: now } } }; + now += 1; + + await expect(runUsageStatus()).resolves.toMatchObject({ + providers: [{ provider: "openai" }], + }); + }); + + it("keeps clientless internal reads blocking", async () => { + const result = (await runUsageStatus(null)) as { refreshing?: boolean; providers: unknown[] }; + expect(result.providers).toHaveLength(1); + expect(result.refreshing).toBeUndefined(); + }); + + it("keeps a failed refresh incomplete for capable clients and recovers", async () => { + mocks.loadProviderUsageSummary.mockRejectedValueOnce(new Error("provider stack down")); + await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true }); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + + // A failed refresh publishes nothing, so the payload stays marked incomplete + // and the client's bounded retry owns reporting it. The next attempt recovers. + await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true }); + await vi.waitFor(async () => { + expect((await runUsageStatus()) as { providers: unknown[] }).toMatchObject({ + providers: [expect.any(Object)], + }); + }); + }); + it("reuses byte-identical results within 60s and refreshes stale data in the background", async () => { - const first = await runUsageStatus(); + expect(await runUsageStatus()).toMatchObject({ refreshing: true }); + const first = await vi.waitFor(async () => { + const value = (await runUsageStatus()) as { providers: unknown[]; refreshing?: boolean }; + expect(value.providers).toHaveLength(1); + return value; + }); + expect(first.refreshing).toBeUndefined(); const repeated = await runUsageStatus(); expect(JSON.stringify(repeated)).toBe(JSON.stringify(first)); @@ -157,10 +253,33 @@ describe("usage.status provider usage cache", () => { expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2); }); + it("invalidates cached usage when the runtime config changes", async () => { + const configFor = (baseUrl: string) => + ({ ...config, models: { providers: { openai: { baseUrl } } } }) as unknown as OpenClawConfig; + const first = configFor("https://one.example/v1"); + expect(await runUsageStatus(refreshingCapableClient, () => first)).toMatchObject({ + refreshing: true, + }); + await vi.waitFor(async () => { + expect( + ((await runUsageStatus(refreshingCapableClient, () => first)) as { providers: unknown[] }) + .providers, + ).toHaveLength(1); + }); + expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1); + + const second = configFor("https://two.example/v1"); + expect(await runUsageStatus(refreshingCapableClient, () => second)).toMatchObject({ + refreshing: true, + }); + await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2)); + }); + it("shares the raw snapshot with models.authStatus and invalidates on credential rotation", async () => { - await runUsageStatus(); + expect(await runUsageStatus()).toMatchObject({ refreshing: true }); const agentId = resolveDefaultAgentId(config); const agentDir = resolveAgentDir(config, agentId); + await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1)); const usage = readProviderUsageStaleWhileRevalidate({ agentId, agentDir, @@ -177,10 +296,13 @@ describe("usage.status provider usage cache", () => { expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1); store.profiles["openai:default"].access = "access-two"; - const rotated = (await runUsageStatus()) as { - providers: Array<{ windows: Array<{ usedPercent: number }> }>; - }; - expect(rotated.providers[0]?.windows[0]?.usedPercent).toBe(20); + expect(await runUsageStatus()).toMatchObject({ refreshing: true }); + await vi.waitFor(async () => { + const rotated = (await runUsageStatus()) as { + providers: Array<{ windows: Array<{ usedPercent: number }> }>; + }; + expect(rotated.providers[0]?.windows[0]?.usedPercent).toBe(20); + }); expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2); }); }); diff --git a/src/gateway/server-methods/usage.ts b/src/gateway/server-methods/usage.ts index 7a1b0a8a35b8..c50468358e83 100644 --- a/src/gateway/server-methods/usage.ts +++ b/src/gateway/server-methods/usage.ts @@ -3,6 +3,10 @@ import fs from "node:fs"; import { expectDefined } from "@openclaw/normalization-core"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { + GATEWAY_CLIENT_CAPS, + hasGatewayClientCap, +} from "../../../packages/gateway-protocol/src/client-info.js"; import { ErrorCodes, errorShape, @@ -1133,9 +1137,21 @@ export const testApi = { export type { SessionUsageEntry, SessionsUsageAggregates, SessionsUsageResult }; export const usageHandlers: GatewayRequestHandlers = { - "usage.status": async ({ respond, context }) => { + "usage.status": async ({ respond, context, client }) => { + // Only clients with bounded retry machinery may receive an incomplete cold result. + // In-process dispatch reuses the originating request's client, capabilities + // included, so a plugin proxying this method inside a capable UI request + // would inherit the marker without any way to converge it. Such a caller + // must pass a capless client, the way board bindings force `client: null`. + const coldRead = hasGatewayClientCap( + client?.connect?.caps, + GATEWAY_CLIENT_CAPS.USAGE_REFRESHING, + ) + ? ("refresh-marker" as const) + : undefined; const summary = await loadUsageStatusStaleWhileRevalidate({ config: context.getRuntimeConfig(), + coldRead, }); respond(true, summary, undefined); }, diff --git a/src/gateway/server-model-catalog-auth.ts b/src/gateway/server-model-catalog-auth.ts index 9b946ca2fb55..309ff02fc366 100644 --- a/src/gateway/server-model-catalog-auth.ts +++ b/src/gateway/server-model-catalog-auth.ts @@ -66,5 +66,13 @@ export async function readPreparedCatalog( context: Pick, agentId: string, ): Promise { - return await requirePrivateAccess(context).readPrepared({ agentId }); + return await readPreparedCatalogSnapshot(context, { agentId }); +} + +/** Reads a prepared owner snapshot without exposing private auth facts on the request context. */ +export async function readPreparedCatalogSnapshot( + context: Pick, + params?: Omit, +): Promise { + return await requirePrivateAccess(context).readPrepared(params); } diff --git a/src/infra/provider-usage.types.ts b/src/infra/provider-usage.types.ts index 84080d63117f..24917de7e38b 100644 --- a/src/infra/provider-usage.types.ts +++ b/src/infra/provider-usage.types.ts @@ -86,6 +86,8 @@ export type ProviderUsageSnapshot = { export type UsageSummary = { updatedAt: number; providers: ProviderUsageSnapshot[]; + /** A background refresh owns the real values; an empty list is incomplete. */ + refreshing?: boolean; }; /** Normalized provider id. Usage providers are discovered from plugin hooks at runtime. */ diff --git a/ui/src/api/gateway.node.test.ts b/ui/src/api/gateway.node.test.ts index d182515cd457..2ebe9b5d1927 100644 --- a/ui/src/api/gateway.node.test.ts +++ b/ui/src/api/gateway.node.test.ts @@ -471,6 +471,7 @@ describe("GatewayBrowserClient", () => { GATEWAY_CLIENT_CAPS.TOOL_EVENTS, GATEWAY_CLIENT_CAPS.INLINE_WIDGETS, GATEWAY_CLIENT_CAPS.UI_COMMANDS, + GATEWAY_CLIENT_CAPS.USAGE_REFRESHING, ]); expect(connectFrame.params?.scopes).toEqual([...CONTROL_UI_OPERATOR_SCOPES]); }); diff --git a/ui/src/api/gateway.ts b/ui/src/api/gateway.ts index 4a0f879eb5d4..acfef52110df 100644 --- a/ui/src/api/gateway.ts +++ b/ui/src/api/gateway.ts @@ -483,6 +483,7 @@ export class GatewayBrowserClient { GATEWAY_CLIENT_CAPS.TOOL_EVENTS, GATEWAY_CLIENT_CAPS.INLINE_WIDGETS, GATEWAY_CLIENT_CAPS.UI_COMMANDS, + GATEWAY_CLIENT_CAPS.USAGE_REFRESHING, ], auth: buildGatewayConnectAuth(selectedAuth), userAgent: navigator.userAgent, diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index d2101ffd2b0f..46f4f2d060bb 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -4369,6 +4369,7 @@ export const en: TranslationMap = { providerUsage: { title: "Provider plans & billing", subtitle: "Live plan, quota, balance, and budget data reported by configured providers.", + stalled: "Provider usage did not finish loading. Refresh to retry.", balance: "Balance", spend: "Usage", budget: "Budget", diff --git a/ui/src/lib/incomplete-usage-retry.test.ts b/ui/src/lib/incomplete-usage-retry.test.ts new file mode 100644 index 000000000000..e8b67577c8fd --- /dev/null +++ b/ui/src/lib/incomplete-usage-retry.test.ts @@ -0,0 +1,73 @@ +// @vitest-environment jsdom +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { IncompleteUsageRetry, isUsageIncomplete } from "./incomplete-usage-retry.ts"; + +describe("IncompleteUsageRetry", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it("recognizes only an explicit incomplete marker", () => { + expect(isUsageIncomplete({ refreshing: true })).toBe(true); + expect(isUsageIncomplete({ refreshing: false })).toBe(false); + expect(isUsageIncomplete(null)).toBe(false); + }); + + it("reports exhaustion once the three delayed attempts are spent", () => { + const retry = vi.fn(); + const policy = new IncompleteUsageRetry({ retry }); + for (let attempt = 0; attempt < 3; attempt += 1) { + expect(policy.observe(true)).toBe("retrying"); + vi.advanceTimersByTime(5_000); + } + expect(retry).toHaveBeenCalledTimes(3); + // Nothing schedules another attempt, so the page owns the outcome from here. + expect(policy.observe(true)).toBe("exhausted"); + vi.advanceTimersByTime(5_000); + expect(retry).toHaveBeenCalledTimes(3); + }); + + it("resets on completion or connection replacement", () => { + const retry = vi.fn(); + const policy = new IncompleteUsageRetry({ retry }); + const first = {}; + for (let attempt = 0; attempt < 4; attempt += 1) { + policy.observe(true, first); + vi.advanceTimersByTime(5_000); + } + expect(retry).toHaveBeenCalledTimes(3); + + policy.observe(true, {}); + vi.advanceTimersByTime(5_000); + expect(retry).toHaveBeenCalledTimes(4); + expect(policy.observe(false)).toBe("complete"); + }); + + it("cancels timers on connection replacement and disposal", () => { + const retry = vi.fn(); + const policy = new IncompleteUsageRetry({ retry }); + policy.observe(true, {}); + policy.useConnection({}); + vi.advanceTimersByTime(5_000); + expect(retry).not.toHaveBeenCalled(); + + policy.observe(true); + policy.dispose(); + vi.advanceTimersByTime(5_000); + expect(retry).not.toHaveBeenCalled(); + }); + + it("lets an independent cycle retry after exhaustion without poll self-rearming", () => { + const retry = vi.fn(); + const policy = new IncompleteUsageRetry({ retry }); + for (let attempt = 0; attempt < 4; attempt += 1) { + policy.observe(true); + vi.advanceTimersByTime(5_000); + } + expect(retry).toHaveBeenCalledTimes(3); + + policy.startCycle(); + expect(policy.observe(true)).toBe("retrying"); + vi.advanceTimersByTime(5_000); + expect(retry).toHaveBeenCalledTimes(4); + }); +}); diff --git a/ui/src/lib/incomplete-usage-retry.ts b/ui/src/lib/incomplete-usage-retry.ts new file mode 100644 index 000000000000..02dd72e62004 --- /dev/null +++ b/ui/src/lib/incomplete-usage-retry.ts @@ -0,0 +1,85 @@ +const INCOMPLETE_USAGE_RETRY_MS = 5_000; +const INCOMPLETE_USAGE_RETRY_LIMIT = 3; + +type IncompleteUsageRetryOptions = { + retry: () => void; + retryMs?: number; + limit?: number; +}; + +type UsageRetryHost = { + addController: (controller: { hostDisconnected: () => void }) => void; +}; + +/** Closed convergence state: an incomplete payload is never a rendered answer. */ +export type UsageRetryState = "complete" | "retrying" | "exhausted"; + +export function isUsageIncomplete(usage: { refreshing?: boolean } | null | undefined): boolean { + return usage?.refreshing === true; +} + +export function createUsageRetry( + host: UsageRetryHost, + retry: () => void, + options?: Omit, +): IncompleteUsageRetry { + const controller = new IncompleteUsageRetry({ retry, ...options }); + host.addController({ hostDisconnected: () => controller.dispose() }); + return controller; +} + +/** Keeps incomplete usage cache-cold while bounding automatic convergence attempts. */ +export class IncompleteUsageRetry { + private timer: number | null = null; + private attempts = 0; + private connection: unknown; + + constructor(private readonly options: IncompleteUsageRetryOptions) {} + + observe(incomplete: boolean, connection?: unknown): UsageRetryState { + this.useConnection(connection); + this.clear(); + if (!incomplete) { + this.attempts = 0; + return "complete"; + } + if (this.attempts >= (this.options.limit ?? INCOMPLETE_USAGE_RETRY_LIMIT)) { + // Nothing will converge this payload on its own, so the caller has to + // report it. Rendering the empty provider list as a loaded answer is the + // silent-failure this marker exists to avoid. + return "exhausted"; + } + this.attempts += 1; + this.timer = window.setTimeout(() => { + this.timer = null; + this.options.retry(); + }, this.options.retryMs ?? INCOMPLETE_USAGE_RETRY_MS); + return "retrying"; + } + + /** Starts a user/lifecycle-owned refresh cycle without letting poll callbacks rearm it. */ + startCycle(): void { + this.attempts = 0; + this.clear(); + } + + useConnection(connection: unknown): void { + if (connection === this.connection) { + return; + } + this.connection = connection; + this.startCycle(); + } + + dispose(): void { + this.clear(); + } + + private clear(): void { + if (this.timer === null) { + return; + } + window.clearTimeout(this.timer); + this.timer = null; + } +} diff --git a/ui/src/pages/model-providers/model-providers-page.test.ts b/ui/src/pages/model-providers/model-providers-page.test.ts index 7842234a81e3..8ecabbfb1b39 100644 --- a/ui/src/pages/model-providers/model-providers-page.test.ts +++ b/ui/src/pages/model-providers/model-providers-page.test.ts @@ -27,6 +27,7 @@ type ModelProvidersPageTestElement = HTMLElement & { probe: (cardId: string, providers: string[]) => Promise; probeResults: Record; routeData: ModelProvidersRouteData | undefined; + requestUpdate: () => void; saveDefaultModels: () => Promise; saveKey: (provider: string, configKey: string) => Promise; selectedAgentId: string; @@ -53,6 +54,7 @@ function createHarness(initialScopeId: string) { }); return () => releaseAuthStatus?.(); }; + let usageStatus: unknown = { updatedAt: 1, providers: [] }; const request = vi.fn(async (method: string): Promise => { switch (method) { case "models.authStatus": { @@ -74,7 +76,7 @@ function createHarness(initialScopeId: string) { case "config.get": return { config: {}, hash: "hash" }; case "usage.status": - return { updatedAt: 1, providers: [] }; + return usageStatus; case "sessions.usage": return { aggregates: { byProvider: [] } }; default: @@ -170,6 +172,9 @@ function createHarness(initialScopeId: string) { request, runtimeConfig, snapshot, + setUsageStatus: (value: unknown) => { + usageStatus = value; + }, }; } @@ -184,9 +189,62 @@ function appendPage(context: ApplicationContext) { afterEach(() => { document.body.replaceChildren(); + vi.useRealTimers(); vi.restoreAllMocks(); }); +describe("ModelProvidersPage usage convergence", () => { + it("restarts an exhausted retry cycle on same-client reconnect", async () => { + vi.useFakeTimers(); + const harness = createHarness("main"); + harness.setUsageStatus({ updatedAt: 1, providers: [], refreshing: true }); + const page = appendPage(harness.context); + await page.updateComplete; + await vi.advanceTimersByTimeAsync(15_000); + + const usageCallsBeforeReconnect = harness.request.mock.calls.filter( + ([method]) => method === "usage.status", + ).length; + expect(usageCallsBeforeReconnect).toBe(4); + + harness.snapshot.phase = "offline"; + page.requestUpdate(); + await page.updateComplete; + harness.snapshot.phase = "connected"; + page.requestUpdate(); + await page.updateComplete; + await vi.advanceTimersByTimeAsync(0); + + expect(harness.request.mock.calls.filter(([method]) => method === "usage.status").length).toBe( + 5, + ); + }); + + it("replaces a pending pre-disconnect load before it can publish", async () => { + const harness = createHarness("main"); + harness.setUsageStatus({ updatedAt: 1, providers: [] }); + const releaseOldLoad = harness.deferNextAuthStatus(); + const page = appendPage(harness.context); + await page.updateComplete; + + harness.snapshot.phase = "offline"; + page.requestUpdate(); + await page.updateComplete; + harness.setUsageStatus({ updatedAt: 2, providers: [] }); + harness.snapshot.phase = "connected"; + page.requestUpdate(); + await page.updateComplete; + + await vi.waitFor(() => + expect( + harness.request.mock.calls.filter(([method]) => method === "usage.status").length, + ).toBe(2), + ); + releaseOldLoad(); + await vi.waitFor(() => expect(page.data?.providerUsage?.updatedAt).toBe(2)); + }); +}); + describe("ModelProvidersPage agent scope", () => { it("switches application ownership from the concrete agent picker", async () => { const { agentSelection, context } = createHarness("main"); diff --git a/ui/src/pages/model-providers/model-providers-page.ts b/ui/src/pages/model-providers/model-providers-page.ts index c47c597b3bce..71cd158f6590 100644 --- a/ui/src/pages/model-providers/model-providers-page.ts +++ b/ui/src/pages/model-providers/model-providers-page.ts @@ -15,6 +15,7 @@ import { t } from "../../i18n/index.ts"; import { normalizeAgentLabel } from "../../lib/agents/display.ts"; import { createGatewayConnectionLifecycle } from "../../lib/gateway-connection-lifecycle.ts"; import { isGatewayMethodAdvertised } from "../../lib/gateway-methods.ts"; +import { createUsageRetry, isUsageIncomplete } from "../../lib/incomplete-usage-retry.ts"; import { normalizeAgentId } from "../../lib/sessions/session-key.ts"; import { OpenClawLightDomElement } from "../../lit/openclaw-element.ts"; import { SubscriptionsController } from "../../lit/subscriptions-controller.ts"; @@ -44,6 +45,7 @@ import { buildProviderApiKeyPatch, DEFAULT_MODELS_REPLACE_PATHS, } from "./mutations.ts"; +import { mergeProbeResults } from "./probe-results.ts"; import { renderModelProviders, type ModelProviderRowMessage } from "./view.ts"; const MODEL_PROVIDERS_DOCS_URL = "https://docs.openclaw.ai/concepts/model-providers"; @@ -62,39 +64,6 @@ function isMissingMethodError(error: unknown): boolean { ); } -const PROBE_FAILURE_PRIORITY: readonly ModelsProbeResult["status"][] = [ - "auth", - "billing", - "rate_limit", - "timeout", - "format", - "no_model", - "unknown", -]; - -function mergeProbeResults(cardId: string, results: ModelsProbeResult[]): ModelsProbeResult { - if (results.length === 1) { - return results[0]!; - } - const status = results.some((result) => result.status === "ok") - ? "ok" - : (PROBE_FAILURE_PRIORITY.find((candidate) => - results.some((result) => result.status === candidate), - ) ?? "unknown"); - const error = results.find((result) => result.status === status)?.error; - return { - provider: cardId, - status, - ...(error ? { error } : {}), - results: results.flatMap((result) => - result.results.map((target) => ({ - ...target, - label: `${result.provider}: ${target.label}`, - })), - ), - }; -} - export class ModelProvidersPage extends OpenClawLightDomElement { @consume({ context: applicationContext, subscribe: true }) private context!: ApplicationContext; @@ -142,11 +111,9 @@ export class ModelProvidersPage extends OpenClawLightDomElement { signal, }).then((data) => ({ client, data })) : initialState, - onComplete: ({ client, data }) => { - this.data = data; - this.dataClient = client; - }, + onComplete: ({ client, data }) => this.applyLoadedData(data, client), }); + private readonly usageRetry = createUsageRetry(this, () => void this.refresh({ force: false })); private readonly subscriptions = new SubscriptionsController(this) .watch( () => this.context?.gateway, @@ -186,20 +153,34 @@ export class ModelProvidersPage extends OpenClawLightDomElement { if (changed.has("routeData") && this.routeData) { const selectedAgentId = this.resolveSelectedAgentId(); this.setSelectedAgent(selectedAgentId); - if ((this.routeData.agentId ?? "") === selectedAgentId) { - this.data = this.routeData.data; - this.dataClient = this.routeData.client; - } else { - this.data = null; - this.dataClient = null; - } + const matches = (this.routeData.agentId ?? "") === selectedAgentId; + this.applyLoadedData( + matches ? this.routeData.data : null, + matches ? this.routeData.client : null, + ); } } + private applyLoadedData(data: ModelProvidersData | null, client: GatewayBrowserClient | null) { + this.data = data; + this.dataClient = client; + // The connection epoch scopes the retry budget: a reconnect is a fresh + // Gateway cache generation and must not inherit the old attempt count. + // The "exhausted" state is deliberately dropped here: these cards render from + // models.authStatus and stay useful without usage, and this page's + // failed-usage notice is owned separately. Usage owns the visible outcome. + this.usageRetry.observe( + data !== null && isUsageIncomplete(data.providerUsage), + this.connectionLifecycle.epoch, + ); + } + override updated() { const snapshot = this.context.gateway.snapshot; - if (this.connectionLifecycle.transition(snapshot)) { - this.resetConnectionState(snapshot.client, snapshot.phase === "connected"); + const connected = snapshot.phase === "connected"; + const changed = this.connectionLifecycle.transition(snapshot); + if (changed) { + this.resetConnectionState(snapshot.client, connected); } if ( !this.context.agents.state.agentsList && @@ -208,6 +189,13 @@ export class ModelProvidersPage extends OpenClawLightDomElement { ) { void this.context.agents.ensureList(); } + if (changed && connected && snapshot.client) { + // A reconnect reuses the client object, so the staleness check below + // cannot recognize it as a new Gateway cache generation. Refetch here + // rather than relying on the reset above having nulled `data`. + void this.refresh({ force: false }); + return; + } if ( snapshot.phase !== "connected" || !snapshot.client || @@ -222,6 +210,9 @@ export class ModelProvidersPage extends OpenClawLightDomElement { } private resetConnectionState(client: GatewayBrowserClient | null, connected: boolean) { + this.usageRetry.useConnection(this.connectionLifecycle.epoch); + // A null run advances Task's call id, so a late pre-transition result cannot + // publish even when the underlying request ignores AbortSignal. void this.refreshTask.run([null, this.selectedAgentId, false]); this.busy = {}; this.messages = {}; diff --git a/ui/src/pages/model-providers/probe-results.ts b/ui/src/pages/model-providers/probe-results.ts new file mode 100644 index 000000000000..00b8d8bef5f6 --- /dev/null +++ b/ui/src/pages/model-providers/probe-results.ts @@ -0,0 +1,34 @@ +import type { ModelsProbeResult } from "../../api/types.ts"; + +const PROBE_FAILURE_PRIORITY: readonly ModelsProbeResult["status"][] = [ + "auth", + "billing", + "rate_limit", + "timeout", + "format", + "no_model", + "unknown", +]; + +export function mergeProbeResults(cardId: string, results: ModelsProbeResult[]): ModelsProbeResult { + if (results.length === 1) { + return results[0]!; + } + const status = results.some((result) => result.status === "ok") + ? "ok" + : (PROBE_FAILURE_PRIORITY.find((candidate) => + results.some((result) => result.status === candidate), + ) ?? "unknown"); + const error = results.find((result) => result.status === status)?.error; + return { + provider: cardId, + status, + ...(error ? { error } : {}), + results: results.flatMap((result) => + result.results.map((target) => ({ + ...target, + label: `${result.provider}: ${target.label}`, + })), + ), + }; +} diff --git a/ui/src/pages/usage/refresh-policy.test.ts b/ui/src/pages/usage/refresh-policy.test.ts index 27ceeff23b62..af712e85e0eb 100644 --- a/ui/src/pages/usage/refresh-policy.test.ts +++ b/ui/src/pages/usage/refresh-policy.test.ts @@ -71,4 +71,23 @@ describe("UsageRefreshPolicy", () => { policy.request("focus"); expect(reload).toHaveBeenCalledOnce(); }); + + it("restarts an exhausted retry budget for manual and focus cycles", () => { + const { policy, reload } = createPolicy(); + for (let attempt = 0; attempt < 4; attempt += 1) { + policy.markLoaded({ incomplete: true }); + vi.advanceTimersByTime(5_000); + } + expect(reload).toHaveBeenCalledTimes(3); + + policy.request("manual"); + policy.markLoaded({ incomplete: true }); + vi.advanceTimersByTime(5_000); + expect(reload).toHaveBeenCalledTimes(5); + + policy.request("focus"); + policy.markLoaded({ incomplete: true }); + vi.advanceTimersByTime(5_000); + expect(reload).toHaveBeenCalledTimes(7); + }); }); diff --git a/ui/src/pages/usage/refresh-policy.ts b/ui/src/pages/usage/refresh-policy.ts index c44fc143ad28..1155615a4081 100644 --- a/ui/src/pages/usage/refresh-policy.ts +++ b/ui/src/pages/usage/refresh-policy.ts @@ -1,3 +1,5 @@ +import { IncompleteUsageRetry, type UsageRetryState } from "../../lib/incomplete-usage-retry.ts"; + const USAGE_PAYLOAD_TTL_MS = 5 * 60_000; type UsageRefreshReason = "focus" | "manual" | "poll" | "reconnect"; @@ -39,22 +41,43 @@ export class UsageRefreshPolicy { private lastLoadedAtMs: number | null = null; private pendingAutomaticRefresh = false; private reloadPending = false; + private readonly incompleteUsageRetry = new IncompleteUsageRetry({ + retry: () => this.request("poll"), + }); constructor(private readonly options: UsageRefreshPolicyOptions) {} - setLastLoadedAtMs(value: number | null): void { - this.lastLoadedAtMs = value; + setLastLoadedAtMs( + value: number | null, + params?: { incomplete?: boolean; connection?: unknown }, + ): UsageRetryState { + return this.applyLoadState(value, params?.incomplete === true, params?.connection); } - markLoaded(): void { - this.lastLoadedAtMs = Date.now(); + markLoaded(params?: { incomplete?: boolean; connection?: unknown }): UsageRetryState { + return this.applyLoadState(Date.now(), params?.incomplete === true, params?.connection); } resetPayload(): void { - this.lastLoadedAtMs = null; + this.applyLoadState(null, false); this.reloadPending = false; } + dispose(): void { + this.incompleteUsageRetry.dispose(); + } + + private applyLoadState( + loadedAtMs: number | null, + incomplete: boolean, + connection?: unknown, + ): UsageRetryState { + const state = this.incompleteUsageRetry.observe(incomplete, connection); + // Incomplete provider usage must not start the TTL or focus/reconnect can skip recovery. + this.lastLoadedAtMs = state === "complete" ? loadedAtMs : null; + return state; + } + interrupt(): void { this.reloadPending ||= this.options.isLoading(); } @@ -86,6 +109,9 @@ export class UsageRefreshPolicy { lastLoadedAtMs: this.lastLoadedAtMs, }); if (decision === "fetch") { + if (reason !== "poll") { + this.incompleteUsageRetry.startCycle(); + } this.reload(); } } diff --git a/ui/src/pages/usage/types.ts b/ui/src/pages/usage/types.ts index 7aba5974dca9..549008ba3b14 100644 --- a/ui/src/pages/usage/types.ts +++ b/ui/src/pages/usage/types.ts @@ -1,3 +1,4 @@ +import type { CostUsageSummary } from "../../api/types.ts"; import type { PanelRefreshStatus } from "../../components/panel-refresh-status.ts"; // Control UI view renders usageTypes screen content. import type { @@ -14,6 +15,13 @@ export type UsageTotals = SessionsUsageTotals; export type CostDailyEntry = CostUsageDailyEntry; export type UsageAggregates = SessionsUsageResult["aggregates"]; +export type UsageTaskValue = { + epoch: object; + result: SessionsUsageResult; + costSummary: CostUsageSummary; + providerUsageSummary: ProviderUsageSummary | null; +}; + export type UsageColumnId = | "channel" | "agent" @@ -48,6 +56,8 @@ type UsageDataState = { costDaily: CostDailyEntry[]; cacheStatus: SessionsUsageResult["cacheStatus"]; providerUsage: ProviderUsageSummary["providers"]; + /** The gateway never converged the refresh; the empty list is not an answer. */ + providerUsageStalled: boolean; }; export type UsageFilterState = { diff --git a/ui/src/pages/usage/usage-page.test.ts b/ui/src/pages/usage/usage-page.test.ts index 55a94095a9c8..78024bbc745d 100644 --- a/ui/src/pages/usage/usage-page.test.ts +++ b/ui/src/pages/usage/usage-page.test.ts @@ -15,6 +15,8 @@ type TestUsagePage = HTMLElement & { usageTimeSeriesStatus: { error: string | null; hasLoaded: boolean; stale: boolean }; usageSessionLogs: SessionLogEntry[] | null; usageSessionLogsStatus: { error: string | null; hasLoaded: boolean; stale: boolean }; + providerUsageStalled: boolean; + routeData?: unknown; loadSessionTimeSeries: (sessionKey: string) => Promise; loadSessionLogs: (sessionKey: string) => Promise; render: () => unknown; @@ -78,6 +80,44 @@ afterEach(() => { }); describe("UsagePage detail requests", () => { + it("marks provider usage stalled once the retry budget is spent", async () => { + const client = { request: vi.fn(async () => ({})) } as unknown as GatewayBrowserClient; + const page = await createPage(client); + const gateway = page.context.gateway; + const routeDataAt = (loadedAtMs: number) => ({ + gateway, + gatewaySnapshot: gateway.snapshot, + client, + query: { + startDate: "2026-05-14", + endDate: "2026-05-14", + scope: "family" as const, + timeZone: "local" as const, + agentId: null, + }, + result: null, + costSummary: null, + providerUsageSummary: { updatedAt: 1, providers: [], refreshing: true }, + loadedAtMs, + error: null, + }); + + // Three bounded attempts, then the page owns the outcome. Without the wiring + // the empty provider list renders as a loaded answer. + for (let attempt = 0; attempt < 3; attempt += 1) { + page.routeData = routeDataAt(attempt); + await page.updateComplete; + expect(page.providerUsageStalled).toBe(false); + } + page.routeData = routeDataAt(3); + await page.updateComplete; + expect(page.providerUsageStalled).toBe(true); + + page.routeData = { ...routeDataAt(4), providerUsageSummary: { updatedAt: 2, providers: [] } }; + await page.updateComplete; + expect(page.providerUsageStalled).toBe(false); + }); + it("commits only the latest time-series selection", async () => { const first = deferred(); const second = deferred(); diff --git a/ui/src/pages/usage/usage-page.ts b/ui/src/pages/usage/usage-page.ts index d4a85d402376..d64a1365dd68 100644 --- a/ui/src/pages/usage/usage-page.ts +++ b/ui/src/pages/usage/usage-page.ts @@ -25,6 +25,7 @@ import { formatMissingOperatorReadScopeMessage, isMissingOperatorReadScopeError, } from "../../lib/gateway-errors.ts"; +import { isUsageIncomplete } from "../../lib/incomplete-usage-retry.ts"; import { requestSessionUsageLogs, requestSessionUsageTimeSeries, @@ -52,6 +53,7 @@ import { type SessionLogEntry, type SessionLogRole, type UsageProps, + type UsageTaskValue, } from "./types.ts"; import { renderUsage } from "./view.ts"; @@ -87,6 +89,7 @@ class UsagePage extends OpenClawLightDomElement { @state() private usageResult: SessionsUsageResult | null = null; @state() private usageCostSummary: CostUsageSummary | null = null; @state() private providerUsageSummary: ProviderUsageSummary | null = null; + @state() private providerUsageStalled = false; @state() private usageError: string | null = null; @state() private usageStartDate = currentLocalDate(); @state() private usageEndDate = currentLocalDate(); @@ -129,6 +132,8 @@ class UsagePage extends OpenClawLightDomElement { // Invalidation runs the Task with a null client to supersede stale completions. // Track real gateway work separately so that no-op runs cannot block reconnect retries. private usageTaskActiveClient: GatewayBrowserClient | null = null; + // The client survives transport reconnects, so retry budgets need a separate epoch. + private connectionEpoch: object = {}; private routeDataInitialized = false; private routeDataEnabled = true; private readonly refreshPolicy = new UsageRefreshPolicy({ @@ -182,8 +187,16 @@ class UsagePage extends OpenClawLightDomElement { return initialState; } this.refreshPolicy.beginLoad(); + const epoch = this.connectionEpoch; const agentId = normalizedAgentId || undefined; - return requestUsageSnapshot(client, { startDate, endDate, agentId, scope, timeZone }, signal); + return { + epoch, + ...(await requestUsageSnapshot( + client, + { startDate, endDate, agentId, scope, timeZone }, + signal, + )), + } satisfies UsageTaskValue; }, onComplete: (value) => { this.usageTaskActiveClient = null; @@ -191,7 +204,11 @@ class UsagePage extends OpenClawLightDomElement { this.usageCostSummary = value.costSummary; this.providerUsageSummary = value.providerUsageSummary; this.usageError = null; - this.refreshPolicy.markLoaded(); + this.providerUsageStalled = + this.refreshPolicy.markLoaded({ + incomplete: isUsageIncomplete(value.providerUsageSummary), + connection: value.epoch, + }) === "exhausted"; this.refreshPolicy.flushPending(); }, onError: (error) => { @@ -274,6 +291,7 @@ class UsagePage extends OpenClawLightDomElement { this.subscriptions.clear(); this.clearDateDebounce(); this.clearQueryDebounce(); + this.refreshPolicy.dispose(); this.usageTaskActiveClient = null; void this.usageTask.run(this.usageTaskArgs(null)); void this.usageTimeSeriesTask.run([null, ""]); @@ -312,7 +330,11 @@ class UsagePage extends OpenClawLightDomElement { this.usageResult = data.result; this.usageCostSummary = data.costSummary; this.providerUsageSummary = data.providerUsageSummary; - this.refreshPolicy.setLastLoadedAtMs(data.loadedAtMs); + this.providerUsageStalled = + this.refreshPolicy.setLastLoadedAtMs(data.loadedAtMs, { + incomplete: isUsageIncomplete(data.providerUsageSummary), + connection: this.connectionEpoch, + }) === "exhausted"; this.usageError = data.error; } @@ -339,6 +361,7 @@ class UsagePage extends OpenClawLightDomElement { this.usageResult = null; this.usageCostSummary = null; this.providerUsageSummary = null; + this.providerUsageStalled = false; this.refreshPolicy.resetPayload(); this.usageError = null; this.usageAgentId = this.context.agentSelection.state.scopeId; @@ -448,8 +471,11 @@ class UsagePage extends OpenClawLightDomElement { return; } void this.context.agents.ensureList(); - if (this.routeDataInitialized && (change.identityChanged || change.becameConnected)) { - this.refreshPolicy.request("reconnect"); + if (change.identityChanged || change.becameConnected) { + this.connectionEpoch = {}; + if (this.routeDataInitialized) { + this.refreshPolicy.request("reconnect"); + } } } @@ -502,6 +528,7 @@ class UsagePage extends OpenClawLightDomElement { this.usageCostSummary?.cacheStatus, ), providerUsage: this.providerUsageSummary?.providers ?? [], + providerUsageStalled: this.providerUsageStalled, }, filters: { startDate: this.usageStartDate, diff --git a/ui/src/pages/usage/view.test.ts b/ui/src/pages/usage/view.test.ts index 588e07918b51..77ff5a9ca4b2 100644 --- a/ui/src/pages/usage/view.test.ts +++ b/ui/src/pages/usage/view.test.ts @@ -63,6 +63,7 @@ function createUsageProps(overrides: Partial = {}): UsageProps { costDaily: [], cacheStatus: undefined, providerUsage: [], + providerUsageStalled: false, }, filters: { startDate: "2026-05-14", @@ -367,6 +368,28 @@ describe("renderUsage", () => { expect(onQueryDraftChange).toHaveBeenCalledWith(expect.stringContaining("provider:clear")); }); + it("reports a stalled provider refresh instead of hiding the section", () => { + const container = document.createElement("div"); + + render( + renderUsage( + createUsageProps({ + data: { + ...createUsageProps().data, + providerUsage: [], + providerUsageStalled: true, + }, + }), + ), + container, + ); + + const callout = container.querySelector(".usage-callout"); + expect(callout?.textContent?.trim()).toBe( + "Provider usage did not finish loading. Refresh to retry.", + ); + }); + it("renders provider plans, quotas, and billing independently of session usage", () => { const container = document.createElement("div"); diff --git a/ui/src/pages/usage/view.ts b/ui/src/pages/usage/view.ts index 738e39986d80..a8565a2aeebf 100644 --- a/ui/src/pages/usage/view.ts +++ b/ui/src/pages/usage/view.ts @@ -143,7 +143,14 @@ function renderUsageEmptyState(onRefresh: () => void) { type ProviderUsageSnapshot = ProviderUsageSummary["providers"][number]; -function renderProviderUsage(providers: ProviderUsageSnapshot[]) { +function renderProviderUsage(providers: ProviderUsageSnapshot[], stalled: boolean) { + if (stalled) { + // The stalled notice replaces the section: an empty grid under the normal + // header reads as "this operator has no providers", which is the lie. + return html` +
${t("usage.providerUsage.stalled")}
+ `; + } if (providers.length === 0) { return nothing; } @@ -798,7 +805,7 @@ export function renderUsage(props: UsageProps) { - ${renderProviderUsage(data.providerUsage)} + ${renderProviderUsage(data.providerUsage, data.providerUsageStalled)} ${isEmpty ? renderUsageEmptyState(filterActions.onRefresh) : html`