fix(gateway): refresh provider usage asynchronously

This commit is contained in:
Sasan
2026-08-13 22:58:13 -04:00
committed by Sasan Sotoodehfar
parent 98d7b38d48
commit 58f9fa2db4
38 changed files with 1290 additions and 103 deletions
@@ -184,6 +184,8 @@ private const val CRON_JOBS_PAGE_SIZE = 200
private const val CRON_JOBS_MAX_PAGES = 100
private const val CRON_JOBS_MAX_COUNT = CRON_JOBS_PAGE_SIZE * CRON_JOBS_MAX_PAGES
private const val CRON_JOBS_SNAPSHOT_MAX_ATTEMPTS = 3
private const val USAGE_INCOMPLETE_RETRY_DELAY_MS = 5_000L
private const val USAGE_INCOMPLETE_RETRY_LIMIT = 3
private const val OperatorAdminScope = "operator.admin"
private const val OperatorPairingScope = "operator.pairing"
@@ -1282,6 +1284,8 @@ class NodeRuntime private constructor(
val usageRefreshing: StateFlow<Boolean> = _usageRefreshing.asStateFlow()
private val _usageErrorText = MutableStateFlow<NativeText?>(null)
val usageErrorText: StateFlow<String?> = _usageErrorText.resolveOptionalNativeText()
private val usageRefreshGuard = LatestGatewayRefreshGuard()
private var usageIncompleteRetryJob: Job? = null
private val _skillsSummary = MutableStateFlow(GatewaySkillsSummary(skills = emptyList()))
val skillsSummary: StateFlow<GatewaySkillsSummary> = _skillsSummary.asStateFlow()
private val _skillsRefreshing = MutableStateFlow(false)
@@ -1361,6 +1365,9 @@ class NodeRuntime private constructor(
@Volatile internal var gatewayDataRequestTimeoutObserverForTests: ((method: String, timeoutMs: Long) -> Unit)? = null
@Volatile internal var clawHubSkillInstallBeforeClaimObserverForTests: (() -> Unit)? = null
@Volatile internal var usageIncompleteRetryDelayMsForTests: Long? = null
private val _channelsSummary = MutableStateFlow(GatewayChannelsSummary(channels = emptyList()))
val channelsSummary: StateFlow<GatewayChannelsSummary> = _channelsSummary.asStateFlow()
private val _channelsRefreshing = MutableStateFlow(false)
@@ -1732,6 +1739,8 @@ class NodeRuntime private constructor(
if (retirePendingCronRuns) {
pendingCronRunRegistry.clear { _pendingCronRunJobIds.value = it }
}
usageIncompleteRetryJob?.cancel()
usageRefreshGuard.invalidate()
_usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList())
_usageRefreshing.value = false
_usageErrorText.value = null
@@ -6331,20 +6340,85 @@ class NodeRuntime private constructor(
}
}
private suspend fun refreshUsageFromGateway() =
refreshGatewaySummary(
summary = _usageSummary,
refreshing = _usageRefreshing,
errorText = _usageErrorText,
disconnectedSummary = GatewayUsageSummary(updatedAtMs = null, providers = emptyList()),
failureText = nativeText("Could not load usage."),
) { gatewayScope ->
val root = json.parseToJsonElement(requestGatewayData(gatewayScope, "usage.status", "{}")).asObjectOrNull()
GatewayUsageSummary(
updatedAtMs = root.long("updatedAt"),
providers = parseUsageProviders(root?.get("providers") as? JsonArray),
)
private suspend fun refreshUsageFromGateway() {
usageIncompleteRetryJob?.cancel()
val gatewayScope = captureGatewayDataScope() ?: return
val refreshGeneration = usageRefreshGuard.begin()
if (refreshUsageOnceFromGateway(gatewayScope, refreshGeneration) && _usageSummary.value.refreshing) {
scheduleIncompleteUsageRetry(gatewayScope, refreshGeneration)
}
}
private fun publishUsageRefresh(
gatewayScope: GatewayDataScope,
refreshGeneration: Long,
publish: () -> Unit,
): Boolean {
var refreshCurrent = false
val scopeCurrent =
publishGatewayData(gatewayScope) {
refreshCurrent = usageRefreshGuard.publishIfCurrent(refreshGeneration, publish)
}
return scopeCurrent && refreshCurrent
}
private suspend fun refreshUsageOnceFromGateway(
gatewayScope: GatewayDataScope,
refreshGeneration: Long,
): Boolean {
publishUsageRefresh(gatewayScope, refreshGeneration) {
_usageRefreshing.value = true
_usageErrorText.value = null
}
if (!operatorConnected) {
return publishUsageRefresh(gatewayScope, refreshGeneration) {
_usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList())
_usageRefreshing.value = false
}
}
return try {
val root = json.parseToJsonElement(requestGatewayData(gatewayScope, "usage.status", "{}")).asObjectOrNull()
val nextSummary =
GatewayUsageSummary(
updatedAtMs = root.long("updatedAt"),
providers = parseUsageProviders(root?.get("providers") as? JsonArray),
refreshing = root.boolean("refreshing"),
)
publishUsageRefresh(gatewayScope, refreshGeneration) { _usageSummary.value = nextSummary }
} catch (_: Throwable) {
publishUsageRefresh(gatewayScope, refreshGeneration) {
// Preserve same-identity provider rows across a transient refresh failure.
_usageSummary.value = _usageSummary.value.copy(refreshing = false)
_usageErrorText.value = nativeText("Could not load usage.")
}
} finally {
publishUsageRefresh(gatewayScope, refreshGeneration) { _usageRefreshing.value = false }
}
}
private fun scheduleIncompleteUsageRetry(
gatewayScope: GatewayDataScope,
refreshGeneration: Long,
) {
// Mirrors the shared clients: three delayed retries, cancelled by a new cycle or gateway.
usageIncompleteRetryJob?.cancel()
usageIncompleteRetryJob =
scope.launch {
repeat(USAGE_INCOMPLETE_RETRY_LIMIT) {
delay(usageIncompleteRetryDelayMsForTests ?: USAGE_INCOMPLETE_RETRY_DELAY_MS)
if (!isGatewayDataScopeCurrent(gatewayScope)) return@launch
if (!refreshUsageOnceFromGateway(gatewayScope, refreshGeneration)) return@launch
if (!_usageSummary.value.refreshing) return@launch
}
publishUsageRefresh(gatewayScope, refreshGeneration) {
// Clearing the marker alone renders as "No usage data yet.", which claims
// the operator has no providers. Reuse the transient-failure text so a
// spent retry budget reads as the load failure it is.
_usageSummary.value = _usageSummary.value.copy(refreshing = false)
_usageErrorText.value = nativeText("Could not load usage.")
}
}
}
private suspend fun refreshSkillsFromGateway(): Boolean =
refreshGatewaySummary(
@@ -8776,6 +8850,7 @@ data class GatewayCronJobSummary(
data class GatewayUsageSummary(
val updatedAtMs: Long?,
val providers: List<GatewayUsageProviderSummary>,
val refreshing: Boolean = false,
)
data class GatewayUsageProviderSummary(
@@ -54,6 +54,7 @@ class ConnectionManager internal constructor(
internal const val AGENT_KIND_CLIENT_CAPABILITY = "agent-kind"
internal const val INLINE_WIDGETS_CLIENT_CAPABILITY = "inline-widgets"
internal const val USAGE_REFRESHING_CLIENT_CAPABILITY = "usage-refreshing"
internal fun operatorScopesForStoredDeviceToken(storedScopes: List<String>): List<String> {
val normalized =
@@ -233,6 +234,7 @@ class ConnectionManager internal constructor(
buildList {
add(AGENT_KIND_CLIENT_CAPABILITY)
if (inlineWidgetsAvailable()) add(INLINE_WIDGETS_CLIENT_CAPABILITY)
add(USAGE_REFRESHING_CLIENT_CAPABILITY)
},
commands = emptyList(),
permissions = emptyMap(),
@@ -247,6 +247,7 @@ private fun UsageSettingsScreen(
val isConnected by viewModel.isConnected.collectAsState()
val providerCount = usageSummary.providers.size
val issueCount = usageSummary.providers.count { it.error != null }
val usageConverging = usageRefreshVisible(usageRefreshing, usageSummary.refreshing)
LaunchedEffect(isConnected) {
if (isConnected) {
@@ -264,7 +265,7 @@ private fun UsageSettingsScreen(
),
)
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) {
ClawSecondaryButton(text = if (usageRefreshing) nativeString("Refreshing") else nativeString("Refresh"), onClick = viewModel::refreshUsage, enabled = isConnected && !usageRefreshing, modifier = Modifier.weight(1f))
ClawSecondaryButton(text = if (usageConverging) nativeString("Refreshing") else nativeString("Refresh"), onClick = viewModel::refreshUsage, enabled = isConnected && !usageConverging, modifier = Modifier.weight(1f))
}
usageErrorText?.let { errorText ->
ClawPanel {
@@ -276,14 +277,17 @@ private fun UsageSettingsScreen(
ClawPanel {
Text(text = nativeString("Connect the gateway to load usage."), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
}
usageSummary.providers.isEmpty() ->
usageSummary.providers.isNotEmpty() -> UsageProvidersPanel(providers = usageSummary.providers)
// The warning panel above already reports a failed load; adding
// "No usage data yet." beside it claims the operator has no providers.
usageErrorText != null -> Unit
else ->
ClawPanel {
Column(verticalArrangement = Arrangement.spacedBy(3.dp)) {
Text(text = nativeString("No usage data yet."), style = ClawTheme.type.section, color = ClawTheme.colors.text)
Text(text = if (usageConverging) nativeString("Refreshing") else nativeString("No usage data yet."), style = ClawTheme.type.section, color = ClawTheme.colors.text)
Text(text = nativeString("Provider limits will appear here when your gateway reports them."), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
}
}
else -> UsageProvidersPanel(providers = usageSummary.providers)
}
}
}
@@ -2553,6 +2557,11 @@ private fun UsageProvidersPanel(providers: List<GatewayUsageProviderSummary>) {
}
}
internal fun usageRefreshVisible(
requestRefreshing: Boolean,
summaryRefreshing: Boolean,
): Boolean = requestRefreshing || summaryRefreshing
@Composable
private fun UsageProviderListRow(provider: GatewayUsageProviderSummary) {
val hasIssue = provider.error != null
@@ -0,0 +1,137 @@
package ai.openclaw.app
import ai.openclaw.app.gateway.GatewayEndpoint
import ai.openclaw.app.ui.usageRefreshVisible
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.RuntimeEnvironment
import org.robolectric.annotation.Config
import java.lang.reflect.Field
import java.util.UUID
import java.util.concurrent.atomic.AtomicInteger
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
class UsageStatusRuntimeTest {
@Test
fun incompleteUsageConvergesOnACompletedPayload() {
val runtime = createRuntime()
connect(runtime)
runtime.usageIncompleteRetryDelayMsForTests = 10L
val calls = AtomicInteger()
runtime.gatewayDataRequestOverrideForTests = { _, method, _ ->
check(method == "usage.status")
if (calls.incrementAndGet() == 1) {
"""{"updatedAt":1,"providers":[],"refreshing":true}"""
} else {
"""{"updatedAt":2,"providers":[{"displayName":"Claude","plan":"Pro","windows":[]}]}"""
}
}
runtime.refreshUsage()
waitUntil {
runtime.usageSummary.value.providers
.isNotEmpty()
}
assertEquals(2, calls.get())
assertFalse(runtime.usageSummary.value.refreshing)
}
@Test
fun incompleteUsageRetriesStayBounded() {
val runtime = createRuntime()
connect(runtime)
runtime.usageIncompleteRetryDelayMsForTests = 10L
val calls = AtomicInteger()
runtime.gatewayDataRequestOverrideForTests = { _, _, _ ->
calls.incrementAndGet()
"""{"updatedAt":1,"providers":[],"refreshing":true}"""
}
runtime.refreshUsage()
waitUntil { calls.get() == 4 }
Thread.sleep(100)
assertEquals(4, calls.get())
assertFalse(runtime.usageSummary.value.refreshing)
assertFalse(
usageRefreshVisible(
requestRefreshing = runtime.usageRefreshing.value,
summaryRefreshing = runtime.usageSummary.value.refreshing,
),
)
// Clearing the marker without this would render "No usage data yet.",
// claiming the operator has no providers instead of a failed load.
assertNotNull(runtime.usageErrorText.value)
}
@Test
fun aTransientFailurePreservesRowsAndStopsTheRetryChain() {
val runtime = createRuntime()
connect(runtime)
runtime.usageIncompleteRetryDelayMsForTests = 10L
val calls = AtomicInteger()
runtime.gatewayDataRequestOverrideForTests = { _, _, _ ->
if (calls.incrementAndGet() == 1) {
"""{"updatedAt":1,"providers":[{"displayName":"Claude","plan":"Pro","windows":[]}],"refreshing":true}"""
} else {
error("usage unavailable")
}
}
runtime.refreshUsage()
waitUntil {
runtime.usageErrorText.value != null
}
Thread.sleep(100)
assertEquals(2, calls.get())
assertEquals(
"Claude",
runtime.usageSummary.value.providers
.single()
.displayName,
)
assertFalse(runtime.usageSummary.value.refreshing)
}
private fun createRuntime(): NodeRuntime {
val app = RuntimeEnvironment.getApplication()
val prefs =
app.getSharedPreferences(
"usage.${UUID.randomUUID()}",
android.content.Context.MODE_PRIVATE,
)
return NodeRuntime(app, SecurePrefs(app, securePrefsOverride = prefs))
}
private fun connect(runtime: NodeRuntime) {
field(runtime, "connectedEndpoint").set(runtime, GatewayEndpoint.manual("127.0.0.1", 18789))
field(runtime, "operatorConnected").set(runtime, true)
}
private fun waitUntil(condition: () -> Boolean) {
repeat(200) {
if (condition()) return
Thread.sleep(10)
}
error("condition did not become true")
}
private fun field(
target: Any,
name: String,
): Field {
var type: Class<*>? = target.javaClass
while (type != null) {
try {
return type.getDeclaredField(name).apply { isAccessible = true }
} catch (_: NoSuchFieldException) {
type = type.superclass
}
}
error("field $name not found")
}
}
@@ -139,6 +139,7 @@ class ConnectionManagerTest {
listOf(
ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY,
ConnectionManager.INLINE_WIDGETS_CLIENT_CAPABILITY,
ConnectionManager.USAGE_REFRESHING_CLIENT_CAPABILITY,
),
options.caps,
)
@@ -148,7 +149,13 @@ class ConnectionManagerTest {
fun buildOperatorConnectOptions_omitsInlineWidgetsWithoutIsolatedWebViews() {
val options = newManager(inlineWidgetsAvailable = false).buildOperatorConnectOptions()
assertEquals(listOf(ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY), options.caps)
assertEquals(
listOf(
ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY,
ConnectionManager.USAGE_REFRESHING_CLIENT_CAPABILITY,
),
options.caps,
)
}
@Test
@@ -349,6 +349,13 @@ class SettingsScreensTest {
assertEquals("None", formatCronTimestamp(null))
}
@Test
fun usageRefreshStaysVisibleBetweenIncompleteRetries() {
assertTrue(usageRefreshVisible(requestRefreshing = true, summaryRefreshing = false))
assertTrue(usageRefreshVisible(requestRefreshing = false, summaryRefreshing = true))
assertFalse(usageRefreshVisible(requestRefreshing = false, summaryRefreshing = false))
}
@Test
fun approvalCardShowsTheWholeMonospacedCommandBeforeStackedActions() {
val source = settingsScreensSource()
@@ -14,8 +14,11 @@ private let gatewayConnectionLogger = Logger(subsystem: "ai.openclaw", category:
actor GatewayConnection {
static let shared = GatewayConnection(
endpointProvider: GatewayConnection.defaultEndpointProvider)
nonisolated static let operatorClientCaps =
[OpenClawGatewayClientCapability.agentKind, OpenClawGatewayClientCapability.inlineWidgets]
nonisolated static let operatorClientCaps = [
OpenClawGatewayClientCapability.agentKind,
OpenClawGatewayClientCapability.inlineWidgets,
OpenClawGatewayClientCapability.usageRefreshing,
]
typealias Config = (url: URL, token: String?, password: String?)
@@ -29,6 +29,11 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
private var cachedUsageSummary: GatewayUsageSummary?
private var usageCacheUpdatedAt: Date?
private let usageRefreshIntervalSeconds: TimeInterval = 30
private var usageRetryTask: Task<Void, Never>?
private var usageRetryAttempts = 0
private var usageLoadGeneration = 0
private var usageRetryIntervalSeconds: TimeInterval = 5
private let usageRetryLimit = 3
private var cachedCostSummary: GatewayCostUsageSummary?
private var cachedCostErrorText: String?
private var costCacheUpdatedAt: Date?
@@ -36,6 +41,9 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
private let nodesStore = NodesStore.shared
#if DEBUG
private var testControlChannelConnected: Bool?
private var testUsageLoad: (() async throws -> GatewayUsageSummary)?
private var testUsageLoadDidFinish: (@MainActor () -> Void)?
private var testUsageRetryDidExhaust: (@MainActor () -> Void)?
#endif
func install(into statusItem: NSStatusItem) {
@@ -420,7 +428,8 @@ extension MenuSessionsInjector {
private func insertUsageSection(into menu: NSMenu, at cursor: Int, width: CGFloat) -> Int {
let rows = self.usageRows
if rows.isEmpty {
let stalled = self.isUsageStalled
if rows.isEmpty, !stalled {
return cursor
}
@@ -444,6 +453,16 @@ extension MenuSessionsInjector {
menu.insertItem(headerItem, at: cursor)
cursor += 1
if rows.isEmpty {
menu.insertItem(
self.makeMessageItem(
text: "Usage did not finish loading",
symbolName: "exclamationmark.triangle",
width: width),
at: cursor)
return cursor + 1
}
if let selectedProvider = self.selectedUsageProviderId,
let primary = rows.first(where: { $0.providerId.lowercased() == selectedProvider }),
rows.count > 1
@@ -516,6 +535,17 @@ extension MenuSessionsInjector {
return summary.primaryRows()
}
/// Retry budget spent with the cold marker still set. The empty row list is
/// the Gateway saying "not loaded", so the menu must not hide the section
/// the way it does for an operator who genuinely has no usage providers.
/// Requires a live control channel: while disconnected the honest answer is
/// the disconnected state, not a usage-specific warning.
private var isUsageStalled: Bool {
self.isControlChannelConnected
&& self.cachedUsageSummary?.refreshing == true
&& self.usageRetryAttempts >= self.usageRetryLimit
}
private func buildUsageOverflowMenu(rows: [UsageRow], width: CGFloat) -> NSMenu {
let menu = NSMenu()
// Keep submenu delegate nil: reusing the status-menu delegate here causes
@@ -776,19 +806,73 @@ extension MenuSessionsInjector {
return
}
self.usageLoadGeneration += 1
let generation = self.usageLoadGeneration
guard self.isControlChannelConnected else {
self.usageCacheUpdatedAt = Date()
return
}
self.usageRetryTask?.cancel()
self.usageRetryTask = nil
self.usageRetryAttempts = 0
await self.loadUsageSummaryOnce(generation: generation)
}
private func loadUsageSummaryOnce(generation: Int) async {
#if DEBUG
defer { self.testUsageLoadDidFinish?() }
#endif
do {
self.cachedUsageSummary = try await UsageLoader.loadSummary()
let summary = try await self.loadUsageSummary()
guard generation == self.usageLoadGeneration else { return }
self.cachedUsageSummary = summary
if summary.refreshing == true {
// A cold Gateway marker is not a cacheable answer; converge with bounded retries.
self.usageCacheUpdatedAt = nil
self.scheduleUsageRetry(generation: generation)
return
}
} catch {
guard generation == self.usageLoadGeneration else { return }
self.cachedUsageSummary = nil
}
self.usageCacheUpdatedAt = Date()
}
private func scheduleUsageRetry(generation: Int) {
guard self.usageRetryAttempts < self.usageRetryLimit else {
#if DEBUG
self.testUsageRetryDidExhaust?()
#endif
return
}
self.usageRetryAttempts += 1
let interval = self.usageRetryIntervalSeconds
self.usageRetryTask = Task { [weak self] in
try? await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000))
guard let self,
!Task.isCancelled,
self.isControlChannelConnected,
generation == self.usageLoadGeneration
else { return }
await self.loadUsageSummaryOnce(generation: generation)
// The final attempt leaves the marker set, so repaint on exhaustion
// too or an operator watching the open menu never sees the outcome.
if self.cachedUsageSummary?.refreshing != true || self.isUsageStalled {
await self.repaintOpenMenu(self.statusItem?.menu)
}
}
}
private func loadUsageSummary() async throws -> GatewayUsageSummary {
#if DEBUG
if let load = self.testUsageLoad { return try await load() }
#endif
return try await UsageLoader.loadSummary()
}
private func refreshCostUsageCache(force: Bool) async {
if !force,
let updated = self.costCacheUpdatedAt,
@@ -1295,6 +1379,34 @@ extension MenuSessionsInjector {
self.usageCacheUpdatedAt = Date()
}
func setTestingUsageLoader(_ load: (() async throws -> GatewayUsageSummary)?) {
self.testUsageLoad = load
}
func setTestingUsageLoadDidFinish(_ didFinish: (@MainActor () -> Void)?) {
self.testUsageLoadDidFinish = didFinish
}
func setTestingUsageRetryDidExhaust(_ didExhaust: (@MainActor () -> Void)?) {
self.testUsageRetryDidExhaust = didExhaust
}
func setTestingUsageRetryInterval(_ seconds: TimeInterval) {
self.usageRetryIntervalSeconds = seconds
}
func refreshUsageCacheForTesting(force: Bool) async {
await self.refreshUsageCache(force: force)
}
var testingCachedUsageSummary: GatewayUsageSummary? {
self.cachedUsageSummary
}
var testingUsageCacheUpdatedAt: Date? {
self.usageCacheUpdatedAt
}
func setTestingCostUsageSummary(_ summary: GatewayCostUsageSummary?, errorText: String? = nil) {
self.cachedCostSummary = summary
self.cachedCostErrorText = errorText
@@ -16,6 +16,13 @@ struct GatewayUsageProvider: Codable {
struct GatewayUsageSummary: Codable {
let updatedAt: Double
let providers: [GatewayUsageProvider]
let refreshing: Bool?
init(updatedAt: Double, providers: [GatewayUsageProvider], refreshing: Bool? = nil) {
self.updatedAt = updatedAt
self.providers = providers
self.refreshing = refreshing
}
}
struct UsageRow: Identifiable {
@@ -10,6 +10,7 @@ struct MacGatewayChatTransportMappingTests {
#expect(GatewayConnection.operatorClientCaps == [
OpenClawGatewayClientCapability.agentKind,
OpenClawGatewayClientCapability.inlineWidgets,
OpenClawGatewayClientCapability.usageRefreshing,
])
}
@@ -175,6 +175,125 @@ struct MenuSessionsInjectorTests {
#expect(usageCostItem?.submenu?.delegate == nil)
}
@Test func `cold incomplete usage converges without starting the cache ttl`() async {
let injector = MenuSessionsInjector()
injector.setTestingControlChannelConnected(true)
injector.setTestingUsageRetryInterval(0)
let events = UsageLoadEvents()
injector.setTestingUsageLoadDidFinish { events.finished() }
var calls = 0
injector.setTestingUsageLoader {
calls += 1
if calls == 1 {
return GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true)
}
return GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false)
}
await injector.refreshUsageCacheForTesting(force: true)
#expect(injector.testingUsageCacheUpdatedAt == nil)
#expect(await events.waitFor(count: 2))
#expect(calls == 2)
#expect(injector.testingUsageCacheUpdatedAt != nil)
}
@Test func `incomplete usage retry is bounded`() async {
let injector = MenuSessionsInjector()
injector.setTestingControlChannelConnected(true)
injector.setTestingUsageRetryInterval(0)
let events = UsageLoadEvents()
injector.setTestingUsageLoadDidFinish { events.finished() }
injector.setTestingUsageRetryDidExhaust { events.exhausted() }
injector.setTestingUsageLoader {
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true)
}
await injector.refreshUsageCacheForTesting(force: true)
#expect(await events.waitFor(count: 4))
#expect(await events.waitForExhaustion())
#expect(injector.testingUsageCacheUpdatedAt == nil)
}
@Test func `stalled usage keeps a visible menu section`() async {
let injector = MenuSessionsInjector()
injector.setTestingControlChannelConnected(true)
injector.setTestingUsageRetryInterval(0)
let events = UsageLoadEvents()
injector.setTestingUsageLoadDidFinish { events.finished() }
injector.setTestingUsageRetryDidExhaust { events.exhausted() }
// An operator with no usage providers still gets no usage section.
injector.setTestingUsageSummary(
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false))
let quiet = Self.makeMenuShell()
injector.injectForTesting(into: quiet)
let quietItems = quiet.items.count(where: { $0.tag == 9_415_557 })
injector.setTestingUsageLoader {
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true)
}
await injector.refreshUsageCacheForTesting(force: true)
#expect(await events.waitForExhaustion())
// Spent budget with the marker still set: separator, header, and the
// stalled row, never the silent menu an empty provider list produces.
let stalled = Self.makeMenuShell()
injector.injectForTesting(into: stalled)
#expect(stalled.items.count(where: { $0.tag == 9_415_557 }) == quietItems + 3)
}
private static func makeMenuShell() -> NSMenu {
let menu = NSMenu()
menu.addItem(NSMenuItem(title: "Header", action: nil, keyEquivalent: ""))
menu.addItem(.separator())
menu.addItem(NSMenuItem(title: "Send Heartbeats", action: nil, keyEquivalent: ""))
menu.addItem(.separator())
menu.addItem(NSMenuItem(title: "Settings…", action: nil, keyEquivalent: ""))
return menu
}
@Test func `late usage result from a replaced gateway is ignored`() async {
let injector = MenuSessionsInjector()
injector.setTestingControlChannelConnected(true)
let loads = DeferredUsageLoads()
injector.setTestingUsageLoader { try await loads.load() }
let first = Task { await injector.refreshUsageCacheForTesting(force: true) }
#expect(await loads.waitForRequests(count: 1))
let second = Task { await injector.refreshUsageCacheForTesting(force: true) }
#expect(await loads.waitForRequests(count: 2))
loads.complete(
at: 1,
with: GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false))
await second.value
loads.complete(
at: 0,
with: GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false))
await first.value
#expect(injector.testingCachedUsageSummary?.updatedAt == 2)
}
@Test func `fresh no-op does not invalidate a forced usage load`() async {
let injector = MenuSessionsInjector()
injector.setTestingControlChannelConnected(true)
injector.setTestingUsageSummary(
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false))
let loads = DeferredUsageLoads()
injector.setTestingUsageLoader { try await loads.load() }
let forced = Task { await injector.refreshUsageCacheForTesting(force: true) }
#expect(await loads.waitForRequests(count: 1))
await injector.refreshUsageCacheForTesting(force: false)
loads.complete(
at: 0,
with: GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false))
await forced.value
#expect(injector.testingCachedUsageSummary?.updatedAt == 2)
}
@Test func `status text keeps useful error detail`() {
let injector = MenuSessionsInjector()
let longError = """
@@ -237,3 +356,85 @@ struct MenuSessionsInjectorTests {
connected: connected)
}
}
@MainActor
private final class UsageLoadEvents {
private struct Snapshot: Sendable {
let completed: Int
let exhausted: Bool
}
private var completed = 0
private var didExhaust = false
private let stream: AsyncStream<Snapshot>
private let continuation: AsyncStream<Snapshot>.Continuation
init() {
(self.stream, self.continuation) = AsyncStream.makeStream(
of: Snapshot.self,
bufferingPolicy: .bufferingNewest(1))
}
func finished() {
self.completed += 1
self.publish()
}
func exhausted() {
self.didExhaust = true
self.publish()
}
func waitFor(count: Int) async -> Bool {
if self.completed >= count { return true }
return await self.wait { $0.completed >= count }
}
func waitForExhaustion() async -> Bool {
if self.didExhaust { return true }
return await self.wait { $0.exhausted }
}
private func publish() {
self.continuation.yield(Snapshot(completed: self.completed, exhausted: self.didExhaust))
}
private func wait(_ predicate: @escaping @Sendable (Snapshot) -> Bool) async -> Bool {
let stream = self.stream
return await withTaskGroup(of: Bool.self) { group in
group.addTask {
for await snapshot in stream where predicate(snapshot) { return true }
return false
}
group.addTask {
try? await Task.sleep(nanoseconds: 5_000_000_000)
return false
}
let result = await group.next() ?? false
group.cancelAll()
return result
}
}
}
@MainActor
private final class DeferredUsageLoads {
private var continuations: [CheckedContinuation<GatewayUsageSummary, Never>] = []
func load() async throws -> GatewayUsageSummary {
await withCheckedContinuation { continuation in
self.continuations.append(continuation)
}
}
func waitForRequests(count: Int) async -> Bool {
for _ in 0..<100 where self.continuations.count < count {
await Task.yield()
}
return self.continuations.count >= count
}
func complete(at index: Int, with summary: GatewayUsageSummary) {
self.continuations[index].resume(returning: summary)
}
}
@@ -3,6 +3,7 @@ import OpenClawProtocol
public enum OpenClawGatewayClientCapability {
public static let agentKind = "agent-kind"
public static let inlineWidgets = "inline-widgets"
public static let usageRefreshing = "usage-refreshing"
}
public struct GatewayConnectOptions: Sendable {
+7 -1
View File
@@ -97,9 +97,15 @@ const caps = [GATEWAY_CLIENT_CAPS.TOOL_EVENTS];
The current registry contains `approvals`, `exec-approvals`, `inline-widgets`,
`run-tool-bindings`, `session-scoped-events`, `plugin-approvals`,
`task-suggestions`, `terminal-offset-seq`, `tool-events`, and `ui-commands`.
`task-suggestions`, `terminal-offset-seq`, `tool-events`, `ui-commands`, and
`usage-refreshing`.
Advertise only capabilities the client actually implements.
`usage-refreshing` allows a cold `usage.status` request to return immediately
with `refreshing: true` and an empty provider list. A client advertising it must
keep that payload cache-cold and refetch on a short bounded schedule. Other
clients retain the blocking cold read.
<Warning>
`tool-events` gates live tool-execution streaming. The Gateway registers only
connections that advertise this capability as recipients for a run's structured
+1 -1
View File
@@ -535,7 +535,7 @@ methods. Treat this as feature discovery, not a full enumeration of
<Accordion title="Models and usage">
- `models.list` returns the runtime-allowed model catalog. See "`models.list` views" below.
- `usage.status` returns provider usage windows/remaining quota summaries.
- `usage.status` returns provider usage windows/remaining quota summaries. Clients advertising `usage-refreshing` receive an immediate `refreshing: true` placeholder on a cold cache and must refetch on a bounded schedule; other callers block for the cold provider read.
- `usage.cost` returns aggregated cost usage summaries for a date range. Pass `agentId` for one agent, or `agentScope: "all"` to aggregate configured agents.
- `doctor.memory.status` returns vector-memory / cached embedding readiness for the active default agent workspace. Pass `{ "probe": true }` or `{ "deep": true }` only for an explicit live embedding provider ping. Pass `{ "agentId": "agent-id" }` to scope Dreaming store stats to one agent workspace; omitting it aggregates configured Dreaming workspaces.
- `doctor.memory.dreamDiary`, `doctor.memory.backfillDreamDiary`, `doctor.memory.resetDreamDiary`, `doctor.memory.resetGroundedShortTerm`, `doctor.memory.repairDreamingArtifacts`, and `doctor.memory.dedupeDreamDiary` accept optional `{ "agentId": "agent-id" }`; omitted, they operate on the configured default agent workspace.
@@ -90,6 +90,7 @@ export const GATEWAY_CLIENT_CAPS = {
TERMINAL_OFFSET_SEQ: "terminal-offset-seq",
TOOL_EVENTS: "tool-events",
UI_COMMANDS: "ui-commands",
USAGE_REFRESHING: "usage-refreshing",
} as const;
/** Optional capability advertised by clients during gateway handshake. */
+2 -1
View File
@@ -15,7 +15,8 @@ type GatewayHandlerInvocation = Parameters<GatewayRequestHandlers[string]>[0];
const BOARD_DATA_HANDLERS: Record<BoardDataBindingId, GatewayRequestHandlers[string]> = {
"sessions.list": sessionReadHandlers["sessions.list"]!,
"usage.status": usageHandlers["usage.status"]!,
// Board reads are one-shot and cannot converge an incomplete marker.
"usage.status": (invocation) => usageHandlers["usage.status"]!({ ...invocation, client: null }),
"usage.cost": usageHandlers["usage.cost"]!,
"cron.list": cronHandlers["cron.list"]!,
"cron.status": cronHandlers["cron.status"]!,
@@ -197,8 +197,9 @@ function scheduleProviderUsageRefresh(params: {
return usage;
})
.catch((err: unknown) => {
// Usage is auxiliary and stale data remains valid. Keep failures visible
// without delaying fresh auth-health responses.
// Usage is auxiliary and stale data remains valid. A failed refresh
// publishes nothing, so a capable client keeps seeing the incomplete
// marker and reports it once its retry budget is spent.
log.debug(
`usage refresh failed: providers=${params.providerIds.join(",")} error=${formatForLog(err)}`,
);
@@ -225,26 +226,48 @@ type ProviderUsageCacheParams = {
agentDir: string;
configRef: OpenClawConfig;
credentialKey: string;
coldRead?: "refresh-marker";
forceRefresh?: boolean;
providerIds: UsageProviderId[];
now: number;
};
/**
* Credential identity without the selection bookkeeping. `usageStats` is stamped
* by every successful run (`markAuthProfileSuccess`), so treating it as an
* identity change discards a good snapshot several times a minute on a busy
* gateway — and a capable client would then receive a cold marker on every poll.
* It still belongs in the refresh trigger because it can reorder profiles.
*/
function providerUsageCredentialIdentity(credentialKey: string): string {
try {
const { usageStats: _selectionBookkeeping, ...identity } = JSON.parse(credentialKey) as Record<
string,
unknown
>;
return JSON.stringify(identity);
} catch {
return credentialKey;
}
}
function resolveProviderUsageCacheRead(params: ProviderUsageCacheParams) {
const providerIds = params.providerIds.toSorted();
const providerKey = providerUsageCacheKey(providerIds);
const credentialKey = scopeProviderUsageCredentialKey(params.credentialKey, providerIds);
const credentialIdentity = providerUsageCredentialIdentity(credentialKey);
const cached = usageCacheByAgentId.get(params.agentId);
const matching =
cached?.agentDir === params.agentDir &&
cached.configRef === params.configRef &&
cached.credentialKey === credentialKey &&
providerUsageCredentialIdentity(cached.credentialKey) === credentialIdentity &&
cached.providerKey === providerKey
? cached
: undefined;
const needsRefresh =
params.forceRefresh === true ||
!matching ||
matching.credentialKey !== credentialKey ||
params.now - matching.refreshedAt >= USAGE_CACHE_TTL_MS;
return { credentialKey, matching, needsRefresh, providerIds, providerKey };
}
@@ -273,7 +296,7 @@ export function readProviderUsageStaleWhileRevalidate(
return matching?.usageByProvider ?? new Map();
}
/** Returns cached provider usage, awaiting only a cold miss and refreshing stale data in place. */
/** Returns cached provider usage while network refreshes run in the background for capable clients. */
async function loadProviderUsageSummaryStaleWhileRevalidate(
params: ProviderUsageCacheParams,
): Promise<UsageSummary> {
@@ -298,12 +321,17 @@ async function loadProviderUsageSummaryStaleWhileRevalidate(
void refresh.catch(() => {});
return matching.summary;
}
return await refresh;
if (params.coldRead !== "refresh-marker") {
return await refresh;
}
void refresh.catch(() => {});
return { updatedAt: params.now, providers: [], refreshing: true };
}
/** Shares the models.authStatus cache contract with the unscoped usage.status RPC. */
export async function loadUsageStatusStaleWhileRevalidate(params: {
config: OpenClawConfig;
coldRead?: "refresh-marker";
now?: number;
}): Promise<UsageSummary> {
const agentId = resolveLegacyInheritedAuthAgentId(params.config);
@@ -342,6 +370,7 @@ export async function loadUsageStatusStaleWhileRevalidate(params: {
store,
}),
providerIds,
coldRead: params.coldRead,
now: params.now ?? Date.now(),
});
}
@@ -1,5 +1,9 @@
import { describe, expect, it, vi } from "vitest";
import { readPreparedServerMethodModelCatalog } from "./optional-model-catalog.js";
import { registerGatewayModelCatalogPrivateAccess } from "../server-model-catalog-auth.js";
import {
loadOptionalServerMethodModelCatalogSnapshot,
readPreparedServerMethodModelCatalog,
} from "./optional-model-catalog.js";
import type { GatewayRequestContext } from "./types.js";
describe("readPreparedServerMethodModelCatalog", () => {
@@ -20,3 +24,38 @@ describe("readPreparedServerMethodModelCatalog", () => {
expect(loadGatewayModelCatalog).not.toHaveBeenCalled();
});
});
describe("loadOptionalServerMethodModelCatalogSnapshot", () => {
it("uses a prepared snapshot before cold discovery", async () => {
const snapshot = {
agentId: "work",
agentDir: "/tmp/work-agent",
catalogComplete: true,
workspaceDir: "/tmp/work",
config: {},
entries: [],
routeVariants: [],
authModes: {},
authStore: { version: 1, profiles: {} },
authMaterializations: [],
metadataSnapshot: { index: { plugins: [] }, plugins: [] } as never,
};
const loadGatewayModelCatalogSnapshot = vi.fn();
const context = {
loadGatewayModelCatalogSnapshot,
logGateway: { debug: vi.fn() },
} as unknown as GatewayRequestContext;
const readPrepared = vi.fn(async () => snapshot);
registerGatewayModelCatalogPrivateAccess(loadGatewayModelCatalogSnapshot, {
loadDeferred: vi.fn(),
readPrepared,
});
await expect(
loadOptionalServerMethodModelCatalogSnapshot(context, "chat.startup", {
loadParams: { agentId: "work" },
}),
).resolves.toBe(snapshot);
expect(loadGatewayModelCatalogSnapshot).not.toHaveBeenCalled();
});
});
@@ -1,6 +1,7 @@
// Optional model-catalog access gives session/tool methods metadata when ready
// while keeping provider discovery out of ordinary request hot paths.
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
import { readPreparedCatalogSnapshot } from "../server-model-catalog-auth.js";
import type { GatewayModelCatalogSnapshot } from "../server-model-catalog.types.js";
import type { GatewayRequestContext } from "./types.js";
@@ -74,7 +75,11 @@ export function startOptionalServerMethodModelCatalogSnapshotLoad(
loadParams?: Parameters<GatewayRequestContext["loadGatewayModelCatalogSnapshot"]>[0],
): OptionalServerMethodModelCatalogLoad<GatewayModelCatalogSnapshot> {
return startOptionalServerMethodModelCatalogValueLoad({
load: () => context.loadGatewayModelCatalogSnapshot(loadParams),
load: async () =>
// Prepared startup facts avoid provider discovery. The cold loader remains
// the fallback because it publishes the generation used by capability checks.
(await readPreparedCatalogSnapshot(context, loadParams)) ??
(await context.loadGatewayModelCatalogSnapshot(loadParams)),
normalize: normalizeOptionalModelCatalogSnapshot,
});
}
@@ -46,6 +46,8 @@ const config = {
agents: { list: [{ id: "main", default: true }] },
} as OpenClawConfig;
const refreshingCapableClient = { connect: { caps: ["usage-refreshing"] } };
function createStore(access = "access-one") {
return {
version: 1,
@@ -61,7 +63,7 @@ function createStore(access = "access-one") {
};
}
async function runUsageStatus() {
async function runUsageStatus(client?: unknown, getRuntimeConfig = () => config) {
const respond = vi.fn();
await expectDefined(
usageHandlers["usage.status"],
@@ -69,7 +71,8 @@ async function runUsageStatus() {
)({
respond,
params: {},
context: { getRuntimeConfig: () => config },
context: { getRuntimeConfig },
client: client === undefined ? refreshingCapableClient : client,
} as unknown as Parameters<(typeof usageHandlers)["usage.status"]>[0]);
expect(respond).toHaveBeenCalledTimes(1);
expect(respond.mock.calls[0]?.[0]).toBe(true);
@@ -114,7 +117,7 @@ describe("usage.status provider usage cache", () => {
vi.restoreAllMocks();
});
it("loads the cached provider snapshot from the exact runtime config", async () => {
function mockExactConfigProviderUsage() {
mocks.loadProviderUsageSummary.mockImplementation(async (options) => ({
updatedAt: now,
providers:
@@ -129,15 +132,108 @@ describe("usage.status provider usage cache", () => {
]
: [],
}));
}
const result = (await runUsageStatus()) as {
it("loads the cached provider snapshot from the exact runtime config", async () => {
mockExactConfigProviderUsage();
// Clientless internal reads stay blocking, so the first response already
// carries the snapshot the exact-config loader produced.
const result = (await runUsageStatus(null)) as {
providers: Array<{ accountEmail?: string }>;
};
expect(result.providers[0]?.accountEmail).toBe("configured@example.com");
});
it("hands the exact runtime config to the background refresh", async () => {
mockExactConfigProviderUsage();
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
await vi.waitFor(async () => {
expect(
(await runUsageStatus()) as { providers: Array<{ accountEmail?: string }> },
).toMatchObject({ providers: [{ accountEmail: "configured@example.com" }] });
});
});
it("returns a cold marker only to clients that can converge it", async () => {
let finish: ((value: { updatedAt: number; providers: never[] }) => void) | undefined;
mocks.loadProviderUsageSummary.mockImplementationOnce(
() =>
new Promise((resolve) => {
finish = resolve;
}),
);
await expect(runUsageStatus()).resolves.toEqual({
updatedAt: now,
providers: [],
refreshing: true,
});
const legacy = runUsageStatus({ connect: { caps: [] } });
const pending = Symbol("pending");
await expect(
Promise.race([
legacy,
new Promise((resolve) => {
setTimeout(() => resolve(pending), 25);
}),
]),
).resolves.toBe(pending);
finish?.({ updatedAt: now, providers: [] });
await expect(legacy).resolves.toEqual({ updatedAt: now, providers: [] });
});
it("keeps serving usage while run bookkeeping churns the profile store", async () => {
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
await vi.waitFor(async () => {
expect(await runUsageStatus()).toMatchObject({ providers: [{ provider: "openai" }] });
});
// Every completed agent run stamps usage bookkeeping on the selected profile
// (markAuthProfileSuccess). That must not read as a credential change, or a
// busy gateway hands capable clients a cold marker on every poll and they
// report "did not finish loading" while every refresh is succeeding.
store = { ...store, usageStats: { "openai:default": { lastUsed: now } } };
now += 1;
await expect(runUsageStatus()).resolves.toMatchObject({
providers: [{ provider: "openai" }],
});
});
it("keeps clientless internal reads blocking", async () => {
const result = (await runUsageStatus(null)) as { refreshing?: boolean; providers: unknown[] };
expect(result.providers).toHaveLength(1);
expect(result.refreshing).toBeUndefined();
});
it("keeps a failed refresh incomplete for capable clients and recovers", async () => {
mocks.loadProviderUsageSummary.mockRejectedValueOnce(new Error("provider stack down"));
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
await new Promise((resolve) => {
setTimeout(resolve, 0);
});
// A failed refresh publishes nothing, so the payload stays marked incomplete
// and the client's bounded retry owns reporting it. The next attempt recovers.
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
await vi.waitFor(async () => {
expect((await runUsageStatus()) as { providers: unknown[] }).toMatchObject({
providers: [expect.any(Object)],
});
});
});
it("reuses byte-identical results within 60s and refreshes stale data in the background", async () => {
const first = await runUsageStatus();
expect(await runUsageStatus()).toMatchObject({ refreshing: true });
const first = await vi.waitFor(async () => {
const value = (await runUsageStatus()) as { providers: unknown[]; refreshing?: boolean };
expect(value.providers).toHaveLength(1);
return value;
});
expect(first.refreshing).toBeUndefined();
const repeated = await runUsageStatus();
expect(JSON.stringify(repeated)).toBe(JSON.stringify(first));
@@ -157,10 +253,33 @@ describe("usage.status provider usage cache", () => {
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2);
});
it("invalidates cached usage when the runtime config changes", async () => {
const configFor = (baseUrl: string) =>
({ ...config, models: { providers: { openai: { baseUrl } } } }) as unknown as OpenClawConfig;
const first = configFor("https://one.example/v1");
expect(await runUsageStatus(refreshingCapableClient, () => first)).toMatchObject({
refreshing: true,
});
await vi.waitFor(async () => {
expect(
((await runUsageStatus(refreshingCapableClient, () => first)) as { providers: unknown[] })
.providers,
).toHaveLength(1);
});
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1);
const second = configFor("https://two.example/v1");
expect(await runUsageStatus(refreshingCapableClient, () => second)).toMatchObject({
refreshing: true,
});
await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2));
});
it("shares the raw snapshot with models.authStatus and invalidates on credential rotation", async () => {
await runUsageStatus();
expect(await runUsageStatus()).toMatchObject({ refreshing: true });
const agentId = resolveDefaultAgentId(config);
const agentDir = resolveAgentDir(config, agentId);
await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1));
const usage = readProviderUsageStaleWhileRevalidate({
agentId,
agentDir,
@@ -177,10 +296,13 @@ describe("usage.status provider usage cache", () => {
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1);
store.profiles["openai:default"].access = "access-two";
const rotated = (await runUsageStatus()) as {
providers: Array<{ windows: Array<{ usedPercent: number }> }>;
};
expect(rotated.providers[0]?.windows[0]?.usedPercent).toBe(20);
expect(await runUsageStatus()).toMatchObject({ refreshing: true });
await vi.waitFor(async () => {
const rotated = (await runUsageStatus()) as {
providers: Array<{ windows: Array<{ usedPercent: number }> }>;
};
expect(rotated.providers[0]?.windows[0]?.usedPercent).toBe(20);
});
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2);
});
});
+17 -1
View File
@@ -3,6 +3,10 @@
import fs from "node:fs";
import { expectDefined } from "@openclaw/normalization-core";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import {
GATEWAY_CLIENT_CAPS,
hasGatewayClientCap,
} from "../../../packages/gateway-protocol/src/client-info.js";
import {
ErrorCodes,
errorShape,
@@ -1133,9 +1137,21 @@ export const testApi = {
export type { SessionUsageEntry, SessionsUsageAggregates, SessionsUsageResult };
export const usageHandlers: GatewayRequestHandlers = {
"usage.status": async ({ respond, context }) => {
"usage.status": async ({ respond, context, client }) => {
// Only clients with bounded retry machinery may receive an incomplete cold result.
// In-process dispatch reuses the originating request's client, capabilities
// included, so a plugin proxying this method inside a capable UI request
// would inherit the marker without any way to converge it. Such a caller
// must pass a capless client, the way board bindings force `client: null`.
const coldRead = hasGatewayClientCap(
client?.connect?.caps,
GATEWAY_CLIENT_CAPS.USAGE_REFRESHING,
)
? ("refresh-marker" as const)
: undefined;
const summary = await loadUsageStatusStaleWhileRevalidate({
config: context.getRuntimeConfig(),
coldRead,
});
respond(true, summary, undefined);
},
+9 -1
View File
@@ -66,5 +66,13 @@ export async function readPreparedCatalog(
context: Pick<GatewayRequestContext, "loadGatewayModelCatalogSnapshot">,
agentId: string,
): Promise<PreparedGatewayModelCatalogSnapshot | undefined> {
return await requirePrivateAccess(context).readPrepared({ agentId });
return await readPreparedCatalogSnapshot(context, { agentId });
}
/** Reads a prepared owner snapshot without exposing private auth facts on the request context. */
export async function readPreparedCatalogSnapshot(
context: Pick<GatewayRequestContext, "loadGatewayModelCatalogSnapshot">,
params?: Omit<GatewayModelCatalogReadParams, "readOnly">,
): Promise<PreparedGatewayModelCatalogSnapshot | undefined> {
return await requirePrivateAccess(context).readPrepared(params);
}
+2
View File
@@ -86,6 +86,8 @@ export type ProviderUsageSnapshot = {
export type UsageSummary = {
updatedAt: number;
providers: ProviderUsageSnapshot[];
/** A background refresh owns the real values; an empty list is incomplete. */
refreshing?: boolean;
};
/** Normalized provider id. Usage providers are discovered from plugin hooks at runtime. */
+1
View File
@@ -471,6 +471,7 @@ describe("GatewayBrowserClient", () => {
GATEWAY_CLIENT_CAPS.TOOL_EVENTS,
GATEWAY_CLIENT_CAPS.INLINE_WIDGETS,
GATEWAY_CLIENT_CAPS.UI_COMMANDS,
GATEWAY_CLIENT_CAPS.USAGE_REFRESHING,
]);
expect(connectFrame.params?.scopes).toEqual([...CONTROL_UI_OPERATOR_SCOPES]);
});
+1
View File
@@ -483,6 +483,7 @@ export class GatewayBrowserClient {
GATEWAY_CLIENT_CAPS.TOOL_EVENTS,
GATEWAY_CLIENT_CAPS.INLINE_WIDGETS,
GATEWAY_CLIENT_CAPS.UI_COMMANDS,
GATEWAY_CLIENT_CAPS.USAGE_REFRESHING,
],
auth: buildGatewayConnectAuth(selectedAuth),
userAgent: navigator.userAgent,
+1
View File
@@ -4369,6 +4369,7 @@ export const en: TranslationMap = {
providerUsage: {
title: "Provider plans & billing",
subtitle: "Live plan, quota, balance, and budget data reported by configured providers.",
stalled: "Provider usage did not finish loading. Refresh to retry.",
balance: "Balance",
spend: "Usage",
budget: "Budget",
+73
View File
@@ -0,0 +1,73 @@
// @vitest-environment jsdom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { IncompleteUsageRetry, isUsageIncomplete } from "./incomplete-usage-retry.ts";
describe("IncompleteUsageRetry", () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it("recognizes only an explicit incomplete marker", () => {
expect(isUsageIncomplete({ refreshing: true })).toBe(true);
expect(isUsageIncomplete({ refreshing: false })).toBe(false);
expect(isUsageIncomplete(null)).toBe(false);
});
it("reports exhaustion once the three delayed attempts are spent", () => {
const retry = vi.fn();
const policy = new IncompleteUsageRetry({ retry });
for (let attempt = 0; attempt < 3; attempt += 1) {
expect(policy.observe(true)).toBe("retrying");
vi.advanceTimersByTime(5_000);
}
expect(retry).toHaveBeenCalledTimes(3);
// Nothing schedules another attempt, so the page owns the outcome from here.
expect(policy.observe(true)).toBe("exhausted");
vi.advanceTimersByTime(5_000);
expect(retry).toHaveBeenCalledTimes(3);
});
it("resets on completion or connection replacement", () => {
const retry = vi.fn();
const policy = new IncompleteUsageRetry({ retry });
const first = {};
for (let attempt = 0; attempt < 4; attempt += 1) {
policy.observe(true, first);
vi.advanceTimersByTime(5_000);
}
expect(retry).toHaveBeenCalledTimes(3);
policy.observe(true, {});
vi.advanceTimersByTime(5_000);
expect(retry).toHaveBeenCalledTimes(4);
expect(policy.observe(false)).toBe("complete");
});
it("cancels timers on connection replacement and disposal", () => {
const retry = vi.fn();
const policy = new IncompleteUsageRetry({ retry });
policy.observe(true, {});
policy.useConnection({});
vi.advanceTimersByTime(5_000);
expect(retry).not.toHaveBeenCalled();
policy.observe(true);
policy.dispose();
vi.advanceTimersByTime(5_000);
expect(retry).not.toHaveBeenCalled();
});
it("lets an independent cycle retry after exhaustion without poll self-rearming", () => {
const retry = vi.fn();
const policy = new IncompleteUsageRetry({ retry });
for (let attempt = 0; attempt < 4; attempt += 1) {
policy.observe(true);
vi.advanceTimersByTime(5_000);
}
expect(retry).toHaveBeenCalledTimes(3);
policy.startCycle();
expect(policy.observe(true)).toBe("retrying");
vi.advanceTimersByTime(5_000);
expect(retry).toHaveBeenCalledTimes(4);
});
});
+85
View File
@@ -0,0 +1,85 @@
const INCOMPLETE_USAGE_RETRY_MS = 5_000;
const INCOMPLETE_USAGE_RETRY_LIMIT = 3;
type IncompleteUsageRetryOptions = {
retry: () => void;
retryMs?: number;
limit?: number;
};
type UsageRetryHost = {
addController: (controller: { hostDisconnected: () => void }) => void;
};
/** Closed convergence state: an incomplete payload is never a rendered answer. */
export type UsageRetryState = "complete" | "retrying" | "exhausted";
export function isUsageIncomplete(usage: { refreshing?: boolean } | null | undefined): boolean {
return usage?.refreshing === true;
}
export function createUsageRetry(
host: UsageRetryHost,
retry: () => void,
options?: Omit<IncompleteUsageRetryOptions, "retry">,
): IncompleteUsageRetry {
const controller = new IncompleteUsageRetry({ retry, ...options });
host.addController({ hostDisconnected: () => controller.dispose() });
return controller;
}
/** Keeps incomplete usage cache-cold while bounding automatic convergence attempts. */
export class IncompleteUsageRetry {
private timer: number | null = null;
private attempts = 0;
private connection: unknown;
constructor(private readonly options: IncompleteUsageRetryOptions) {}
observe(incomplete: boolean, connection?: unknown): UsageRetryState {
this.useConnection(connection);
this.clear();
if (!incomplete) {
this.attempts = 0;
return "complete";
}
if (this.attempts >= (this.options.limit ?? INCOMPLETE_USAGE_RETRY_LIMIT)) {
// Nothing will converge this payload on its own, so the caller has to
// report it. Rendering the empty provider list as a loaded answer is the
// silent-failure this marker exists to avoid.
return "exhausted";
}
this.attempts += 1;
this.timer = window.setTimeout(() => {
this.timer = null;
this.options.retry();
}, this.options.retryMs ?? INCOMPLETE_USAGE_RETRY_MS);
return "retrying";
}
/** Starts a user/lifecycle-owned refresh cycle without letting poll callbacks rearm it. */
startCycle(): void {
this.attempts = 0;
this.clear();
}
useConnection(connection: unknown): void {
if (connection === this.connection) {
return;
}
this.connection = connection;
this.startCycle();
}
dispose(): void {
this.clear();
}
private clear(): void {
if (this.timer === null) {
return;
}
window.clearTimeout(this.timer);
this.timer = null;
}
}
@@ -27,6 +27,7 @@ type ModelProvidersPageTestElement = HTMLElement & {
probe: (cardId: string, providers: string[]) => Promise<void>;
probeResults: Record<string, ModelsProbeResult>;
routeData: ModelProvidersRouteData | undefined;
requestUpdate: () => void;
saveDefaultModels: () => Promise<void>;
saveKey: (provider: string, configKey: string) => Promise<void>;
selectedAgentId: string;
@@ -53,6 +54,7 @@ function createHarness(initialScopeId: string) {
});
return () => releaseAuthStatus?.();
};
let usageStatus: unknown = { updatedAt: 1, providers: [] };
const request = vi.fn(async (method: string): Promise<unknown> => {
switch (method) {
case "models.authStatus": {
@@ -74,7 +76,7 @@ function createHarness(initialScopeId: string) {
case "config.get":
return { config: {}, hash: "hash" };
case "usage.status":
return { updatedAt: 1, providers: [] };
return usageStatus;
case "sessions.usage":
return { aggregates: { byProvider: [] } };
default:
@@ -170,6 +172,9 @@ function createHarness(initialScopeId: string) {
request,
runtimeConfig,
snapshot,
setUsageStatus: (value: unknown) => {
usageStatus = value;
},
};
}
@@ -184,9 +189,62 @@ function appendPage(context: ApplicationContext) {
afterEach(() => {
document.body.replaceChildren();
vi.useRealTimers();
vi.restoreAllMocks();
});
describe("ModelProvidersPage usage convergence", () => {
it("restarts an exhausted retry cycle on same-client reconnect", async () => {
vi.useFakeTimers();
const harness = createHarness("main");
harness.setUsageStatus({ updatedAt: 1, providers: [], refreshing: true });
const page = appendPage(harness.context);
await page.updateComplete;
await vi.advanceTimersByTimeAsync(15_000);
const usageCallsBeforeReconnect = harness.request.mock.calls.filter(
([method]) => method === "usage.status",
).length;
expect(usageCallsBeforeReconnect).toBe(4);
harness.snapshot.phase = "offline";
page.requestUpdate();
await page.updateComplete;
harness.snapshot.phase = "connected";
page.requestUpdate();
await page.updateComplete;
await vi.advanceTimersByTimeAsync(0);
expect(harness.request.mock.calls.filter(([method]) => method === "usage.status").length).toBe(
5,
);
});
it("replaces a pending pre-disconnect load before it can publish", async () => {
const harness = createHarness("main");
harness.setUsageStatus({ updatedAt: 1, providers: [] });
const releaseOldLoad = harness.deferNextAuthStatus();
const page = appendPage(harness.context);
await page.updateComplete;
harness.snapshot.phase = "offline";
page.requestUpdate();
await page.updateComplete;
harness.setUsageStatus({ updatedAt: 2, providers: [] });
harness.snapshot.phase = "connected";
page.requestUpdate();
await page.updateComplete;
await vi.waitFor(() =>
expect(
harness.request.mock.calls.filter(([method]) => method === "usage.status").length,
).toBe(2),
);
releaseOldLoad();
await vi.waitFor(() => expect(page.data?.providerUsage?.updatedAt).toBe(2));
});
});
describe("ModelProvidersPage agent scope", () => {
it("switches application ownership from the concrete agent picker", async () => {
const { agentSelection, context } = createHarness("main");
@@ -15,6 +15,7 @@ import { t } from "../../i18n/index.ts";
import { normalizeAgentLabel } from "../../lib/agents/display.ts";
import { createGatewayConnectionLifecycle } from "../../lib/gateway-connection-lifecycle.ts";
import { isGatewayMethodAdvertised } from "../../lib/gateway-methods.ts";
import { createUsageRetry, isUsageIncomplete } from "../../lib/incomplete-usage-retry.ts";
import { normalizeAgentId } from "../../lib/sessions/session-key.ts";
import { OpenClawLightDomElement } from "../../lit/openclaw-element.ts";
import { SubscriptionsController } from "../../lit/subscriptions-controller.ts";
@@ -44,6 +45,7 @@ import {
buildProviderApiKeyPatch,
DEFAULT_MODELS_REPLACE_PATHS,
} from "./mutations.ts";
import { mergeProbeResults } from "./probe-results.ts";
import { renderModelProviders, type ModelProviderRowMessage } from "./view.ts";
const MODEL_PROVIDERS_DOCS_URL = "https://docs.openclaw.ai/concepts/model-providers";
@@ -62,39 +64,6 @@ function isMissingMethodError(error: unknown): boolean {
);
}
const PROBE_FAILURE_PRIORITY: readonly ModelsProbeResult["status"][] = [
"auth",
"billing",
"rate_limit",
"timeout",
"format",
"no_model",
"unknown",
];
function mergeProbeResults(cardId: string, results: ModelsProbeResult[]): ModelsProbeResult {
if (results.length === 1) {
return results[0]!;
}
const status = results.some((result) => result.status === "ok")
? "ok"
: (PROBE_FAILURE_PRIORITY.find((candidate) =>
results.some((result) => result.status === candidate),
) ?? "unknown");
const error = results.find((result) => result.status === status)?.error;
return {
provider: cardId,
status,
...(error ? { error } : {}),
results: results.flatMap((result) =>
result.results.map((target) => ({
...target,
label: `${result.provider}: ${target.label}`,
})),
),
};
}
export class ModelProvidersPage extends OpenClawLightDomElement {
@consume({ context: applicationContext, subscribe: true })
private context!: ApplicationContext;
@@ -142,11 +111,9 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
signal,
}).then((data) => ({ client, data }))
: initialState,
onComplete: ({ client, data }) => {
this.data = data;
this.dataClient = client;
},
onComplete: ({ client, data }) => this.applyLoadedData(data, client),
});
private readonly usageRetry = createUsageRetry(this, () => void this.refresh({ force: false }));
private readonly subscriptions = new SubscriptionsController(this)
.watch(
() => this.context?.gateway,
@@ -186,20 +153,34 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
if (changed.has("routeData") && this.routeData) {
const selectedAgentId = this.resolveSelectedAgentId();
this.setSelectedAgent(selectedAgentId);
if ((this.routeData.agentId ?? "") === selectedAgentId) {
this.data = this.routeData.data;
this.dataClient = this.routeData.client;
} else {
this.data = null;
this.dataClient = null;
}
const matches = (this.routeData.agentId ?? "") === selectedAgentId;
this.applyLoadedData(
matches ? this.routeData.data : null,
matches ? this.routeData.client : null,
);
}
}
private applyLoadedData(data: ModelProvidersData | null, client: GatewayBrowserClient | null) {
this.data = data;
this.dataClient = client;
// The connection epoch scopes the retry budget: a reconnect is a fresh
// Gateway cache generation and must not inherit the old attempt count.
// The "exhausted" state is deliberately dropped here: these cards render from
// models.authStatus and stay useful without usage, and this page's
// failed-usage notice is owned separately. Usage owns the visible outcome.
this.usageRetry.observe(
data !== null && isUsageIncomplete(data.providerUsage),
this.connectionLifecycle.epoch,
);
}
override updated() {
const snapshot = this.context.gateway.snapshot;
if (this.connectionLifecycle.transition(snapshot)) {
this.resetConnectionState(snapshot.client, snapshot.phase === "connected");
const connected = snapshot.phase === "connected";
const changed = this.connectionLifecycle.transition(snapshot);
if (changed) {
this.resetConnectionState(snapshot.client, connected);
}
if (
!this.context.agents.state.agentsList &&
@@ -208,6 +189,13 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
) {
void this.context.agents.ensureList();
}
if (changed && connected && snapshot.client) {
// A reconnect reuses the client object, so the staleness check below
// cannot recognize it as a new Gateway cache generation. Refetch here
// rather than relying on the reset above having nulled `data`.
void this.refresh({ force: false });
return;
}
if (
snapshot.phase !== "connected" ||
!snapshot.client ||
@@ -222,6 +210,9 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
}
private resetConnectionState(client: GatewayBrowserClient | null, connected: boolean) {
this.usageRetry.useConnection(this.connectionLifecycle.epoch);
// A null run advances Task's call id, so a late pre-transition result cannot
// publish even when the underlying request ignores AbortSignal.
void this.refreshTask.run([null, this.selectedAgentId, false]);
this.busy = {};
this.messages = {};
@@ -0,0 +1,34 @@
import type { ModelsProbeResult } from "../../api/types.ts";
const PROBE_FAILURE_PRIORITY: readonly ModelsProbeResult["status"][] = [
"auth",
"billing",
"rate_limit",
"timeout",
"format",
"no_model",
"unknown",
];
export function mergeProbeResults(cardId: string, results: ModelsProbeResult[]): ModelsProbeResult {
if (results.length === 1) {
return results[0]!;
}
const status = results.some((result) => result.status === "ok")
? "ok"
: (PROBE_FAILURE_PRIORITY.find((candidate) =>
results.some((result) => result.status === candidate),
) ?? "unknown");
const error = results.find((result) => result.status === status)?.error;
return {
provider: cardId,
status,
...(error ? { error } : {}),
results: results.flatMap((result) =>
result.results.map((target) => ({
...target,
label: `${result.provider}: ${target.label}`,
})),
),
};
}
+19
View File
@@ -71,4 +71,23 @@ describe("UsageRefreshPolicy", () => {
policy.request("focus");
expect(reload).toHaveBeenCalledOnce();
});
it("restarts an exhausted retry budget for manual and focus cycles", () => {
const { policy, reload } = createPolicy();
for (let attempt = 0; attempt < 4; attempt += 1) {
policy.markLoaded({ incomplete: true });
vi.advanceTimersByTime(5_000);
}
expect(reload).toHaveBeenCalledTimes(3);
policy.request("manual");
policy.markLoaded({ incomplete: true });
vi.advanceTimersByTime(5_000);
expect(reload).toHaveBeenCalledTimes(5);
policy.request("focus");
policy.markLoaded({ incomplete: true });
vi.advanceTimersByTime(5_000);
expect(reload).toHaveBeenCalledTimes(7);
});
});
+31 -5
View File
@@ -1,3 +1,5 @@
import { IncompleteUsageRetry, type UsageRetryState } from "../../lib/incomplete-usage-retry.ts";
const USAGE_PAYLOAD_TTL_MS = 5 * 60_000;
type UsageRefreshReason = "focus" | "manual" | "poll" | "reconnect";
@@ -39,22 +41,43 @@ export class UsageRefreshPolicy {
private lastLoadedAtMs: number | null = null;
private pendingAutomaticRefresh = false;
private reloadPending = false;
private readonly incompleteUsageRetry = new IncompleteUsageRetry({
retry: () => this.request("poll"),
});
constructor(private readonly options: UsageRefreshPolicyOptions) {}
setLastLoadedAtMs(value: number | null): void {
this.lastLoadedAtMs = value;
setLastLoadedAtMs(
value: number | null,
params?: { incomplete?: boolean; connection?: unknown },
): UsageRetryState {
return this.applyLoadState(value, params?.incomplete === true, params?.connection);
}
markLoaded(): void {
this.lastLoadedAtMs = Date.now();
markLoaded(params?: { incomplete?: boolean; connection?: unknown }): UsageRetryState {
return this.applyLoadState(Date.now(), params?.incomplete === true, params?.connection);
}
resetPayload(): void {
this.lastLoadedAtMs = null;
this.applyLoadState(null, false);
this.reloadPending = false;
}
dispose(): void {
this.incompleteUsageRetry.dispose();
}
private applyLoadState(
loadedAtMs: number | null,
incomplete: boolean,
connection?: unknown,
): UsageRetryState {
const state = this.incompleteUsageRetry.observe(incomplete, connection);
// Incomplete provider usage must not start the TTL or focus/reconnect can skip recovery.
this.lastLoadedAtMs = state === "complete" ? loadedAtMs : null;
return state;
}
interrupt(): void {
this.reloadPending ||= this.options.isLoading();
}
@@ -86,6 +109,9 @@ export class UsageRefreshPolicy {
lastLoadedAtMs: this.lastLoadedAtMs,
});
if (decision === "fetch") {
if (reason !== "poll") {
this.incompleteUsageRetry.startCycle();
}
this.reload();
}
}
+10
View File
@@ -1,3 +1,4 @@
import type { CostUsageSummary } from "../../api/types.ts";
import type { PanelRefreshStatus } from "../../components/panel-refresh-status.ts";
// Control UI view renders usageTypes screen content.
import type {
@@ -14,6 +15,13 @@ export type UsageTotals = SessionsUsageTotals;
export type CostDailyEntry = CostUsageDailyEntry;
export type UsageAggregates = SessionsUsageResult["aggregates"];
export type UsageTaskValue = {
epoch: object;
result: SessionsUsageResult;
costSummary: CostUsageSummary;
providerUsageSummary: ProviderUsageSummary | null;
};
export type UsageColumnId =
| "channel"
| "agent"
@@ -48,6 +56,8 @@ type UsageDataState = {
costDaily: CostDailyEntry[];
cacheStatus: SessionsUsageResult["cacheStatus"];
providerUsage: ProviderUsageSummary["providers"];
/** The gateway never converged the refresh; the empty list is not an answer. */
providerUsageStalled: boolean;
};
export type UsageFilterState = {
+40
View File
@@ -15,6 +15,8 @@ type TestUsagePage = HTMLElement & {
usageTimeSeriesStatus: { error: string | null; hasLoaded: boolean; stale: boolean };
usageSessionLogs: SessionLogEntry[] | null;
usageSessionLogsStatus: { error: string | null; hasLoaded: boolean; stale: boolean };
providerUsageStalled: boolean;
routeData?: unknown;
loadSessionTimeSeries: (sessionKey: string) => Promise<void>;
loadSessionLogs: (sessionKey: string) => Promise<void>;
render: () => unknown;
@@ -78,6 +80,44 @@ afterEach(() => {
});
describe("UsagePage detail requests", () => {
it("marks provider usage stalled once the retry budget is spent", async () => {
const client = { request: vi.fn(async () => ({})) } as unknown as GatewayBrowserClient;
const page = await createPage(client);
const gateway = page.context.gateway;
const routeDataAt = (loadedAtMs: number) => ({
gateway,
gatewaySnapshot: gateway.snapshot,
client,
query: {
startDate: "2026-05-14",
endDate: "2026-05-14",
scope: "family" as const,
timeZone: "local" as const,
agentId: null,
},
result: null,
costSummary: null,
providerUsageSummary: { updatedAt: 1, providers: [], refreshing: true },
loadedAtMs,
error: null,
});
// Three bounded attempts, then the page owns the outcome. Without the wiring
// the empty provider list renders as a loaded answer.
for (let attempt = 0; attempt < 3; attempt += 1) {
page.routeData = routeDataAt(attempt);
await page.updateComplete;
expect(page.providerUsageStalled).toBe(false);
}
page.routeData = routeDataAt(3);
await page.updateComplete;
expect(page.providerUsageStalled).toBe(true);
page.routeData = { ...routeDataAt(4), providerUsageSummary: { updatedAt: 2, providers: [] } };
await page.updateComplete;
expect(page.providerUsageStalled).toBe(false);
});
it("commits only the latest time-series selection", async () => {
const first = deferred<SessionUsageTimeSeries>();
const second = deferred<SessionUsageTimeSeries>();
+32 -5
View File
@@ -25,6 +25,7 @@ import {
formatMissingOperatorReadScopeMessage,
isMissingOperatorReadScopeError,
} from "../../lib/gateway-errors.ts";
import { isUsageIncomplete } from "../../lib/incomplete-usage-retry.ts";
import {
requestSessionUsageLogs,
requestSessionUsageTimeSeries,
@@ -52,6 +53,7 @@ import {
type SessionLogEntry,
type SessionLogRole,
type UsageProps,
type UsageTaskValue,
} from "./types.ts";
import { renderUsage } from "./view.ts";
@@ -87,6 +89,7 @@ class UsagePage extends OpenClawLightDomElement {
@state() private usageResult: SessionsUsageResult | null = null;
@state() private usageCostSummary: CostUsageSummary | null = null;
@state() private providerUsageSummary: ProviderUsageSummary | null = null;
@state() private providerUsageStalled = false;
@state() private usageError: string | null = null;
@state() private usageStartDate = currentLocalDate();
@state() private usageEndDate = currentLocalDate();
@@ -129,6 +132,8 @@ class UsagePage extends OpenClawLightDomElement {
// Invalidation runs the Task with a null client to supersede stale completions.
// Track real gateway work separately so that no-op runs cannot block reconnect retries.
private usageTaskActiveClient: GatewayBrowserClient | null = null;
// The client survives transport reconnects, so retry budgets need a separate epoch.
private connectionEpoch: object = {};
private routeDataInitialized = false;
private routeDataEnabled = true;
private readonly refreshPolicy = new UsageRefreshPolicy({
@@ -182,8 +187,16 @@ class UsagePage extends OpenClawLightDomElement {
return initialState;
}
this.refreshPolicy.beginLoad();
const epoch = this.connectionEpoch;
const agentId = normalizedAgentId || undefined;
return requestUsageSnapshot(client, { startDate, endDate, agentId, scope, timeZone }, signal);
return {
epoch,
...(await requestUsageSnapshot(
client,
{ startDate, endDate, agentId, scope, timeZone },
signal,
)),
} satisfies UsageTaskValue;
},
onComplete: (value) => {
this.usageTaskActiveClient = null;
@@ -191,7 +204,11 @@ class UsagePage extends OpenClawLightDomElement {
this.usageCostSummary = value.costSummary;
this.providerUsageSummary = value.providerUsageSummary;
this.usageError = null;
this.refreshPolicy.markLoaded();
this.providerUsageStalled =
this.refreshPolicy.markLoaded({
incomplete: isUsageIncomplete(value.providerUsageSummary),
connection: value.epoch,
}) === "exhausted";
this.refreshPolicy.flushPending();
},
onError: (error) => {
@@ -274,6 +291,7 @@ class UsagePage extends OpenClawLightDomElement {
this.subscriptions.clear();
this.clearDateDebounce();
this.clearQueryDebounce();
this.refreshPolicy.dispose();
this.usageTaskActiveClient = null;
void this.usageTask.run(this.usageTaskArgs(null));
void this.usageTimeSeriesTask.run([null, ""]);
@@ -312,7 +330,11 @@ class UsagePage extends OpenClawLightDomElement {
this.usageResult = data.result;
this.usageCostSummary = data.costSummary;
this.providerUsageSummary = data.providerUsageSummary;
this.refreshPolicy.setLastLoadedAtMs(data.loadedAtMs);
this.providerUsageStalled =
this.refreshPolicy.setLastLoadedAtMs(data.loadedAtMs, {
incomplete: isUsageIncomplete(data.providerUsageSummary),
connection: this.connectionEpoch,
}) === "exhausted";
this.usageError = data.error;
}
@@ -339,6 +361,7 @@ class UsagePage extends OpenClawLightDomElement {
this.usageResult = null;
this.usageCostSummary = null;
this.providerUsageSummary = null;
this.providerUsageStalled = false;
this.refreshPolicy.resetPayload();
this.usageError = null;
this.usageAgentId = this.context.agentSelection.state.scopeId;
@@ -448,8 +471,11 @@ class UsagePage extends OpenClawLightDomElement {
return;
}
void this.context.agents.ensureList();
if (this.routeDataInitialized && (change.identityChanged || change.becameConnected)) {
this.refreshPolicy.request("reconnect");
if (change.identityChanged || change.becameConnected) {
this.connectionEpoch = {};
if (this.routeDataInitialized) {
this.refreshPolicy.request("reconnect");
}
}
}
@@ -502,6 +528,7 @@ class UsagePage extends OpenClawLightDomElement {
this.usageCostSummary?.cacheStatus,
),
providerUsage: this.providerUsageSummary?.providers ?? [],
providerUsageStalled: this.providerUsageStalled,
},
filters: {
startDate: this.usageStartDate,
+23
View File
@@ -63,6 +63,7 @@ function createUsageProps(overrides: Partial<UsageProps> = {}): UsageProps {
costDaily: [],
cacheStatus: undefined,
providerUsage: [],
providerUsageStalled: false,
},
filters: {
startDate: "2026-05-14",
@@ -367,6 +368,28 @@ describe("renderUsage", () => {
expect(onQueryDraftChange).toHaveBeenCalledWith(expect.stringContaining("provider:clear"));
});
it("reports a stalled provider refresh instead of hiding the section", () => {
const container = document.createElement("div");
render(
renderUsage(
createUsageProps({
data: {
...createUsageProps().data,
providerUsage: [],
providerUsageStalled: true,
},
}),
),
container,
);
const callout = container.querySelector(".usage-callout");
expect(callout?.textContent?.trim()).toBe(
"Provider usage did not finish loading. Refresh to retry.",
);
});
it("renders provider plans, quotas, and billing independently of session usage", () => {
const container = document.createElement("div");
+9 -2
View File
@@ -143,7 +143,14 @@ function renderUsageEmptyState(onRefresh: () => void) {
type ProviderUsageSnapshot = ProviderUsageSummary["providers"][number];
function renderProviderUsage(providers: ProviderUsageSnapshot[]) {
function renderProviderUsage(providers: ProviderUsageSnapshot[], stalled: boolean) {
if (stalled) {
// The stalled notice replaces the section: an empty grid under the normal
// header reads as "this operator has no providers", which is the lie.
return html`
<div class="callout warning usage-callout">${t("usage.providerUsage.stalled")}</div>
`;
}
if (providers.length === 0) {
return nothing;
}
@@ -798,7 +805,7 @@ export function renderUsage(props: UsageProps) {
</div>
</section>
${renderProviderUsage(data.providerUsage)}
${renderProviderUsage(data.providerUsage, data.providerUsageStalled)}
${isEmpty
? renderUsageEmptyState(filterActions.onRefresh)
: html`