mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
fix(gateway): refresh provider usage asynchronously
This commit is contained in:
@@ -184,6 +184,8 @@ private const val CRON_JOBS_PAGE_SIZE = 200
|
||||
private const val CRON_JOBS_MAX_PAGES = 100
|
||||
private const val CRON_JOBS_MAX_COUNT = CRON_JOBS_PAGE_SIZE * CRON_JOBS_MAX_PAGES
|
||||
private const val CRON_JOBS_SNAPSHOT_MAX_ATTEMPTS = 3
|
||||
private const val USAGE_INCOMPLETE_RETRY_DELAY_MS = 5_000L
|
||||
private const val USAGE_INCOMPLETE_RETRY_LIMIT = 3
|
||||
private const val OperatorAdminScope = "operator.admin"
|
||||
private const val OperatorPairingScope = "operator.pairing"
|
||||
|
||||
@@ -1282,6 +1284,8 @@ class NodeRuntime private constructor(
|
||||
val usageRefreshing: StateFlow<Boolean> = _usageRefreshing.asStateFlow()
|
||||
private val _usageErrorText = MutableStateFlow<NativeText?>(null)
|
||||
val usageErrorText: StateFlow<String?> = _usageErrorText.resolveOptionalNativeText()
|
||||
private val usageRefreshGuard = LatestGatewayRefreshGuard()
|
||||
private var usageIncompleteRetryJob: Job? = null
|
||||
private val _skillsSummary = MutableStateFlow(GatewaySkillsSummary(skills = emptyList()))
|
||||
val skillsSummary: StateFlow<GatewaySkillsSummary> = _skillsSummary.asStateFlow()
|
||||
private val _skillsRefreshing = MutableStateFlow(false)
|
||||
@@ -1361,6 +1365,9 @@ class NodeRuntime private constructor(
|
||||
@Volatile internal var gatewayDataRequestTimeoutObserverForTests: ((method: String, timeoutMs: Long) -> Unit)? = null
|
||||
|
||||
@Volatile internal var clawHubSkillInstallBeforeClaimObserverForTests: (() -> Unit)? = null
|
||||
|
||||
@Volatile internal var usageIncompleteRetryDelayMsForTests: Long? = null
|
||||
|
||||
private val _channelsSummary = MutableStateFlow(GatewayChannelsSummary(channels = emptyList()))
|
||||
val channelsSummary: StateFlow<GatewayChannelsSummary> = _channelsSummary.asStateFlow()
|
||||
private val _channelsRefreshing = MutableStateFlow(false)
|
||||
@@ -1732,6 +1739,8 @@ class NodeRuntime private constructor(
|
||||
if (retirePendingCronRuns) {
|
||||
pendingCronRunRegistry.clear { _pendingCronRunJobIds.value = it }
|
||||
}
|
||||
usageIncompleteRetryJob?.cancel()
|
||||
usageRefreshGuard.invalidate()
|
||||
_usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList())
|
||||
_usageRefreshing.value = false
|
||||
_usageErrorText.value = null
|
||||
@@ -6331,20 +6340,85 @@ class NodeRuntime private constructor(
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun refreshUsageFromGateway() =
|
||||
refreshGatewaySummary(
|
||||
summary = _usageSummary,
|
||||
refreshing = _usageRefreshing,
|
||||
errorText = _usageErrorText,
|
||||
disconnectedSummary = GatewayUsageSummary(updatedAtMs = null, providers = emptyList()),
|
||||
failureText = nativeText("Could not load usage."),
|
||||
) { gatewayScope ->
|
||||
val root = json.parseToJsonElement(requestGatewayData(gatewayScope, "usage.status", "{}")).asObjectOrNull()
|
||||
GatewayUsageSummary(
|
||||
updatedAtMs = root.long("updatedAt"),
|
||||
providers = parseUsageProviders(root?.get("providers") as? JsonArray),
|
||||
)
|
||||
private suspend fun refreshUsageFromGateway() {
|
||||
usageIncompleteRetryJob?.cancel()
|
||||
val gatewayScope = captureGatewayDataScope() ?: return
|
||||
val refreshGeneration = usageRefreshGuard.begin()
|
||||
if (refreshUsageOnceFromGateway(gatewayScope, refreshGeneration) && _usageSummary.value.refreshing) {
|
||||
scheduleIncompleteUsageRetry(gatewayScope, refreshGeneration)
|
||||
}
|
||||
}
|
||||
|
||||
private fun publishUsageRefresh(
|
||||
gatewayScope: GatewayDataScope,
|
||||
refreshGeneration: Long,
|
||||
publish: () -> Unit,
|
||||
): Boolean {
|
||||
var refreshCurrent = false
|
||||
val scopeCurrent =
|
||||
publishGatewayData(gatewayScope) {
|
||||
refreshCurrent = usageRefreshGuard.publishIfCurrent(refreshGeneration, publish)
|
||||
}
|
||||
return scopeCurrent && refreshCurrent
|
||||
}
|
||||
|
||||
private suspend fun refreshUsageOnceFromGateway(
|
||||
gatewayScope: GatewayDataScope,
|
||||
refreshGeneration: Long,
|
||||
): Boolean {
|
||||
publishUsageRefresh(gatewayScope, refreshGeneration) {
|
||||
_usageRefreshing.value = true
|
||||
_usageErrorText.value = null
|
||||
}
|
||||
if (!operatorConnected) {
|
||||
return publishUsageRefresh(gatewayScope, refreshGeneration) {
|
||||
_usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList())
|
||||
_usageRefreshing.value = false
|
||||
}
|
||||
}
|
||||
return try {
|
||||
val root = json.parseToJsonElement(requestGatewayData(gatewayScope, "usage.status", "{}")).asObjectOrNull()
|
||||
val nextSummary =
|
||||
GatewayUsageSummary(
|
||||
updatedAtMs = root.long("updatedAt"),
|
||||
providers = parseUsageProviders(root?.get("providers") as? JsonArray),
|
||||
refreshing = root.boolean("refreshing"),
|
||||
)
|
||||
publishUsageRefresh(gatewayScope, refreshGeneration) { _usageSummary.value = nextSummary }
|
||||
} catch (_: Throwable) {
|
||||
publishUsageRefresh(gatewayScope, refreshGeneration) {
|
||||
// Preserve same-identity provider rows across a transient refresh failure.
|
||||
_usageSummary.value = _usageSummary.value.copy(refreshing = false)
|
||||
_usageErrorText.value = nativeText("Could not load usage.")
|
||||
}
|
||||
} finally {
|
||||
publishUsageRefresh(gatewayScope, refreshGeneration) { _usageRefreshing.value = false }
|
||||
}
|
||||
}
|
||||
|
||||
private fun scheduleIncompleteUsageRetry(
|
||||
gatewayScope: GatewayDataScope,
|
||||
refreshGeneration: Long,
|
||||
) {
|
||||
// Mirrors the shared clients: three delayed retries, cancelled by a new cycle or gateway.
|
||||
usageIncompleteRetryJob?.cancel()
|
||||
usageIncompleteRetryJob =
|
||||
scope.launch {
|
||||
repeat(USAGE_INCOMPLETE_RETRY_LIMIT) {
|
||||
delay(usageIncompleteRetryDelayMsForTests ?: USAGE_INCOMPLETE_RETRY_DELAY_MS)
|
||||
if (!isGatewayDataScopeCurrent(gatewayScope)) return@launch
|
||||
if (!refreshUsageOnceFromGateway(gatewayScope, refreshGeneration)) return@launch
|
||||
if (!_usageSummary.value.refreshing) return@launch
|
||||
}
|
||||
publishUsageRefresh(gatewayScope, refreshGeneration) {
|
||||
// Clearing the marker alone renders as "No usage data yet.", which claims
|
||||
// the operator has no providers. Reuse the transient-failure text so a
|
||||
// spent retry budget reads as the load failure it is.
|
||||
_usageSummary.value = _usageSummary.value.copy(refreshing = false)
|
||||
_usageErrorText.value = nativeText("Could not load usage.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun refreshSkillsFromGateway(): Boolean =
|
||||
refreshGatewaySummary(
|
||||
@@ -8776,6 +8850,7 @@ data class GatewayCronJobSummary(
|
||||
data class GatewayUsageSummary(
|
||||
val updatedAtMs: Long?,
|
||||
val providers: List<GatewayUsageProviderSummary>,
|
||||
val refreshing: Boolean = false,
|
||||
)
|
||||
|
||||
data class GatewayUsageProviderSummary(
|
||||
|
||||
@@ -54,6 +54,7 @@ class ConnectionManager internal constructor(
|
||||
|
||||
internal const val AGENT_KIND_CLIENT_CAPABILITY = "agent-kind"
|
||||
internal const val INLINE_WIDGETS_CLIENT_CAPABILITY = "inline-widgets"
|
||||
internal const val USAGE_REFRESHING_CLIENT_CAPABILITY = "usage-refreshing"
|
||||
|
||||
internal fun operatorScopesForStoredDeviceToken(storedScopes: List<String>): List<String> {
|
||||
val normalized =
|
||||
@@ -233,6 +234,7 @@ class ConnectionManager internal constructor(
|
||||
buildList {
|
||||
add(AGENT_KIND_CLIENT_CAPABILITY)
|
||||
if (inlineWidgetsAvailable()) add(INLINE_WIDGETS_CLIENT_CAPABILITY)
|
||||
add(USAGE_REFRESHING_CLIENT_CAPABILITY)
|
||||
},
|
||||
commands = emptyList(),
|
||||
permissions = emptyMap(),
|
||||
|
||||
@@ -247,6 +247,7 @@ private fun UsageSettingsScreen(
|
||||
val isConnected by viewModel.isConnected.collectAsState()
|
||||
val providerCount = usageSummary.providers.size
|
||||
val issueCount = usageSummary.providers.count { it.error != null }
|
||||
val usageConverging = usageRefreshVisible(usageRefreshing, usageSummary.refreshing)
|
||||
|
||||
LaunchedEffect(isConnected) {
|
||||
if (isConnected) {
|
||||
@@ -264,7 +265,7 @@ private fun UsageSettingsScreen(
|
||||
),
|
||||
)
|
||||
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
ClawSecondaryButton(text = if (usageRefreshing) nativeString("Refreshing") else nativeString("Refresh"), onClick = viewModel::refreshUsage, enabled = isConnected && !usageRefreshing, modifier = Modifier.weight(1f))
|
||||
ClawSecondaryButton(text = if (usageConverging) nativeString("Refreshing") else nativeString("Refresh"), onClick = viewModel::refreshUsage, enabled = isConnected && !usageConverging, modifier = Modifier.weight(1f))
|
||||
}
|
||||
usageErrorText?.let { errorText ->
|
||||
ClawPanel {
|
||||
@@ -276,14 +277,17 @@ private fun UsageSettingsScreen(
|
||||
ClawPanel {
|
||||
Text(text = nativeString("Connect the gateway to load usage."), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
|
||||
}
|
||||
usageSummary.providers.isEmpty() ->
|
||||
usageSummary.providers.isNotEmpty() -> UsageProvidersPanel(providers = usageSummary.providers)
|
||||
// The warning panel above already reports a failed load; adding
|
||||
// "No usage data yet." beside it claims the operator has no providers.
|
||||
usageErrorText != null -> Unit
|
||||
else ->
|
||||
ClawPanel {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(3.dp)) {
|
||||
Text(text = nativeString("No usage data yet."), style = ClawTheme.type.section, color = ClawTheme.colors.text)
|
||||
Text(text = if (usageConverging) nativeString("Refreshing") else nativeString("No usage data yet."), style = ClawTheme.type.section, color = ClawTheme.colors.text)
|
||||
Text(text = nativeString("Provider limits will appear here when your gateway reports them."), style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
|
||||
}
|
||||
}
|
||||
else -> UsageProvidersPanel(providers = usageSummary.providers)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2553,6 +2557,11 @@ private fun UsageProvidersPanel(providers: List<GatewayUsageProviderSummary>) {
|
||||
}
|
||||
}
|
||||
|
||||
internal fun usageRefreshVisible(
|
||||
requestRefreshing: Boolean,
|
||||
summaryRefreshing: Boolean,
|
||||
): Boolean = requestRefreshing || summaryRefreshing
|
||||
|
||||
@Composable
|
||||
private fun UsageProviderListRow(provider: GatewayUsageProviderSummary) {
|
||||
val hasIssue = provider.error != null
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
package ai.openclaw.app
|
||||
|
||||
import ai.openclaw.app.gateway.GatewayEndpoint
|
||||
import ai.openclaw.app.ui.usageRefreshVisible
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertNotNull
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.RuntimeEnvironment
|
||||
import org.robolectric.annotation.Config
|
||||
import java.lang.reflect.Field
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(sdk = [34])
|
||||
class UsageStatusRuntimeTest {
|
||||
@Test
|
||||
fun incompleteUsageConvergesOnACompletedPayload() {
|
||||
val runtime = createRuntime()
|
||||
connect(runtime)
|
||||
runtime.usageIncompleteRetryDelayMsForTests = 10L
|
||||
val calls = AtomicInteger()
|
||||
runtime.gatewayDataRequestOverrideForTests = { _, method, _ ->
|
||||
check(method == "usage.status")
|
||||
if (calls.incrementAndGet() == 1) {
|
||||
"""{"updatedAt":1,"providers":[],"refreshing":true}"""
|
||||
} else {
|
||||
"""{"updatedAt":2,"providers":[{"displayName":"Claude","plan":"Pro","windows":[]}]}"""
|
||||
}
|
||||
}
|
||||
|
||||
runtime.refreshUsage()
|
||||
waitUntil {
|
||||
runtime.usageSummary.value.providers
|
||||
.isNotEmpty()
|
||||
}
|
||||
assertEquals(2, calls.get())
|
||||
assertFalse(runtime.usageSummary.value.refreshing)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun incompleteUsageRetriesStayBounded() {
|
||||
val runtime = createRuntime()
|
||||
connect(runtime)
|
||||
runtime.usageIncompleteRetryDelayMsForTests = 10L
|
||||
val calls = AtomicInteger()
|
||||
runtime.gatewayDataRequestOverrideForTests = { _, _, _ ->
|
||||
calls.incrementAndGet()
|
||||
"""{"updatedAt":1,"providers":[],"refreshing":true}"""
|
||||
}
|
||||
|
||||
runtime.refreshUsage()
|
||||
waitUntil { calls.get() == 4 }
|
||||
Thread.sleep(100)
|
||||
assertEquals(4, calls.get())
|
||||
assertFalse(runtime.usageSummary.value.refreshing)
|
||||
assertFalse(
|
||||
usageRefreshVisible(
|
||||
requestRefreshing = runtime.usageRefreshing.value,
|
||||
summaryRefreshing = runtime.usageSummary.value.refreshing,
|
||||
),
|
||||
)
|
||||
// Clearing the marker without this would render "No usage data yet.",
|
||||
// claiming the operator has no providers instead of a failed load.
|
||||
assertNotNull(runtime.usageErrorText.value)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aTransientFailurePreservesRowsAndStopsTheRetryChain() {
|
||||
val runtime = createRuntime()
|
||||
connect(runtime)
|
||||
runtime.usageIncompleteRetryDelayMsForTests = 10L
|
||||
val calls = AtomicInteger()
|
||||
runtime.gatewayDataRequestOverrideForTests = { _, _, _ ->
|
||||
if (calls.incrementAndGet() == 1) {
|
||||
"""{"updatedAt":1,"providers":[{"displayName":"Claude","plan":"Pro","windows":[]}],"refreshing":true}"""
|
||||
} else {
|
||||
error("usage unavailable")
|
||||
}
|
||||
}
|
||||
|
||||
runtime.refreshUsage()
|
||||
waitUntil {
|
||||
runtime.usageErrorText.value != null
|
||||
}
|
||||
Thread.sleep(100)
|
||||
assertEquals(2, calls.get())
|
||||
assertEquals(
|
||||
"Claude",
|
||||
runtime.usageSummary.value.providers
|
||||
.single()
|
||||
.displayName,
|
||||
)
|
||||
assertFalse(runtime.usageSummary.value.refreshing)
|
||||
}
|
||||
|
||||
private fun createRuntime(): NodeRuntime {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
val prefs =
|
||||
app.getSharedPreferences(
|
||||
"usage.${UUID.randomUUID()}",
|
||||
android.content.Context.MODE_PRIVATE,
|
||||
)
|
||||
return NodeRuntime(app, SecurePrefs(app, securePrefsOverride = prefs))
|
||||
}
|
||||
|
||||
private fun connect(runtime: NodeRuntime) {
|
||||
field(runtime, "connectedEndpoint").set(runtime, GatewayEndpoint.manual("127.0.0.1", 18789))
|
||||
field(runtime, "operatorConnected").set(runtime, true)
|
||||
}
|
||||
|
||||
private fun waitUntil(condition: () -> Boolean) {
|
||||
repeat(200) {
|
||||
if (condition()) return
|
||||
Thread.sleep(10)
|
||||
}
|
||||
error("condition did not become true")
|
||||
}
|
||||
|
||||
private fun field(
|
||||
target: Any,
|
||||
name: String,
|
||||
): Field {
|
||||
var type: Class<*>? = target.javaClass
|
||||
while (type != null) {
|
||||
try {
|
||||
return type.getDeclaredField(name).apply { isAccessible = true }
|
||||
} catch (_: NoSuchFieldException) {
|
||||
type = type.superclass
|
||||
}
|
||||
}
|
||||
error("field $name not found")
|
||||
}
|
||||
}
|
||||
@@ -139,6 +139,7 @@ class ConnectionManagerTest {
|
||||
listOf(
|
||||
ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY,
|
||||
ConnectionManager.INLINE_WIDGETS_CLIENT_CAPABILITY,
|
||||
ConnectionManager.USAGE_REFRESHING_CLIENT_CAPABILITY,
|
||||
),
|
||||
options.caps,
|
||||
)
|
||||
@@ -148,7 +149,13 @@ class ConnectionManagerTest {
|
||||
fun buildOperatorConnectOptions_omitsInlineWidgetsWithoutIsolatedWebViews() {
|
||||
val options = newManager(inlineWidgetsAvailable = false).buildOperatorConnectOptions()
|
||||
|
||||
assertEquals(listOf(ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY), options.caps)
|
||||
assertEquals(
|
||||
listOf(
|
||||
ConnectionManager.AGENT_KIND_CLIENT_CAPABILITY,
|
||||
ConnectionManager.USAGE_REFRESHING_CLIENT_CAPABILITY,
|
||||
),
|
||||
options.caps,
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -349,6 +349,13 @@ class SettingsScreensTest {
|
||||
assertEquals("None", formatCronTimestamp(null))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun usageRefreshStaysVisibleBetweenIncompleteRetries() {
|
||||
assertTrue(usageRefreshVisible(requestRefreshing = true, summaryRefreshing = false))
|
||||
assertTrue(usageRefreshVisible(requestRefreshing = false, summaryRefreshing = true))
|
||||
assertFalse(usageRefreshVisible(requestRefreshing = false, summaryRefreshing = false))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun approvalCardShowsTheWholeMonospacedCommandBeforeStackedActions() {
|
||||
val source = settingsScreensSource()
|
||||
|
||||
@@ -14,8 +14,11 @@ private let gatewayConnectionLogger = Logger(subsystem: "ai.openclaw", category:
|
||||
actor GatewayConnection {
|
||||
static let shared = GatewayConnection(
|
||||
endpointProvider: GatewayConnection.defaultEndpointProvider)
|
||||
nonisolated static let operatorClientCaps =
|
||||
[OpenClawGatewayClientCapability.agentKind, OpenClawGatewayClientCapability.inlineWidgets]
|
||||
nonisolated static let operatorClientCaps = [
|
||||
OpenClawGatewayClientCapability.agentKind,
|
||||
OpenClawGatewayClientCapability.inlineWidgets,
|
||||
OpenClawGatewayClientCapability.usageRefreshing,
|
||||
]
|
||||
|
||||
typealias Config = (url: URL, token: String?, password: String?)
|
||||
|
||||
|
||||
@@ -29,6 +29,11 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
|
||||
private var cachedUsageSummary: GatewayUsageSummary?
|
||||
private var usageCacheUpdatedAt: Date?
|
||||
private let usageRefreshIntervalSeconds: TimeInterval = 30
|
||||
private var usageRetryTask: Task<Void, Never>?
|
||||
private var usageRetryAttempts = 0
|
||||
private var usageLoadGeneration = 0
|
||||
private var usageRetryIntervalSeconds: TimeInterval = 5
|
||||
private let usageRetryLimit = 3
|
||||
private var cachedCostSummary: GatewayCostUsageSummary?
|
||||
private var cachedCostErrorText: String?
|
||||
private var costCacheUpdatedAt: Date?
|
||||
@@ -36,6 +41,9 @@ final class MenuSessionsInjector: NSObject, NSMenuDelegate {
|
||||
private let nodesStore = NodesStore.shared
|
||||
#if DEBUG
|
||||
private var testControlChannelConnected: Bool?
|
||||
private var testUsageLoad: (() async throws -> GatewayUsageSummary)?
|
||||
private var testUsageLoadDidFinish: (@MainActor () -> Void)?
|
||||
private var testUsageRetryDidExhaust: (@MainActor () -> Void)?
|
||||
#endif
|
||||
|
||||
func install(into statusItem: NSStatusItem) {
|
||||
@@ -420,7 +428,8 @@ extension MenuSessionsInjector {
|
||||
|
||||
private func insertUsageSection(into menu: NSMenu, at cursor: Int, width: CGFloat) -> Int {
|
||||
let rows = self.usageRows
|
||||
if rows.isEmpty {
|
||||
let stalled = self.isUsageStalled
|
||||
if rows.isEmpty, !stalled {
|
||||
return cursor
|
||||
}
|
||||
|
||||
@@ -444,6 +453,16 @@ extension MenuSessionsInjector {
|
||||
menu.insertItem(headerItem, at: cursor)
|
||||
cursor += 1
|
||||
|
||||
if rows.isEmpty {
|
||||
menu.insertItem(
|
||||
self.makeMessageItem(
|
||||
text: "Usage did not finish loading",
|
||||
symbolName: "exclamationmark.triangle",
|
||||
width: width),
|
||||
at: cursor)
|
||||
return cursor + 1
|
||||
}
|
||||
|
||||
if let selectedProvider = self.selectedUsageProviderId,
|
||||
let primary = rows.first(where: { $0.providerId.lowercased() == selectedProvider }),
|
||||
rows.count > 1
|
||||
@@ -516,6 +535,17 @@ extension MenuSessionsInjector {
|
||||
return summary.primaryRows()
|
||||
}
|
||||
|
||||
/// Retry budget spent with the cold marker still set. The empty row list is
|
||||
/// the Gateway saying "not loaded", so the menu must not hide the section
|
||||
/// the way it does for an operator who genuinely has no usage providers.
|
||||
/// Requires a live control channel: while disconnected the honest answer is
|
||||
/// the disconnected state, not a usage-specific warning.
|
||||
private var isUsageStalled: Bool {
|
||||
self.isControlChannelConnected
|
||||
&& self.cachedUsageSummary?.refreshing == true
|
||||
&& self.usageRetryAttempts >= self.usageRetryLimit
|
||||
}
|
||||
|
||||
private func buildUsageOverflowMenu(rows: [UsageRow], width: CGFloat) -> NSMenu {
|
||||
let menu = NSMenu()
|
||||
// Keep submenu delegate nil: reusing the status-menu delegate here causes
|
||||
@@ -776,19 +806,73 @@ extension MenuSessionsInjector {
|
||||
return
|
||||
}
|
||||
|
||||
self.usageLoadGeneration += 1
|
||||
let generation = self.usageLoadGeneration
|
||||
|
||||
guard self.isControlChannelConnected else {
|
||||
self.usageCacheUpdatedAt = Date()
|
||||
return
|
||||
}
|
||||
|
||||
self.usageRetryTask?.cancel()
|
||||
self.usageRetryTask = nil
|
||||
self.usageRetryAttempts = 0
|
||||
await self.loadUsageSummaryOnce(generation: generation)
|
||||
}
|
||||
|
||||
private func loadUsageSummaryOnce(generation: Int) async {
|
||||
#if DEBUG
|
||||
defer { self.testUsageLoadDidFinish?() }
|
||||
#endif
|
||||
do {
|
||||
self.cachedUsageSummary = try await UsageLoader.loadSummary()
|
||||
let summary = try await self.loadUsageSummary()
|
||||
guard generation == self.usageLoadGeneration else { return }
|
||||
self.cachedUsageSummary = summary
|
||||
if summary.refreshing == true {
|
||||
// A cold Gateway marker is not a cacheable answer; converge with bounded retries.
|
||||
self.usageCacheUpdatedAt = nil
|
||||
self.scheduleUsageRetry(generation: generation)
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
guard generation == self.usageLoadGeneration else { return }
|
||||
self.cachedUsageSummary = nil
|
||||
}
|
||||
self.usageCacheUpdatedAt = Date()
|
||||
}
|
||||
|
||||
private func scheduleUsageRetry(generation: Int) {
|
||||
guard self.usageRetryAttempts < self.usageRetryLimit else {
|
||||
#if DEBUG
|
||||
self.testUsageRetryDidExhaust?()
|
||||
#endif
|
||||
return
|
||||
}
|
||||
self.usageRetryAttempts += 1
|
||||
let interval = self.usageRetryIntervalSeconds
|
||||
self.usageRetryTask = Task { [weak self] in
|
||||
try? await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000))
|
||||
guard let self,
|
||||
!Task.isCancelled,
|
||||
self.isControlChannelConnected,
|
||||
generation == self.usageLoadGeneration
|
||||
else { return }
|
||||
await self.loadUsageSummaryOnce(generation: generation)
|
||||
// The final attempt leaves the marker set, so repaint on exhaustion
|
||||
// too or an operator watching the open menu never sees the outcome.
|
||||
if self.cachedUsageSummary?.refreshing != true || self.isUsageStalled {
|
||||
await self.repaintOpenMenu(self.statusItem?.menu)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func loadUsageSummary() async throws -> GatewayUsageSummary {
|
||||
#if DEBUG
|
||||
if let load = self.testUsageLoad { return try await load() }
|
||||
#endif
|
||||
return try await UsageLoader.loadSummary()
|
||||
}
|
||||
|
||||
private func refreshCostUsageCache(force: Bool) async {
|
||||
if !force,
|
||||
let updated = self.costCacheUpdatedAt,
|
||||
@@ -1295,6 +1379,34 @@ extension MenuSessionsInjector {
|
||||
self.usageCacheUpdatedAt = Date()
|
||||
}
|
||||
|
||||
func setTestingUsageLoader(_ load: (() async throws -> GatewayUsageSummary)?) {
|
||||
self.testUsageLoad = load
|
||||
}
|
||||
|
||||
func setTestingUsageLoadDidFinish(_ didFinish: (@MainActor () -> Void)?) {
|
||||
self.testUsageLoadDidFinish = didFinish
|
||||
}
|
||||
|
||||
func setTestingUsageRetryDidExhaust(_ didExhaust: (@MainActor () -> Void)?) {
|
||||
self.testUsageRetryDidExhaust = didExhaust
|
||||
}
|
||||
|
||||
func setTestingUsageRetryInterval(_ seconds: TimeInterval) {
|
||||
self.usageRetryIntervalSeconds = seconds
|
||||
}
|
||||
|
||||
func refreshUsageCacheForTesting(force: Bool) async {
|
||||
await self.refreshUsageCache(force: force)
|
||||
}
|
||||
|
||||
var testingCachedUsageSummary: GatewayUsageSummary? {
|
||||
self.cachedUsageSummary
|
||||
}
|
||||
|
||||
var testingUsageCacheUpdatedAt: Date? {
|
||||
self.usageCacheUpdatedAt
|
||||
}
|
||||
|
||||
func setTestingCostUsageSummary(_ summary: GatewayCostUsageSummary?, errorText: String? = nil) {
|
||||
self.cachedCostSummary = summary
|
||||
self.cachedCostErrorText = errorText
|
||||
|
||||
@@ -16,6 +16,13 @@ struct GatewayUsageProvider: Codable {
|
||||
struct GatewayUsageSummary: Codable {
|
||||
let updatedAt: Double
|
||||
let providers: [GatewayUsageProvider]
|
||||
let refreshing: Bool?
|
||||
|
||||
init(updatedAt: Double, providers: [GatewayUsageProvider], refreshing: Bool? = nil) {
|
||||
self.updatedAt = updatedAt
|
||||
self.providers = providers
|
||||
self.refreshing = refreshing
|
||||
}
|
||||
}
|
||||
|
||||
struct UsageRow: Identifiable {
|
||||
|
||||
@@ -10,6 +10,7 @@ struct MacGatewayChatTransportMappingTests {
|
||||
#expect(GatewayConnection.operatorClientCaps == [
|
||||
OpenClawGatewayClientCapability.agentKind,
|
||||
OpenClawGatewayClientCapability.inlineWidgets,
|
||||
OpenClawGatewayClientCapability.usageRefreshing,
|
||||
])
|
||||
}
|
||||
|
||||
|
||||
@@ -175,6 +175,125 @@ struct MenuSessionsInjectorTests {
|
||||
#expect(usageCostItem?.submenu?.delegate == nil)
|
||||
}
|
||||
|
||||
@Test func `cold incomplete usage converges without starting the cache ttl`() async {
|
||||
let injector = MenuSessionsInjector()
|
||||
injector.setTestingControlChannelConnected(true)
|
||||
injector.setTestingUsageRetryInterval(0)
|
||||
let events = UsageLoadEvents()
|
||||
injector.setTestingUsageLoadDidFinish { events.finished() }
|
||||
var calls = 0
|
||||
injector.setTestingUsageLoader {
|
||||
calls += 1
|
||||
if calls == 1 {
|
||||
return GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true)
|
||||
}
|
||||
return GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false)
|
||||
}
|
||||
|
||||
await injector.refreshUsageCacheForTesting(force: true)
|
||||
#expect(injector.testingUsageCacheUpdatedAt == nil)
|
||||
#expect(await events.waitFor(count: 2))
|
||||
#expect(calls == 2)
|
||||
#expect(injector.testingUsageCacheUpdatedAt != nil)
|
||||
}
|
||||
|
||||
@Test func `incomplete usage retry is bounded`() async {
|
||||
let injector = MenuSessionsInjector()
|
||||
injector.setTestingControlChannelConnected(true)
|
||||
injector.setTestingUsageRetryInterval(0)
|
||||
let events = UsageLoadEvents()
|
||||
injector.setTestingUsageLoadDidFinish { events.finished() }
|
||||
injector.setTestingUsageRetryDidExhaust { events.exhausted() }
|
||||
injector.setTestingUsageLoader {
|
||||
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true)
|
||||
}
|
||||
|
||||
await injector.refreshUsageCacheForTesting(force: true)
|
||||
#expect(await events.waitFor(count: 4))
|
||||
#expect(await events.waitForExhaustion())
|
||||
#expect(injector.testingUsageCacheUpdatedAt == nil)
|
||||
}
|
||||
|
||||
@Test func `stalled usage keeps a visible menu section`() async {
|
||||
let injector = MenuSessionsInjector()
|
||||
injector.setTestingControlChannelConnected(true)
|
||||
injector.setTestingUsageRetryInterval(0)
|
||||
let events = UsageLoadEvents()
|
||||
injector.setTestingUsageLoadDidFinish { events.finished() }
|
||||
injector.setTestingUsageRetryDidExhaust { events.exhausted() }
|
||||
|
||||
// An operator with no usage providers still gets no usage section.
|
||||
injector.setTestingUsageSummary(
|
||||
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false))
|
||||
let quiet = Self.makeMenuShell()
|
||||
injector.injectForTesting(into: quiet)
|
||||
let quietItems = quiet.items.count(where: { $0.tag == 9_415_557 })
|
||||
|
||||
injector.setTestingUsageLoader {
|
||||
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: true)
|
||||
}
|
||||
await injector.refreshUsageCacheForTesting(force: true)
|
||||
#expect(await events.waitForExhaustion())
|
||||
|
||||
// Spent budget with the marker still set: separator, header, and the
|
||||
// stalled row, never the silent menu an empty provider list produces.
|
||||
let stalled = Self.makeMenuShell()
|
||||
injector.injectForTesting(into: stalled)
|
||||
#expect(stalled.items.count(where: { $0.tag == 9_415_557 }) == quietItems + 3)
|
||||
}
|
||||
|
||||
private static func makeMenuShell() -> NSMenu {
|
||||
let menu = NSMenu()
|
||||
menu.addItem(NSMenuItem(title: "Header", action: nil, keyEquivalent: ""))
|
||||
menu.addItem(.separator())
|
||||
menu.addItem(NSMenuItem(title: "Send Heartbeats", action: nil, keyEquivalent: ""))
|
||||
menu.addItem(.separator())
|
||||
menu.addItem(NSMenuItem(title: "Settings…", action: nil, keyEquivalent: ""))
|
||||
return menu
|
||||
}
|
||||
|
||||
@Test func `late usage result from a replaced gateway is ignored`() async {
|
||||
let injector = MenuSessionsInjector()
|
||||
injector.setTestingControlChannelConnected(true)
|
||||
let loads = DeferredUsageLoads()
|
||||
injector.setTestingUsageLoader { try await loads.load() }
|
||||
|
||||
let first = Task { await injector.refreshUsageCacheForTesting(force: true) }
|
||||
#expect(await loads.waitForRequests(count: 1))
|
||||
let second = Task { await injector.refreshUsageCacheForTesting(force: true) }
|
||||
#expect(await loads.waitForRequests(count: 2))
|
||||
|
||||
loads.complete(
|
||||
at: 1,
|
||||
with: GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false))
|
||||
await second.value
|
||||
loads.complete(
|
||||
at: 0,
|
||||
with: GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false))
|
||||
await first.value
|
||||
|
||||
#expect(injector.testingCachedUsageSummary?.updatedAt == 2)
|
||||
}
|
||||
|
||||
@Test func `fresh no-op does not invalidate a forced usage load`() async {
|
||||
let injector = MenuSessionsInjector()
|
||||
injector.setTestingControlChannelConnected(true)
|
||||
injector.setTestingUsageSummary(
|
||||
GatewayUsageSummary(updatedAt: 1, providers: [], refreshing: false))
|
||||
let loads = DeferredUsageLoads()
|
||||
injector.setTestingUsageLoader { try await loads.load() }
|
||||
|
||||
let forced = Task { await injector.refreshUsageCacheForTesting(force: true) }
|
||||
#expect(await loads.waitForRequests(count: 1))
|
||||
await injector.refreshUsageCacheForTesting(force: false)
|
||||
loads.complete(
|
||||
at: 0,
|
||||
with: GatewayUsageSummary(updatedAt: 2, providers: [], refreshing: false))
|
||||
await forced.value
|
||||
|
||||
#expect(injector.testingCachedUsageSummary?.updatedAt == 2)
|
||||
}
|
||||
|
||||
@Test func `status text keeps useful error detail`() {
|
||||
let injector = MenuSessionsInjector()
|
||||
let longError = """
|
||||
@@ -237,3 +356,85 @@ struct MenuSessionsInjectorTests {
|
||||
connected: connected)
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class UsageLoadEvents {
|
||||
private struct Snapshot: Sendable {
|
||||
let completed: Int
|
||||
let exhausted: Bool
|
||||
}
|
||||
|
||||
private var completed = 0
|
||||
private var didExhaust = false
|
||||
private let stream: AsyncStream<Snapshot>
|
||||
private let continuation: AsyncStream<Snapshot>.Continuation
|
||||
|
||||
init() {
|
||||
(self.stream, self.continuation) = AsyncStream.makeStream(
|
||||
of: Snapshot.self,
|
||||
bufferingPolicy: .bufferingNewest(1))
|
||||
}
|
||||
|
||||
func finished() {
|
||||
self.completed += 1
|
||||
self.publish()
|
||||
}
|
||||
|
||||
func exhausted() {
|
||||
self.didExhaust = true
|
||||
self.publish()
|
||||
}
|
||||
|
||||
func waitFor(count: Int) async -> Bool {
|
||||
if self.completed >= count { return true }
|
||||
return await self.wait { $0.completed >= count }
|
||||
}
|
||||
|
||||
func waitForExhaustion() async -> Bool {
|
||||
if self.didExhaust { return true }
|
||||
return await self.wait { $0.exhausted }
|
||||
}
|
||||
|
||||
private func publish() {
|
||||
self.continuation.yield(Snapshot(completed: self.completed, exhausted: self.didExhaust))
|
||||
}
|
||||
|
||||
private func wait(_ predicate: @escaping @Sendable (Snapshot) -> Bool) async -> Bool {
|
||||
let stream = self.stream
|
||||
return await withTaskGroup(of: Bool.self) { group in
|
||||
group.addTask {
|
||||
for await snapshot in stream where predicate(snapshot) { return true }
|
||||
return false
|
||||
}
|
||||
group.addTask {
|
||||
try? await Task.sleep(nanoseconds: 5_000_000_000)
|
||||
return false
|
||||
}
|
||||
let result = await group.next() ?? false
|
||||
group.cancelAll()
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class DeferredUsageLoads {
|
||||
private var continuations: [CheckedContinuation<GatewayUsageSummary, Never>] = []
|
||||
|
||||
func load() async throws -> GatewayUsageSummary {
|
||||
await withCheckedContinuation { continuation in
|
||||
self.continuations.append(continuation)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRequests(count: Int) async -> Bool {
|
||||
for _ in 0..<100 where self.continuations.count < count {
|
||||
await Task.yield()
|
||||
}
|
||||
return self.continuations.count >= count
|
||||
}
|
||||
|
||||
func complete(at index: Int, with summary: GatewayUsageSummary) {
|
||||
self.continuations[index].resume(returning: summary)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import OpenClawProtocol
|
||||
public enum OpenClawGatewayClientCapability {
|
||||
public static let agentKind = "agent-kind"
|
||||
public static let inlineWidgets = "inline-widgets"
|
||||
public static let usageRefreshing = "usage-refreshing"
|
||||
}
|
||||
|
||||
public struct GatewayConnectOptions: Sendable {
|
||||
|
||||
@@ -97,9 +97,15 @@ const caps = [GATEWAY_CLIENT_CAPS.TOOL_EVENTS];
|
||||
|
||||
The current registry contains `approvals`, `exec-approvals`, `inline-widgets`,
|
||||
`run-tool-bindings`, `session-scoped-events`, `plugin-approvals`,
|
||||
`task-suggestions`, `terminal-offset-seq`, `tool-events`, and `ui-commands`.
|
||||
`task-suggestions`, `terminal-offset-seq`, `tool-events`, `ui-commands`, and
|
||||
`usage-refreshing`.
|
||||
Advertise only capabilities the client actually implements.
|
||||
|
||||
`usage-refreshing` allows a cold `usage.status` request to return immediately
|
||||
with `refreshing: true` and an empty provider list. A client advertising it must
|
||||
keep that payload cache-cold and refetch on a short bounded schedule. Other
|
||||
clients retain the blocking cold read.
|
||||
|
||||
<Warning>
|
||||
`tool-events` gates live tool-execution streaming. The Gateway registers only
|
||||
connections that advertise this capability as recipients for a run's structured
|
||||
|
||||
@@ -535,7 +535,7 @@ methods. Treat this as feature discovery, not a full enumeration of
|
||||
|
||||
<Accordion title="Models and usage">
|
||||
- `models.list` returns the runtime-allowed model catalog. See "`models.list` views" below.
|
||||
- `usage.status` returns provider usage windows/remaining quota summaries.
|
||||
- `usage.status` returns provider usage windows/remaining quota summaries. Clients advertising `usage-refreshing` receive an immediate `refreshing: true` placeholder on a cold cache and must refetch on a bounded schedule; other callers block for the cold provider read.
|
||||
- `usage.cost` returns aggregated cost usage summaries for a date range. Pass `agentId` for one agent, or `agentScope: "all"` to aggregate configured agents.
|
||||
- `doctor.memory.status` returns vector-memory / cached embedding readiness for the active default agent workspace. Pass `{ "probe": true }` or `{ "deep": true }` only for an explicit live embedding provider ping. Pass `{ "agentId": "agent-id" }` to scope Dreaming store stats to one agent workspace; omitting it aggregates configured Dreaming workspaces.
|
||||
- `doctor.memory.dreamDiary`, `doctor.memory.backfillDreamDiary`, `doctor.memory.resetDreamDiary`, `doctor.memory.resetGroundedShortTerm`, `doctor.memory.repairDreamingArtifacts`, and `doctor.memory.dedupeDreamDiary` accept optional `{ "agentId": "agent-id" }`; omitted, they operate on the configured default agent workspace.
|
||||
|
||||
@@ -90,6 +90,7 @@ export const GATEWAY_CLIENT_CAPS = {
|
||||
TERMINAL_OFFSET_SEQ: "terminal-offset-seq",
|
||||
TOOL_EVENTS: "tool-events",
|
||||
UI_COMMANDS: "ui-commands",
|
||||
USAGE_REFRESHING: "usage-refreshing",
|
||||
} as const;
|
||||
|
||||
/** Optional capability advertised by clients during gateway handshake. */
|
||||
|
||||
@@ -15,7 +15,8 @@ type GatewayHandlerInvocation = Parameters<GatewayRequestHandlers[string]>[0];
|
||||
|
||||
const BOARD_DATA_HANDLERS: Record<BoardDataBindingId, GatewayRequestHandlers[string]> = {
|
||||
"sessions.list": sessionReadHandlers["sessions.list"]!,
|
||||
"usage.status": usageHandlers["usage.status"]!,
|
||||
// Board reads are one-shot and cannot converge an incomplete marker.
|
||||
"usage.status": (invocation) => usageHandlers["usage.status"]!({ ...invocation, client: null }),
|
||||
"usage.cost": usageHandlers["usage.cost"]!,
|
||||
"cron.list": cronHandlers["cron.list"]!,
|
||||
"cron.status": cronHandlers["cron.status"]!,
|
||||
|
||||
@@ -197,8 +197,9 @@ function scheduleProviderUsageRefresh(params: {
|
||||
return usage;
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
// Usage is auxiliary and stale data remains valid. Keep failures visible
|
||||
// without delaying fresh auth-health responses.
|
||||
// Usage is auxiliary and stale data remains valid. A failed refresh
|
||||
// publishes nothing, so a capable client keeps seeing the incomplete
|
||||
// marker and reports it once its retry budget is spent.
|
||||
log.debug(
|
||||
`usage refresh failed: providers=${params.providerIds.join(",")} error=${formatForLog(err)}`,
|
||||
);
|
||||
@@ -225,26 +226,48 @@ type ProviderUsageCacheParams = {
|
||||
agentDir: string;
|
||||
configRef: OpenClawConfig;
|
||||
credentialKey: string;
|
||||
coldRead?: "refresh-marker";
|
||||
forceRefresh?: boolean;
|
||||
providerIds: UsageProviderId[];
|
||||
now: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Credential identity without the selection bookkeeping. `usageStats` is stamped
|
||||
* by every successful run (`markAuthProfileSuccess`), so treating it as an
|
||||
* identity change discards a good snapshot several times a minute on a busy
|
||||
* gateway — and a capable client would then receive a cold marker on every poll.
|
||||
* It still belongs in the refresh trigger because it can reorder profiles.
|
||||
*/
|
||||
function providerUsageCredentialIdentity(credentialKey: string): string {
|
||||
try {
|
||||
const { usageStats: _selectionBookkeeping, ...identity } = JSON.parse(credentialKey) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
return JSON.stringify(identity);
|
||||
} catch {
|
||||
return credentialKey;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveProviderUsageCacheRead(params: ProviderUsageCacheParams) {
|
||||
const providerIds = params.providerIds.toSorted();
|
||||
const providerKey = providerUsageCacheKey(providerIds);
|
||||
const credentialKey = scopeProviderUsageCredentialKey(params.credentialKey, providerIds);
|
||||
const credentialIdentity = providerUsageCredentialIdentity(credentialKey);
|
||||
const cached = usageCacheByAgentId.get(params.agentId);
|
||||
const matching =
|
||||
cached?.agentDir === params.agentDir &&
|
||||
cached.configRef === params.configRef &&
|
||||
cached.credentialKey === credentialKey &&
|
||||
providerUsageCredentialIdentity(cached.credentialKey) === credentialIdentity &&
|
||||
cached.providerKey === providerKey
|
||||
? cached
|
||||
: undefined;
|
||||
const needsRefresh =
|
||||
params.forceRefresh === true ||
|
||||
!matching ||
|
||||
matching.credentialKey !== credentialKey ||
|
||||
params.now - matching.refreshedAt >= USAGE_CACHE_TTL_MS;
|
||||
return { credentialKey, matching, needsRefresh, providerIds, providerKey };
|
||||
}
|
||||
@@ -273,7 +296,7 @@ export function readProviderUsageStaleWhileRevalidate(
|
||||
return matching?.usageByProvider ?? new Map();
|
||||
}
|
||||
|
||||
/** Returns cached provider usage, awaiting only a cold miss and refreshing stale data in place. */
|
||||
/** Returns cached provider usage while network refreshes run in the background for capable clients. */
|
||||
async function loadProviderUsageSummaryStaleWhileRevalidate(
|
||||
params: ProviderUsageCacheParams,
|
||||
): Promise<UsageSummary> {
|
||||
@@ -298,12 +321,17 @@ async function loadProviderUsageSummaryStaleWhileRevalidate(
|
||||
void refresh.catch(() => {});
|
||||
return matching.summary;
|
||||
}
|
||||
return await refresh;
|
||||
if (params.coldRead !== "refresh-marker") {
|
||||
return await refresh;
|
||||
}
|
||||
void refresh.catch(() => {});
|
||||
return { updatedAt: params.now, providers: [], refreshing: true };
|
||||
}
|
||||
|
||||
/** Shares the models.authStatus cache contract with the unscoped usage.status RPC. */
|
||||
export async function loadUsageStatusStaleWhileRevalidate(params: {
|
||||
config: OpenClawConfig;
|
||||
coldRead?: "refresh-marker";
|
||||
now?: number;
|
||||
}): Promise<UsageSummary> {
|
||||
const agentId = resolveLegacyInheritedAuthAgentId(params.config);
|
||||
@@ -342,6 +370,7 @@ export async function loadUsageStatusStaleWhileRevalidate(params: {
|
||||
store,
|
||||
}),
|
||||
providerIds,
|
||||
coldRead: params.coldRead,
|
||||
now: params.now ?? Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { readPreparedServerMethodModelCatalog } from "./optional-model-catalog.js";
|
||||
import { registerGatewayModelCatalogPrivateAccess } from "../server-model-catalog-auth.js";
|
||||
import {
|
||||
loadOptionalServerMethodModelCatalogSnapshot,
|
||||
readPreparedServerMethodModelCatalog,
|
||||
} from "./optional-model-catalog.js";
|
||||
import type { GatewayRequestContext } from "./types.js";
|
||||
|
||||
describe("readPreparedServerMethodModelCatalog", () => {
|
||||
@@ -20,3 +24,38 @@ describe("readPreparedServerMethodModelCatalog", () => {
|
||||
expect(loadGatewayModelCatalog).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadOptionalServerMethodModelCatalogSnapshot", () => {
|
||||
it("uses a prepared snapshot before cold discovery", async () => {
|
||||
const snapshot = {
|
||||
agentId: "work",
|
||||
agentDir: "/tmp/work-agent",
|
||||
catalogComplete: true,
|
||||
workspaceDir: "/tmp/work",
|
||||
config: {},
|
||||
entries: [],
|
||||
routeVariants: [],
|
||||
authModes: {},
|
||||
authStore: { version: 1, profiles: {} },
|
||||
authMaterializations: [],
|
||||
metadataSnapshot: { index: { plugins: [] }, plugins: [] } as never,
|
||||
};
|
||||
const loadGatewayModelCatalogSnapshot = vi.fn();
|
||||
const context = {
|
||||
loadGatewayModelCatalogSnapshot,
|
||||
logGateway: { debug: vi.fn() },
|
||||
} as unknown as GatewayRequestContext;
|
||||
const readPrepared = vi.fn(async () => snapshot);
|
||||
registerGatewayModelCatalogPrivateAccess(loadGatewayModelCatalogSnapshot, {
|
||||
loadDeferred: vi.fn(),
|
||||
readPrepared,
|
||||
});
|
||||
|
||||
await expect(
|
||||
loadOptionalServerMethodModelCatalogSnapshot(context, "chat.startup", {
|
||||
loadParams: { agentId: "work" },
|
||||
}),
|
||||
).resolves.toBe(snapshot);
|
||||
expect(loadGatewayModelCatalogSnapshot).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Optional model-catalog access gives session/tool methods metadata when ready
|
||||
// while keeping provider discovery out of ordinary request hot paths.
|
||||
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
|
||||
import { readPreparedCatalogSnapshot } from "../server-model-catalog-auth.js";
|
||||
import type { GatewayModelCatalogSnapshot } from "../server-model-catalog.types.js";
|
||||
import type { GatewayRequestContext } from "./types.js";
|
||||
|
||||
@@ -74,7 +75,11 @@ export function startOptionalServerMethodModelCatalogSnapshotLoad(
|
||||
loadParams?: Parameters<GatewayRequestContext["loadGatewayModelCatalogSnapshot"]>[0],
|
||||
): OptionalServerMethodModelCatalogLoad<GatewayModelCatalogSnapshot> {
|
||||
return startOptionalServerMethodModelCatalogValueLoad({
|
||||
load: () => context.loadGatewayModelCatalogSnapshot(loadParams),
|
||||
load: async () =>
|
||||
// Prepared startup facts avoid provider discovery. The cold loader remains
|
||||
// the fallback because it publishes the generation used by capability checks.
|
||||
(await readPreparedCatalogSnapshot(context, loadParams)) ??
|
||||
(await context.loadGatewayModelCatalogSnapshot(loadParams)),
|
||||
normalize: normalizeOptionalModelCatalogSnapshot,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -46,6 +46,8 @@ const config = {
|
||||
agents: { list: [{ id: "main", default: true }] },
|
||||
} as OpenClawConfig;
|
||||
|
||||
const refreshingCapableClient = { connect: { caps: ["usage-refreshing"] } };
|
||||
|
||||
function createStore(access = "access-one") {
|
||||
return {
|
||||
version: 1,
|
||||
@@ -61,7 +63,7 @@ function createStore(access = "access-one") {
|
||||
};
|
||||
}
|
||||
|
||||
async function runUsageStatus() {
|
||||
async function runUsageStatus(client?: unknown, getRuntimeConfig = () => config) {
|
||||
const respond = vi.fn();
|
||||
await expectDefined(
|
||||
usageHandlers["usage.status"],
|
||||
@@ -69,7 +71,8 @@ async function runUsageStatus() {
|
||||
)({
|
||||
respond,
|
||||
params: {},
|
||||
context: { getRuntimeConfig: () => config },
|
||||
context: { getRuntimeConfig },
|
||||
client: client === undefined ? refreshingCapableClient : client,
|
||||
} as unknown as Parameters<(typeof usageHandlers)["usage.status"]>[0]);
|
||||
expect(respond).toHaveBeenCalledTimes(1);
|
||||
expect(respond.mock.calls[0]?.[0]).toBe(true);
|
||||
@@ -114,7 +117,7 @@ describe("usage.status provider usage cache", () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("loads the cached provider snapshot from the exact runtime config", async () => {
|
||||
function mockExactConfigProviderUsage() {
|
||||
mocks.loadProviderUsageSummary.mockImplementation(async (options) => ({
|
||||
updatedAt: now,
|
||||
providers:
|
||||
@@ -129,15 +132,108 @@ describe("usage.status provider usage cache", () => {
|
||||
]
|
||||
: [],
|
||||
}));
|
||||
}
|
||||
|
||||
const result = (await runUsageStatus()) as {
|
||||
it("loads the cached provider snapshot from the exact runtime config", async () => {
|
||||
mockExactConfigProviderUsage();
|
||||
|
||||
// Clientless internal reads stay blocking, so the first response already
|
||||
// carries the snapshot the exact-config loader produced.
|
||||
const result = (await runUsageStatus(null)) as {
|
||||
providers: Array<{ accountEmail?: string }>;
|
||||
};
|
||||
expect(result.providers[0]?.accountEmail).toBe("configured@example.com");
|
||||
});
|
||||
|
||||
it("hands the exact runtime config to the background refresh", async () => {
|
||||
mockExactConfigProviderUsage();
|
||||
|
||||
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
|
||||
await vi.waitFor(async () => {
|
||||
expect(
|
||||
(await runUsageStatus()) as { providers: Array<{ accountEmail?: string }> },
|
||||
).toMatchObject({ providers: [{ accountEmail: "configured@example.com" }] });
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a cold marker only to clients that can converge it", async () => {
|
||||
let finish: ((value: { updatedAt: number; providers: never[] }) => void) | undefined;
|
||||
mocks.loadProviderUsageSummary.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finish = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(runUsageStatus()).resolves.toEqual({
|
||||
updatedAt: now,
|
||||
providers: [],
|
||||
refreshing: true,
|
||||
});
|
||||
|
||||
const legacy = runUsageStatus({ connect: { caps: [] } });
|
||||
const pending = Symbol("pending");
|
||||
await expect(
|
||||
Promise.race([
|
||||
legacy,
|
||||
new Promise((resolve) => {
|
||||
setTimeout(() => resolve(pending), 25);
|
||||
}),
|
||||
]),
|
||||
).resolves.toBe(pending);
|
||||
finish?.({ updatedAt: now, providers: [] });
|
||||
await expect(legacy).resolves.toEqual({ updatedAt: now, providers: [] });
|
||||
});
|
||||
|
||||
it("keeps serving usage while run bookkeeping churns the profile store", async () => {
|
||||
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
|
||||
await vi.waitFor(async () => {
|
||||
expect(await runUsageStatus()).toMatchObject({ providers: [{ provider: "openai" }] });
|
||||
});
|
||||
|
||||
// Every completed agent run stamps usage bookkeeping on the selected profile
|
||||
// (markAuthProfileSuccess). That must not read as a credential change, or a
|
||||
// busy gateway hands capable clients a cold marker on every poll and they
|
||||
// report "did not finish loading" while every refresh is succeeding.
|
||||
store = { ...store, usageStats: { "openai:default": { lastUsed: now } } };
|
||||
now += 1;
|
||||
|
||||
await expect(runUsageStatus()).resolves.toMatchObject({
|
||||
providers: [{ provider: "openai" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps clientless internal reads blocking", async () => {
|
||||
const result = (await runUsageStatus(null)) as { refreshing?: boolean; providers: unknown[] };
|
||||
expect(result.providers).toHaveLength(1);
|
||||
expect(result.refreshing).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps a failed refresh incomplete for capable clients and recovers", async () => {
|
||||
mocks.loadProviderUsageSummary.mockRejectedValueOnce(new Error("provider stack down"));
|
||||
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 0);
|
||||
});
|
||||
|
||||
// A failed refresh publishes nothing, so the payload stays marked incomplete
|
||||
// and the client's bounded retry owns reporting it. The next attempt recovers.
|
||||
await expect(runUsageStatus()).resolves.toMatchObject({ refreshing: true });
|
||||
await vi.waitFor(async () => {
|
||||
expect((await runUsageStatus()) as { providers: unknown[] }).toMatchObject({
|
||||
providers: [expect.any(Object)],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses byte-identical results within 60s and refreshes stale data in the background", async () => {
|
||||
const first = await runUsageStatus();
|
||||
expect(await runUsageStatus()).toMatchObject({ refreshing: true });
|
||||
const first = await vi.waitFor(async () => {
|
||||
const value = (await runUsageStatus()) as { providers: unknown[]; refreshing?: boolean };
|
||||
expect(value.providers).toHaveLength(1);
|
||||
return value;
|
||||
});
|
||||
expect(first.refreshing).toBeUndefined();
|
||||
const repeated = await runUsageStatus();
|
||||
|
||||
expect(JSON.stringify(repeated)).toBe(JSON.stringify(first));
|
||||
@@ -157,10 +253,33 @@ describe("usage.status provider usage cache", () => {
|
||||
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("invalidates cached usage when the runtime config changes", async () => {
|
||||
const configFor = (baseUrl: string) =>
|
||||
({ ...config, models: { providers: { openai: { baseUrl } } } }) as unknown as OpenClawConfig;
|
||||
const first = configFor("https://one.example/v1");
|
||||
expect(await runUsageStatus(refreshingCapableClient, () => first)).toMatchObject({
|
||||
refreshing: true,
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
expect(
|
||||
((await runUsageStatus(refreshingCapableClient, () => first)) as { providers: unknown[] })
|
||||
.providers,
|
||||
).toHaveLength(1);
|
||||
});
|
||||
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1);
|
||||
|
||||
const second = configFor("https://two.example/v1");
|
||||
expect(await runUsageStatus(refreshingCapableClient, () => second)).toMatchObject({
|
||||
refreshing: true,
|
||||
});
|
||||
await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2));
|
||||
});
|
||||
|
||||
it("shares the raw snapshot with models.authStatus and invalidates on credential rotation", async () => {
|
||||
await runUsageStatus();
|
||||
expect(await runUsageStatus()).toMatchObject({ refreshing: true });
|
||||
const agentId = resolveDefaultAgentId(config);
|
||||
const agentDir = resolveAgentDir(config, agentId);
|
||||
await vi.waitFor(() => expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1));
|
||||
const usage = readProviderUsageStaleWhileRevalidate({
|
||||
agentId,
|
||||
agentDir,
|
||||
@@ -177,10 +296,13 @@ describe("usage.status provider usage cache", () => {
|
||||
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(1);
|
||||
|
||||
store.profiles["openai:default"].access = "access-two";
|
||||
const rotated = (await runUsageStatus()) as {
|
||||
providers: Array<{ windows: Array<{ usedPercent: number }> }>;
|
||||
};
|
||||
expect(rotated.providers[0]?.windows[0]?.usedPercent).toBe(20);
|
||||
expect(await runUsageStatus()).toMatchObject({ refreshing: true });
|
||||
await vi.waitFor(async () => {
|
||||
const rotated = (await runUsageStatus()) as {
|
||||
providers: Array<{ windows: Array<{ usedPercent: number }> }>;
|
||||
};
|
||||
expect(rotated.providers[0]?.windows[0]?.usedPercent).toBe(20);
|
||||
});
|
||||
expect(mocks.loadProviderUsageSummary).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
import fs from "node:fs";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import {
|
||||
GATEWAY_CLIENT_CAPS,
|
||||
hasGatewayClientCap,
|
||||
} from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import {
|
||||
ErrorCodes,
|
||||
errorShape,
|
||||
@@ -1133,9 +1137,21 @@ export const testApi = {
|
||||
export type { SessionUsageEntry, SessionsUsageAggregates, SessionsUsageResult };
|
||||
|
||||
export const usageHandlers: GatewayRequestHandlers = {
|
||||
"usage.status": async ({ respond, context }) => {
|
||||
"usage.status": async ({ respond, context, client }) => {
|
||||
// Only clients with bounded retry machinery may receive an incomplete cold result.
|
||||
// In-process dispatch reuses the originating request's client, capabilities
|
||||
// included, so a plugin proxying this method inside a capable UI request
|
||||
// would inherit the marker without any way to converge it. Such a caller
|
||||
// must pass a capless client, the way board bindings force `client: null`.
|
||||
const coldRead = hasGatewayClientCap(
|
||||
client?.connect?.caps,
|
||||
GATEWAY_CLIENT_CAPS.USAGE_REFRESHING,
|
||||
)
|
||||
? ("refresh-marker" as const)
|
||||
: undefined;
|
||||
const summary = await loadUsageStatusStaleWhileRevalidate({
|
||||
config: context.getRuntimeConfig(),
|
||||
coldRead,
|
||||
});
|
||||
respond(true, summary, undefined);
|
||||
},
|
||||
|
||||
@@ -66,5 +66,13 @@ export async function readPreparedCatalog(
|
||||
context: Pick<GatewayRequestContext, "loadGatewayModelCatalogSnapshot">,
|
||||
agentId: string,
|
||||
): Promise<PreparedGatewayModelCatalogSnapshot | undefined> {
|
||||
return await requirePrivateAccess(context).readPrepared({ agentId });
|
||||
return await readPreparedCatalogSnapshot(context, { agentId });
|
||||
}
|
||||
|
||||
/** Reads a prepared owner snapshot without exposing private auth facts on the request context. */
|
||||
export async function readPreparedCatalogSnapshot(
|
||||
context: Pick<GatewayRequestContext, "loadGatewayModelCatalogSnapshot">,
|
||||
params?: Omit<GatewayModelCatalogReadParams, "readOnly">,
|
||||
): Promise<PreparedGatewayModelCatalogSnapshot | undefined> {
|
||||
return await requirePrivateAccess(context).readPrepared(params);
|
||||
}
|
||||
|
||||
@@ -86,6 +86,8 @@ export type ProviderUsageSnapshot = {
|
||||
export type UsageSummary = {
|
||||
updatedAt: number;
|
||||
providers: ProviderUsageSnapshot[];
|
||||
/** A background refresh owns the real values; an empty list is incomplete. */
|
||||
refreshing?: boolean;
|
||||
};
|
||||
|
||||
/** Normalized provider id. Usage providers are discovered from plugin hooks at runtime. */
|
||||
|
||||
@@ -471,6 +471,7 @@ describe("GatewayBrowserClient", () => {
|
||||
GATEWAY_CLIENT_CAPS.TOOL_EVENTS,
|
||||
GATEWAY_CLIENT_CAPS.INLINE_WIDGETS,
|
||||
GATEWAY_CLIENT_CAPS.UI_COMMANDS,
|
||||
GATEWAY_CLIENT_CAPS.USAGE_REFRESHING,
|
||||
]);
|
||||
expect(connectFrame.params?.scopes).toEqual([...CONTROL_UI_OPERATOR_SCOPES]);
|
||||
});
|
||||
|
||||
@@ -483,6 +483,7 @@ export class GatewayBrowserClient {
|
||||
GATEWAY_CLIENT_CAPS.TOOL_EVENTS,
|
||||
GATEWAY_CLIENT_CAPS.INLINE_WIDGETS,
|
||||
GATEWAY_CLIENT_CAPS.UI_COMMANDS,
|
||||
GATEWAY_CLIENT_CAPS.USAGE_REFRESHING,
|
||||
],
|
||||
auth: buildGatewayConnectAuth(selectedAuth),
|
||||
userAgent: navigator.userAgent,
|
||||
|
||||
@@ -4369,6 +4369,7 @@ export const en: TranslationMap = {
|
||||
providerUsage: {
|
||||
title: "Provider plans & billing",
|
||||
subtitle: "Live plan, quota, balance, and budget data reported by configured providers.",
|
||||
stalled: "Provider usage did not finish loading. Refresh to retry.",
|
||||
balance: "Balance",
|
||||
spend: "Usage",
|
||||
budget: "Budget",
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
// @vitest-environment jsdom
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { IncompleteUsageRetry, isUsageIncomplete } from "./incomplete-usage-retry.ts";
|
||||
|
||||
describe("IncompleteUsageRetry", () => {
|
||||
beforeEach(() => vi.useFakeTimers());
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it("recognizes only an explicit incomplete marker", () => {
|
||||
expect(isUsageIncomplete({ refreshing: true })).toBe(true);
|
||||
expect(isUsageIncomplete({ refreshing: false })).toBe(false);
|
||||
expect(isUsageIncomplete(null)).toBe(false);
|
||||
});
|
||||
|
||||
it("reports exhaustion once the three delayed attempts are spent", () => {
|
||||
const retry = vi.fn();
|
||||
const policy = new IncompleteUsageRetry({ retry });
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
expect(policy.observe(true)).toBe("retrying");
|
||||
vi.advanceTimersByTime(5_000);
|
||||
}
|
||||
expect(retry).toHaveBeenCalledTimes(3);
|
||||
// Nothing schedules another attempt, so the page owns the outcome from here.
|
||||
expect(policy.observe(true)).toBe("exhausted");
|
||||
vi.advanceTimersByTime(5_000);
|
||||
expect(retry).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("resets on completion or connection replacement", () => {
|
||||
const retry = vi.fn();
|
||||
const policy = new IncompleteUsageRetry({ retry });
|
||||
const first = {};
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
policy.observe(true, first);
|
||||
vi.advanceTimersByTime(5_000);
|
||||
}
|
||||
expect(retry).toHaveBeenCalledTimes(3);
|
||||
|
||||
policy.observe(true, {});
|
||||
vi.advanceTimersByTime(5_000);
|
||||
expect(retry).toHaveBeenCalledTimes(4);
|
||||
expect(policy.observe(false)).toBe("complete");
|
||||
});
|
||||
|
||||
it("cancels timers on connection replacement and disposal", () => {
|
||||
const retry = vi.fn();
|
||||
const policy = new IncompleteUsageRetry({ retry });
|
||||
policy.observe(true, {});
|
||||
policy.useConnection({});
|
||||
vi.advanceTimersByTime(5_000);
|
||||
expect(retry).not.toHaveBeenCalled();
|
||||
|
||||
policy.observe(true);
|
||||
policy.dispose();
|
||||
vi.advanceTimersByTime(5_000);
|
||||
expect(retry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("lets an independent cycle retry after exhaustion without poll self-rearming", () => {
|
||||
const retry = vi.fn();
|
||||
const policy = new IncompleteUsageRetry({ retry });
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
policy.observe(true);
|
||||
vi.advanceTimersByTime(5_000);
|
||||
}
|
||||
expect(retry).toHaveBeenCalledTimes(3);
|
||||
|
||||
policy.startCycle();
|
||||
expect(policy.observe(true)).toBe("retrying");
|
||||
vi.advanceTimersByTime(5_000);
|
||||
expect(retry).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
const INCOMPLETE_USAGE_RETRY_MS = 5_000;
|
||||
const INCOMPLETE_USAGE_RETRY_LIMIT = 3;
|
||||
|
||||
type IncompleteUsageRetryOptions = {
|
||||
retry: () => void;
|
||||
retryMs?: number;
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
type UsageRetryHost = {
|
||||
addController: (controller: { hostDisconnected: () => void }) => void;
|
||||
};
|
||||
|
||||
/** Closed convergence state: an incomplete payload is never a rendered answer. */
|
||||
export type UsageRetryState = "complete" | "retrying" | "exhausted";
|
||||
|
||||
export function isUsageIncomplete(usage: { refreshing?: boolean } | null | undefined): boolean {
|
||||
return usage?.refreshing === true;
|
||||
}
|
||||
|
||||
export function createUsageRetry(
|
||||
host: UsageRetryHost,
|
||||
retry: () => void,
|
||||
options?: Omit<IncompleteUsageRetryOptions, "retry">,
|
||||
): IncompleteUsageRetry {
|
||||
const controller = new IncompleteUsageRetry({ retry, ...options });
|
||||
host.addController({ hostDisconnected: () => controller.dispose() });
|
||||
return controller;
|
||||
}
|
||||
|
||||
/** Keeps incomplete usage cache-cold while bounding automatic convergence attempts. */
|
||||
export class IncompleteUsageRetry {
|
||||
private timer: number | null = null;
|
||||
private attempts = 0;
|
||||
private connection: unknown;
|
||||
|
||||
constructor(private readonly options: IncompleteUsageRetryOptions) {}
|
||||
|
||||
observe(incomplete: boolean, connection?: unknown): UsageRetryState {
|
||||
this.useConnection(connection);
|
||||
this.clear();
|
||||
if (!incomplete) {
|
||||
this.attempts = 0;
|
||||
return "complete";
|
||||
}
|
||||
if (this.attempts >= (this.options.limit ?? INCOMPLETE_USAGE_RETRY_LIMIT)) {
|
||||
// Nothing will converge this payload on its own, so the caller has to
|
||||
// report it. Rendering the empty provider list as a loaded answer is the
|
||||
// silent-failure this marker exists to avoid.
|
||||
return "exhausted";
|
||||
}
|
||||
this.attempts += 1;
|
||||
this.timer = window.setTimeout(() => {
|
||||
this.timer = null;
|
||||
this.options.retry();
|
||||
}, this.options.retryMs ?? INCOMPLETE_USAGE_RETRY_MS);
|
||||
return "retrying";
|
||||
}
|
||||
|
||||
/** Starts a user/lifecycle-owned refresh cycle without letting poll callbacks rearm it. */
|
||||
startCycle(): void {
|
||||
this.attempts = 0;
|
||||
this.clear();
|
||||
}
|
||||
|
||||
useConnection(connection: unknown): void {
|
||||
if (connection === this.connection) {
|
||||
return;
|
||||
}
|
||||
this.connection = connection;
|
||||
this.startCycle();
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.clear();
|
||||
}
|
||||
|
||||
private clear(): void {
|
||||
if (this.timer === null) {
|
||||
return;
|
||||
}
|
||||
window.clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ type ModelProvidersPageTestElement = HTMLElement & {
|
||||
probe: (cardId: string, providers: string[]) => Promise<void>;
|
||||
probeResults: Record<string, ModelsProbeResult>;
|
||||
routeData: ModelProvidersRouteData | undefined;
|
||||
requestUpdate: () => void;
|
||||
saveDefaultModels: () => Promise<void>;
|
||||
saveKey: (provider: string, configKey: string) => Promise<void>;
|
||||
selectedAgentId: string;
|
||||
@@ -53,6 +54,7 @@ function createHarness(initialScopeId: string) {
|
||||
});
|
||||
return () => releaseAuthStatus?.();
|
||||
};
|
||||
let usageStatus: unknown = { updatedAt: 1, providers: [] };
|
||||
const request = vi.fn(async (method: string): Promise<unknown> => {
|
||||
switch (method) {
|
||||
case "models.authStatus": {
|
||||
@@ -74,7 +76,7 @@ function createHarness(initialScopeId: string) {
|
||||
case "config.get":
|
||||
return { config: {}, hash: "hash" };
|
||||
case "usage.status":
|
||||
return { updatedAt: 1, providers: [] };
|
||||
return usageStatus;
|
||||
case "sessions.usage":
|
||||
return { aggregates: { byProvider: [] } };
|
||||
default:
|
||||
@@ -170,6 +172,9 @@ function createHarness(initialScopeId: string) {
|
||||
request,
|
||||
runtimeConfig,
|
||||
snapshot,
|
||||
setUsageStatus: (value: unknown) => {
|
||||
usageStatus = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -184,9 +189,62 @@ function appendPage(context: ApplicationContext) {
|
||||
|
||||
afterEach(() => {
|
||||
document.body.replaceChildren();
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("ModelProvidersPage usage convergence", () => {
|
||||
it("restarts an exhausted retry cycle on same-client reconnect", async () => {
|
||||
vi.useFakeTimers();
|
||||
const harness = createHarness("main");
|
||||
harness.setUsageStatus({ updatedAt: 1, providers: [], refreshing: true });
|
||||
const page = appendPage(harness.context);
|
||||
await page.updateComplete;
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
|
||||
const usageCallsBeforeReconnect = harness.request.mock.calls.filter(
|
||||
([method]) => method === "usage.status",
|
||||
).length;
|
||||
expect(usageCallsBeforeReconnect).toBe(4);
|
||||
|
||||
harness.snapshot.phase = "offline";
|
||||
page.requestUpdate();
|
||||
await page.updateComplete;
|
||||
harness.snapshot.phase = "connected";
|
||||
page.requestUpdate();
|
||||
await page.updateComplete;
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(harness.request.mock.calls.filter(([method]) => method === "usage.status").length).toBe(
|
||||
5,
|
||||
);
|
||||
});
|
||||
|
||||
it("replaces a pending pre-disconnect load before it can publish", async () => {
|
||||
const harness = createHarness("main");
|
||||
harness.setUsageStatus({ updatedAt: 1, providers: [] });
|
||||
const releaseOldLoad = harness.deferNextAuthStatus();
|
||||
const page = appendPage(harness.context);
|
||||
await page.updateComplete;
|
||||
|
||||
harness.snapshot.phase = "offline";
|
||||
page.requestUpdate();
|
||||
await page.updateComplete;
|
||||
harness.setUsageStatus({ updatedAt: 2, providers: [] });
|
||||
harness.snapshot.phase = "connected";
|
||||
page.requestUpdate();
|
||||
await page.updateComplete;
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(
|
||||
harness.request.mock.calls.filter(([method]) => method === "usage.status").length,
|
||||
).toBe(2),
|
||||
);
|
||||
releaseOldLoad();
|
||||
await vi.waitFor(() => expect(page.data?.providerUsage?.updatedAt).toBe(2));
|
||||
});
|
||||
});
|
||||
|
||||
describe("ModelProvidersPage agent scope", () => {
|
||||
it("switches application ownership from the concrete agent picker", async () => {
|
||||
const { agentSelection, context } = createHarness("main");
|
||||
|
||||
@@ -15,6 +15,7 @@ import { t } from "../../i18n/index.ts";
|
||||
import { normalizeAgentLabel } from "../../lib/agents/display.ts";
|
||||
import { createGatewayConnectionLifecycle } from "../../lib/gateway-connection-lifecycle.ts";
|
||||
import { isGatewayMethodAdvertised } from "../../lib/gateway-methods.ts";
|
||||
import { createUsageRetry, isUsageIncomplete } from "../../lib/incomplete-usage-retry.ts";
|
||||
import { normalizeAgentId } from "../../lib/sessions/session-key.ts";
|
||||
import { OpenClawLightDomElement } from "../../lit/openclaw-element.ts";
|
||||
import { SubscriptionsController } from "../../lit/subscriptions-controller.ts";
|
||||
@@ -44,6 +45,7 @@ import {
|
||||
buildProviderApiKeyPatch,
|
||||
DEFAULT_MODELS_REPLACE_PATHS,
|
||||
} from "./mutations.ts";
|
||||
import { mergeProbeResults } from "./probe-results.ts";
|
||||
import { renderModelProviders, type ModelProviderRowMessage } from "./view.ts";
|
||||
|
||||
const MODEL_PROVIDERS_DOCS_URL = "https://docs.openclaw.ai/concepts/model-providers";
|
||||
@@ -62,39 +64,6 @@ function isMissingMethodError(error: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
const PROBE_FAILURE_PRIORITY: readonly ModelsProbeResult["status"][] = [
|
||||
"auth",
|
||||
"billing",
|
||||
"rate_limit",
|
||||
"timeout",
|
||||
"format",
|
||||
"no_model",
|
||||
"unknown",
|
||||
];
|
||||
|
||||
function mergeProbeResults(cardId: string, results: ModelsProbeResult[]): ModelsProbeResult {
|
||||
if (results.length === 1) {
|
||||
return results[0]!;
|
||||
}
|
||||
const status = results.some((result) => result.status === "ok")
|
||||
? "ok"
|
||||
: (PROBE_FAILURE_PRIORITY.find((candidate) =>
|
||||
results.some((result) => result.status === candidate),
|
||||
) ?? "unknown");
|
||||
const error = results.find((result) => result.status === status)?.error;
|
||||
return {
|
||||
provider: cardId,
|
||||
status,
|
||||
...(error ? { error } : {}),
|
||||
results: results.flatMap((result) =>
|
||||
result.results.map((target) => ({
|
||||
...target,
|
||||
label: `${result.provider}: ${target.label}`,
|
||||
})),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export class ModelProvidersPage extends OpenClawLightDomElement {
|
||||
@consume({ context: applicationContext, subscribe: true })
|
||||
private context!: ApplicationContext;
|
||||
@@ -142,11 +111,9 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
|
||||
signal,
|
||||
}).then((data) => ({ client, data }))
|
||||
: initialState,
|
||||
onComplete: ({ client, data }) => {
|
||||
this.data = data;
|
||||
this.dataClient = client;
|
||||
},
|
||||
onComplete: ({ client, data }) => this.applyLoadedData(data, client),
|
||||
});
|
||||
private readonly usageRetry = createUsageRetry(this, () => void this.refresh({ force: false }));
|
||||
private readonly subscriptions = new SubscriptionsController(this)
|
||||
.watch(
|
||||
() => this.context?.gateway,
|
||||
@@ -186,20 +153,34 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
|
||||
if (changed.has("routeData") && this.routeData) {
|
||||
const selectedAgentId = this.resolveSelectedAgentId();
|
||||
this.setSelectedAgent(selectedAgentId);
|
||||
if ((this.routeData.agentId ?? "") === selectedAgentId) {
|
||||
this.data = this.routeData.data;
|
||||
this.dataClient = this.routeData.client;
|
||||
} else {
|
||||
this.data = null;
|
||||
this.dataClient = null;
|
||||
}
|
||||
const matches = (this.routeData.agentId ?? "") === selectedAgentId;
|
||||
this.applyLoadedData(
|
||||
matches ? this.routeData.data : null,
|
||||
matches ? this.routeData.client : null,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private applyLoadedData(data: ModelProvidersData | null, client: GatewayBrowserClient | null) {
|
||||
this.data = data;
|
||||
this.dataClient = client;
|
||||
// The connection epoch scopes the retry budget: a reconnect is a fresh
|
||||
// Gateway cache generation and must not inherit the old attempt count.
|
||||
// The "exhausted" state is deliberately dropped here: these cards render from
|
||||
// models.authStatus and stay useful without usage, and this page's
|
||||
// failed-usage notice is owned separately. Usage owns the visible outcome.
|
||||
this.usageRetry.observe(
|
||||
data !== null && isUsageIncomplete(data.providerUsage),
|
||||
this.connectionLifecycle.epoch,
|
||||
);
|
||||
}
|
||||
|
||||
override updated() {
|
||||
const snapshot = this.context.gateway.snapshot;
|
||||
if (this.connectionLifecycle.transition(snapshot)) {
|
||||
this.resetConnectionState(snapshot.client, snapshot.phase === "connected");
|
||||
const connected = snapshot.phase === "connected";
|
||||
const changed = this.connectionLifecycle.transition(snapshot);
|
||||
if (changed) {
|
||||
this.resetConnectionState(snapshot.client, connected);
|
||||
}
|
||||
if (
|
||||
!this.context.agents.state.agentsList &&
|
||||
@@ -208,6 +189,13 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
|
||||
) {
|
||||
void this.context.agents.ensureList();
|
||||
}
|
||||
if (changed && connected && snapshot.client) {
|
||||
// A reconnect reuses the client object, so the staleness check below
|
||||
// cannot recognize it as a new Gateway cache generation. Refetch here
|
||||
// rather than relying on the reset above having nulled `data`.
|
||||
void this.refresh({ force: false });
|
||||
return;
|
||||
}
|
||||
if (
|
||||
snapshot.phase !== "connected" ||
|
||||
!snapshot.client ||
|
||||
@@ -222,6 +210,9 @@ export class ModelProvidersPage extends OpenClawLightDomElement {
|
||||
}
|
||||
|
||||
private resetConnectionState(client: GatewayBrowserClient | null, connected: boolean) {
|
||||
this.usageRetry.useConnection(this.connectionLifecycle.epoch);
|
||||
// A null run advances Task's call id, so a late pre-transition result cannot
|
||||
// publish even when the underlying request ignores AbortSignal.
|
||||
void this.refreshTask.run([null, this.selectedAgentId, false]);
|
||||
this.busy = {};
|
||||
this.messages = {};
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import type { ModelsProbeResult } from "../../api/types.ts";
|
||||
|
||||
const PROBE_FAILURE_PRIORITY: readonly ModelsProbeResult["status"][] = [
|
||||
"auth",
|
||||
"billing",
|
||||
"rate_limit",
|
||||
"timeout",
|
||||
"format",
|
||||
"no_model",
|
||||
"unknown",
|
||||
];
|
||||
|
||||
export function mergeProbeResults(cardId: string, results: ModelsProbeResult[]): ModelsProbeResult {
|
||||
if (results.length === 1) {
|
||||
return results[0]!;
|
||||
}
|
||||
const status = results.some((result) => result.status === "ok")
|
||||
? "ok"
|
||||
: (PROBE_FAILURE_PRIORITY.find((candidate) =>
|
||||
results.some((result) => result.status === candidate),
|
||||
) ?? "unknown");
|
||||
const error = results.find((result) => result.status === status)?.error;
|
||||
return {
|
||||
provider: cardId,
|
||||
status,
|
||||
...(error ? { error } : {}),
|
||||
results: results.flatMap((result) =>
|
||||
result.results.map((target) => ({
|
||||
...target,
|
||||
label: `${result.provider}: ${target.label}`,
|
||||
})),
|
||||
),
|
||||
};
|
||||
}
|
||||
@@ -71,4 +71,23 @@ describe("UsageRefreshPolicy", () => {
|
||||
policy.request("focus");
|
||||
expect(reload).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("restarts an exhausted retry budget for manual and focus cycles", () => {
|
||||
const { policy, reload } = createPolicy();
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
policy.markLoaded({ incomplete: true });
|
||||
vi.advanceTimersByTime(5_000);
|
||||
}
|
||||
expect(reload).toHaveBeenCalledTimes(3);
|
||||
|
||||
policy.request("manual");
|
||||
policy.markLoaded({ incomplete: true });
|
||||
vi.advanceTimersByTime(5_000);
|
||||
expect(reload).toHaveBeenCalledTimes(5);
|
||||
|
||||
policy.request("focus");
|
||||
policy.markLoaded({ incomplete: true });
|
||||
vi.advanceTimersByTime(5_000);
|
||||
expect(reload).toHaveBeenCalledTimes(7);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { IncompleteUsageRetry, type UsageRetryState } from "../../lib/incomplete-usage-retry.ts";
|
||||
|
||||
const USAGE_PAYLOAD_TTL_MS = 5 * 60_000;
|
||||
|
||||
type UsageRefreshReason = "focus" | "manual" | "poll" | "reconnect";
|
||||
@@ -39,22 +41,43 @@ export class UsageRefreshPolicy {
|
||||
private lastLoadedAtMs: number | null = null;
|
||||
private pendingAutomaticRefresh = false;
|
||||
private reloadPending = false;
|
||||
private readonly incompleteUsageRetry = new IncompleteUsageRetry({
|
||||
retry: () => this.request("poll"),
|
||||
});
|
||||
|
||||
constructor(private readonly options: UsageRefreshPolicyOptions) {}
|
||||
|
||||
setLastLoadedAtMs(value: number | null): void {
|
||||
this.lastLoadedAtMs = value;
|
||||
setLastLoadedAtMs(
|
||||
value: number | null,
|
||||
params?: { incomplete?: boolean; connection?: unknown },
|
||||
): UsageRetryState {
|
||||
return this.applyLoadState(value, params?.incomplete === true, params?.connection);
|
||||
}
|
||||
|
||||
markLoaded(): void {
|
||||
this.lastLoadedAtMs = Date.now();
|
||||
markLoaded(params?: { incomplete?: boolean; connection?: unknown }): UsageRetryState {
|
||||
return this.applyLoadState(Date.now(), params?.incomplete === true, params?.connection);
|
||||
}
|
||||
|
||||
resetPayload(): void {
|
||||
this.lastLoadedAtMs = null;
|
||||
this.applyLoadState(null, false);
|
||||
this.reloadPending = false;
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.incompleteUsageRetry.dispose();
|
||||
}
|
||||
|
||||
private applyLoadState(
|
||||
loadedAtMs: number | null,
|
||||
incomplete: boolean,
|
||||
connection?: unknown,
|
||||
): UsageRetryState {
|
||||
const state = this.incompleteUsageRetry.observe(incomplete, connection);
|
||||
// Incomplete provider usage must not start the TTL or focus/reconnect can skip recovery.
|
||||
this.lastLoadedAtMs = state === "complete" ? loadedAtMs : null;
|
||||
return state;
|
||||
}
|
||||
|
||||
interrupt(): void {
|
||||
this.reloadPending ||= this.options.isLoading();
|
||||
}
|
||||
@@ -86,6 +109,9 @@ export class UsageRefreshPolicy {
|
||||
lastLoadedAtMs: this.lastLoadedAtMs,
|
||||
});
|
||||
if (decision === "fetch") {
|
||||
if (reason !== "poll") {
|
||||
this.incompleteUsageRetry.startCycle();
|
||||
}
|
||||
this.reload();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { CostUsageSummary } from "../../api/types.ts";
|
||||
import type { PanelRefreshStatus } from "../../components/panel-refresh-status.ts";
|
||||
// Control UI view renders usageTypes screen content.
|
||||
import type {
|
||||
@@ -14,6 +15,13 @@ export type UsageTotals = SessionsUsageTotals;
|
||||
export type CostDailyEntry = CostUsageDailyEntry;
|
||||
export type UsageAggregates = SessionsUsageResult["aggregates"];
|
||||
|
||||
export type UsageTaskValue = {
|
||||
epoch: object;
|
||||
result: SessionsUsageResult;
|
||||
costSummary: CostUsageSummary;
|
||||
providerUsageSummary: ProviderUsageSummary | null;
|
||||
};
|
||||
|
||||
export type UsageColumnId =
|
||||
| "channel"
|
||||
| "agent"
|
||||
@@ -48,6 +56,8 @@ type UsageDataState = {
|
||||
costDaily: CostDailyEntry[];
|
||||
cacheStatus: SessionsUsageResult["cacheStatus"];
|
||||
providerUsage: ProviderUsageSummary["providers"];
|
||||
/** The gateway never converged the refresh; the empty list is not an answer. */
|
||||
providerUsageStalled: boolean;
|
||||
};
|
||||
|
||||
export type UsageFilterState = {
|
||||
|
||||
@@ -15,6 +15,8 @@ type TestUsagePage = HTMLElement & {
|
||||
usageTimeSeriesStatus: { error: string | null; hasLoaded: boolean; stale: boolean };
|
||||
usageSessionLogs: SessionLogEntry[] | null;
|
||||
usageSessionLogsStatus: { error: string | null; hasLoaded: boolean; stale: boolean };
|
||||
providerUsageStalled: boolean;
|
||||
routeData?: unknown;
|
||||
loadSessionTimeSeries: (sessionKey: string) => Promise<void>;
|
||||
loadSessionLogs: (sessionKey: string) => Promise<void>;
|
||||
render: () => unknown;
|
||||
@@ -78,6 +80,44 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("UsagePage detail requests", () => {
|
||||
it("marks provider usage stalled once the retry budget is spent", async () => {
|
||||
const client = { request: vi.fn(async () => ({})) } as unknown as GatewayBrowserClient;
|
||||
const page = await createPage(client);
|
||||
const gateway = page.context.gateway;
|
||||
const routeDataAt = (loadedAtMs: number) => ({
|
||||
gateway,
|
||||
gatewaySnapshot: gateway.snapshot,
|
||||
client,
|
||||
query: {
|
||||
startDate: "2026-05-14",
|
||||
endDate: "2026-05-14",
|
||||
scope: "family" as const,
|
||||
timeZone: "local" as const,
|
||||
agentId: null,
|
||||
},
|
||||
result: null,
|
||||
costSummary: null,
|
||||
providerUsageSummary: { updatedAt: 1, providers: [], refreshing: true },
|
||||
loadedAtMs,
|
||||
error: null,
|
||||
});
|
||||
|
||||
// Three bounded attempts, then the page owns the outcome. Without the wiring
|
||||
// the empty provider list renders as a loaded answer.
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
page.routeData = routeDataAt(attempt);
|
||||
await page.updateComplete;
|
||||
expect(page.providerUsageStalled).toBe(false);
|
||||
}
|
||||
page.routeData = routeDataAt(3);
|
||||
await page.updateComplete;
|
||||
expect(page.providerUsageStalled).toBe(true);
|
||||
|
||||
page.routeData = { ...routeDataAt(4), providerUsageSummary: { updatedAt: 2, providers: [] } };
|
||||
await page.updateComplete;
|
||||
expect(page.providerUsageStalled).toBe(false);
|
||||
});
|
||||
|
||||
it("commits only the latest time-series selection", async () => {
|
||||
const first = deferred<SessionUsageTimeSeries>();
|
||||
const second = deferred<SessionUsageTimeSeries>();
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
formatMissingOperatorReadScopeMessage,
|
||||
isMissingOperatorReadScopeError,
|
||||
} from "../../lib/gateway-errors.ts";
|
||||
import { isUsageIncomplete } from "../../lib/incomplete-usage-retry.ts";
|
||||
import {
|
||||
requestSessionUsageLogs,
|
||||
requestSessionUsageTimeSeries,
|
||||
@@ -52,6 +53,7 @@ import {
|
||||
type SessionLogEntry,
|
||||
type SessionLogRole,
|
||||
type UsageProps,
|
||||
type UsageTaskValue,
|
||||
} from "./types.ts";
|
||||
import { renderUsage } from "./view.ts";
|
||||
|
||||
@@ -87,6 +89,7 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
@state() private usageResult: SessionsUsageResult | null = null;
|
||||
@state() private usageCostSummary: CostUsageSummary | null = null;
|
||||
@state() private providerUsageSummary: ProviderUsageSummary | null = null;
|
||||
@state() private providerUsageStalled = false;
|
||||
@state() private usageError: string | null = null;
|
||||
@state() private usageStartDate = currentLocalDate();
|
||||
@state() private usageEndDate = currentLocalDate();
|
||||
@@ -129,6 +132,8 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
// Invalidation runs the Task with a null client to supersede stale completions.
|
||||
// Track real gateway work separately so that no-op runs cannot block reconnect retries.
|
||||
private usageTaskActiveClient: GatewayBrowserClient | null = null;
|
||||
// The client survives transport reconnects, so retry budgets need a separate epoch.
|
||||
private connectionEpoch: object = {};
|
||||
private routeDataInitialized = false;
|
||||
private routeDataEnabled = true;
|
||||
private readonly refreshPolicy = new UsageRefreshPolicy({
|
||||
@@ -182,8 +187,16 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
return initialState;
|
||||
}
|
||||
this.refreshPolicy.beginLoad();
|
||||
const epoch = this.connectionEpoch;
|
||||
const agentId = normalizedAgentId || undefined;
|
||||
return requestUsageSnapshot(client, { startDate, endDate, agentId, scope, timeZone }, signal);
|
||||
return {
|
||||
epoch,
|
||||
...(await requestUsageSnapshot(
|
||||
client,
|
||||
{ startDate, endDate, agentId, scope, timeZone },
|
||||
signal,
|
||||
)),
|
||||
} satisfies UsageTaskValue;
|
||||
},
|
||||
onComplete: (value) => {
|
||||
this.usageTaskActiveClient = null;
|
||||
@@ -191,7 +204,11 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
this.usageCostSummary = value.costSummary;
|
||||
this.providerUsageSummary = value.providerUsageSummary;
|
||||
this.usageError = null;
|
||||
this.refreshPolicy.markLoaded();
|
||||
this.providerUsageStalled =
|
||||
this.refreshPolicy.markLoaded({
|
||||
incomplete: isUsageIncomplete(value.providerUsageSummary),
|
||||
connection: value.epoch,
|
||||
}) === "exhausted";
|
||||
this.refreshPolicy.flushPending();
|
||||
},
|
||||
onError: (error) => {
|
||||
@@ -274,6 +291,7 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
this.subscriptions.clear();
|
||||
this.clearDateDebounce();
|
||||
this.clearQueryDebounce();
|
||||
this.refreshPolicy.dispose();
|
||||
this.usageTaskActiveClient = null;
|
||||
void this.usageTask.run(this.usageTaskArgs(null));
|
||||
void this.usageTimeSeriesTask.run([null, ""]);
|
||||
@@ -312,7 +330,11 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
this.usageResult = data.result;
|
||||
this.usageCostSummary = data.costSummary;
|
||||
this.providerUsageSummary = data.providerUsageSummary;
|
||||
this.refreshPolicy.setLastLoadedAtMs(data.loadedAtMs);
|
||||
this.providerUsageStalled =
|
||||
this.refreshPolicy.setLastLoadedAtMs(data.loadedAtMs, {
|
||||
incomplete: isUsageIncomplete(data.providerUsageSummary),
|
||||
connection: this.connectionEpoch,
|
||||
}) === "exhausted";
|
||||
this.usageError = data.error;
|
||||
}
|
||||
|
||||
@@ -339,6 +361,7 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
this.usageResult = null;
|
||||
this.usageCostSummary = null;
|
||||
this.providerUsageSummary = null;
|
||||
this.providerUsageStalled = false;
|
||||
this.refreshPolicy.resetPayload();
|
||||
this.usageError = null;
|
||||
this.usageAgentId = this.context.agentSelection.state.scopeId;
|
||||
@@ -448,8 +471,11 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
return;
|
||||
}
|
||||
void this.context.agents.ensureList();
|
||||
if (this.routeDataInitialized && (change.identityChanged || change.becameConnected)) {
|
||||
this.refreshPolicy.request("reconnect");
|
||||
if (change.identityChanged || change.becameConnected) {
|
||||
this.connectionEpoch = {};
|
||||
if (this.routeDataInitialized) {
|
||||
this.refreshPolicy.request("reconnect");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -502,6 +528,7 @@ class UsagePage extends OpenClawLightDomElement {
|
||||
this.usageCostSummary?.cacheStatus,
|
||||
),
|
||||
providerUsage: this.providerUsageSummary?.providers ?? [],
|
||||
providerUsageStalled: this.providerUsageStalled,
|
||||
},
|
||||
filters: {
|
||||
startDate: this.usageStartDate,
|
||||
|
||||
@@ -63,6 +63,7 @@ function createUsageProps(overrides: Partial<UsageProps> = {}): UsageProps {
|
||||
costDaily: [],
|
||||
cacheStatus: undefined,
|
||||
providerUsage: [],
|
||||
providerUsageStalled: false,
|
||||
},
|
||||
filters: {
|
||||
startDate: "2026-05-14",
|
||||
@@ -367,6 +368,28 @@ describe("renderUsage", () => {
|
||||
expect(onQueryDraftChange).toHaveBeenCalledWith(expect.stringContaining("provider:clear"));
|
||||
});
|
||||
|
||||
it("reports a stalled provider refresh instead of hiding the section", () => {
|
||||
const container = document.createElement("div");
|
||||
|
||||
render(
|
||||
renderUsage(
|
||||
createUsageProps({
|
||||
data: {
|
||||
...createUsageProps().data,
|
||||
providerUsage: [],
|
||||
providerUsageStalled: true,
|
||||
},
|
||||
}),
|
||||
),
|
||||
container,
|
||||
);
|
||||
|
||||
const callout = container.querySelector(".usage-callout");
|
||||
expect(callout?.textContent?.trim()).toBe(
|
||||
"Provider usage did not finish loading. Refresh to retry.",
|
||||
);
|
||||
});
|
||||
|
||||
it("renders provider plans, quotas, and billing independently of session usage", () => {
|
||||
const container = document.createElement("div");
|
||||
|
||||
|
||||
@@ -143,7 +143,14 @@ function renderUsageEmptyState(onRefresh: () => void) {
|
||||
|
||||
type ProviderUsageSnapshot = ProviderUsageSummary["providers"][number];
|
||||
|
||||
function renderProviderUsage(providers: ProviderUsageSnapshot[]) {
|
||||
function renderProviderUsage(providers: ProviderUsageSnapshot[], stalled: boolean) {
|
||||
if (stalled) {
|
||||
// The stalled notice replaces the section: an empty grid under the normal
|
||||
// header reads as "this operator has no providers", which is the lie.
|
||||
return html`
|
||||
<div class="callout warning usage-callout">${t("usage.providerUsage.stalled")}</div>
|
||||
`;
|
||||
}
|
||||
if (providers.length === 0) {
|
||||
return nothing;
|
||||
}
|
||||
@@ -798,7 +805,7 @@ export function renderUsage(props: UsageProps) {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
${renderProviderUsage(data.providerUsage)}
|
||||
${renderProviderUsage(data.providerUsage, data.providerUsageStalled)}
|
||||
${isEmpty
|
||||
? renderUsageEmptyState(filterActions.onRefresh)
|
||||
: html`
|
||||
|
||||
Reference in New Issue
Block a user