diff --git a/docs/cli/models.md b/docs/cli/models.md index d65ddb5999dc..580eb59c4031 100644 --- a/docs/cli/models.md +++ b/docs/cli/models.md @@ -35,7 +35,7 @@ openclaw models scan Bare `openclaw models` is equivalent to `openclaw models status`. `openclaw models --json` returns the same object as `openclaw models status --json`. -`openclaw models status` shows the resolved default/fallbacks plus an auth overview. For plugin-owned agent runtimes such as Codex, it also checks whether the owning plugin is enabled and passed startup payload verification. A route with valid credentials but an unavailable runtime reports `status: unavailable` instead of `usable`; JSON output includes separate `authStatus`, `runtimeStatus`, and bounded runtime diagnostics. When provider usage snapshots are available, the OAuth/API-key status section includes provider usage windows and quota snapshots. Current usage-window providers: Anthropic, GitHub Copilot, OpenAI, MiniMax, Xiaomi, and z.ai. Usage auth comes from provider-specific hooks when available; otherwise OpenClaw falls back to matching OAuth/API-key credentials from auth profiles, env, or config. +`openclaw models status` shows the resolved default/fallbacks plus an auth overview. Active profile cooldowns appear under **Unavailable auth profiles** with the stored reason and recovery action; JSON output exposes the same data in `auth.unusableProfiles`. For plugin-owned agent runtimes such as Codex, status also checks whether the owning plugin is enabled and passed startup payload verification. A route with valid credentials but an unavailable runtime reports `status: unavailable` instead of `usable`; JSON output includes separate `authStatus`, `runtimeStatus`, and bounded runtime diagnostics. When provider usage snapshots are available, the OAuth/API-key status section includes provider usage windows and quota snapshots. Current usage-window providers: Anthropic, GitHub Copilot, OpenAI, MiniMax, Xiaomi, and z.ai. Usage auth comes from provider-specific hooks when available; otherwise OpenClaw falls back to matching OAuth/API-key credentials from auth profiles, env, or config. In `--json` output, `auth.providers` is the env/config/store-aware provider overview, while `auth.oauth` is auth-store profile health only. @@ -164,7 +164,7 @@ openclaw models auth order clear --provider `models auth add` is the interactive auth helper. It can launch a provider auth flow (OAuth/API key) or guide you into manual token paste, depending on the provider you choose. -`models auth list` lists saved auth profiles for the selected agent without printing token, API-key, or OAuth secret material. Use `--provider ` to filter to one provider, such as `openai`, and `--json` for scripting. +`models auth list` lists saved auth profiles for the selected agent without printing token, API-key, or OAuth secret material. Active cooldown and disable entries include their reason and recovery action. Use `--provider ` to filter to one provider, such as `openai`, and `--json` for scripting. `models auth login` runs a provider plugin's auth flow (OAuth/API key). Use `openclaw plugins list` to see which providers are installed. `login` accepts `--profile-id ` for providers that support named profiles during login (use this to keep multiple logins for the same provider separate), `--method ` to pick a specific auth method, `--device-code` as a shortcut for `--method device-code`, `--set-default` to apply the provider's recommended default model, and `--force` to remove existing profiles for that provider first (use when a cached OAuth profile is stuck or you want to switch accounts). diff --git a/docs/concepts/model-failover.md b/docs/concepts/model-failover.md index 75cbcba242de..4079cf3ce76e 100644 --- a/docs/concepts/model-failover.md +++ b/docs/concepts/model-failover.md @@ -175,6 +175,8 @@ Use a user-pinned profile only when you want to force one account/key for that s When a profile fails due to auth/rate-limit errors (or a timeout that looks like rate limiting), OpenClaw marks it in cooldown and moves to the next profile. +CLI-backed runtimes settle profile health only after their resume, fork, and fresh-session recovery attempts finish. A terminal credential failure cools down the exact selected profile before model fallback; a successful run clears stale failure state. Transcript, format, context, pre-provider timeout, and ambient CLI failures without a selected profile do not change shared profile health. + That rate-limit bucket is broader than plain `429`: it also includes provider messages such as `Too many concurrent requests`, `ThrottlingException`, `concurrency limit reached`, `workers_ai ... quota limit exceeded`, `throttled`, `resource exhausted`, and periodic usage-window limits such as `weekly limit reached` or `monthly limit exhausted`. diff --git a/src/agents/auth-profiles/oauth-refresh-failure.test.ts b/src/agents/auth-profiles/oauth-refresh-failure.test.ts index cba6a936450f..5166e2abd66d 100644 --- a/src/agents/auth-profiles/oauth-refresh-failure.test.ts +++ b/src/agents/auth-profiles/oauth-refresh-failure.test.ts @@ -8,6 +8,7 @@ import type { AddressInfo } from "node:net"; import { describe, expect, it } from "vitest"; import { FailoverError } from "../failover-error.js"; import { + buildAuthProfileUnusableHint, buildOAuthRefreshFailureLoginCommand, classifyOAuthRefreshFailure, classifyOAuthRefreshFailureError, @@ -15,6 +16,37 @@ import { OAuthRefreshFailureError, } from "./oauth-refresh-failure.js"; +describe("buildAuthProfileUnusableHint", () => { + it("keeps Claude subscription and Anthropic API-key recovery distinct", () => { + expect( + buildAuthProfileUnusableHint({ + kind: "cooldown", + reason: "session_expired", + provider: "claude-cli", + profileId: "anthropic:claude-cli", + }), + ).toContain( + "claude auth login && openclaw models auth login --provider anthropic --method cli --profile-id 'anthropic:claude-cli'", + ); + expect( + buildAuthProfileUnusableHint({ + kind: "cooldown", + reason: "auth", + provider: "anthropic", + profileId: "anthropic:api-key", + }), + ).toContain("openclaw models auth login --provider anthropic --profile-id 'anthropic:api-key'"); + expect( + buildAuthProfileUnusableHint({ + kind: "cooldown", + reason: "auth", + provider: "anthropic", + profileId: "anthropic:api-key", + }), + ).not.toContain("claude auth login"); + }); +}); + describe("oauth refresh failure hints", () => { it("builds OpenAI refresh-failure login hints", () => { expect( diff --git a/src/agents/auth-profiles/oauth-refresh-failure.ts b/src/agents/auth-profiles/oauth-refresh-failure.ts index 93a138409296..07fbfc592e33 100644 --- a/src/agents/auth-profiles/oauth-refresh-failure.ts +++ b/src/agents/auth-profiles/oauth-refresh-failure.ts @@ -7,6 +7,7 @@ import { formatCliCommand } from "../../cli/command-format.js"; * commands without trusting raw provider text. */ import { formatInlineCodeSpan } from "../../shared/markdown-code.js"; +import type { AuthProfileFailureReason } from "./types.js"; export type OAuthRefreshFailureReason = | "refresh_token_reused" @@ -238,3 +239,26 @@ export function buildOAuthRefreshFailureLoginCommand( ) : formatCliCommand("openclaw models auth login"); } + +/** Build operator guidance for an active profile cooldown or disable window. */ +export function buildAuthProfileUnusableHint(params: { + kind: "cooldown" | "disabled"; + reason?: AuthProfileFailureReason; + provider: string; + profileId: string; +}): string { + if ( + params.reason === "auth" || + params.reason === "auth_permanent" || + params.reason === "session_expired" + ) { + const command = buildOAuthRefreshFailureLoginCommand(params.provider, { + profileId: params.profileId, + }); + return `Re-authenticate with ${formatOAuthRefreshFailureLoginCommandMarkdown(command)}.`; + } + if (params.kind === "disabled" && params.reason === "billing") { + return "Top up credits (provider billing) or switch provider."; + } + return "Wait for cooldown or switch provider."; +} diff --git a/src/agents/cli-runner.before-agent-reply-cron.test.ts b/src/agents/cli-runner.before-agent-reply-cron.test.ts index b8c2a5b7cc7d..c5e5e1b385fc 100644 --- a/src/agents/cli-runner.before-agent-reply-cron.test.ts +++ b/src/agents/cli-runner.before-agent-reply-cron.test.ts @@ -14,7 +14,9 @@ import { } from "../infra/diagnostic-events.js"; import { testing as cliBackendsTesting } from "./cli-backends.test-support.js"; import type { CliOutput } from "./cli-output.js"; +import { CliAuthProfilePreparationError } from "./cli-runner/auth-profile-preparation-error.js"; import { cliBackendLog } from "./cli-runner/log.js"; +import { FailoverError } from "./failover-error.js"; // vi.mock factories are hoisted above imports, so any references inside them // must come from vi.hoisted() so they exist at hoist time (otherwise they'd @@ -38,6 +40,9 @@ const { closeMcpLoopbackServerMock, retireSessionMcpRuntimeForSessionKeyMock, retireSessionMcpRuntimeMock, + loadAuthProfileStoreForRuntimeMock, + markAuthProfileFailureMock, + markAuthProfileSuccessMock, } = vi.hoisted(() => ({ hasHooksMock: vi.fn<(hookName: string) => boolean>(() => false), runBeforeAgentReplyMock: vi.fn<(event: unknown, ctx: unknown) => Promise>( @@ -51,6 +56,9 @@ const { closeMcpLoopbackServerMock: vi.fn(), retireSessionMcpRuntimeForSessionKeyMock: vi.fn(), retireSessionMcpRuntimeMock: vi.fn(), + loadAuthProfileStoreForRuntimeMock: vi.fn(), + markAuthProfileFailureMock: vi.fn(), + markAuthProfileSuccessMock: vi.fn(), })); vi.mock("../plugins/hook-runner-global.js", () => ({ @@ -98,6 +106,8 @@ const baseRunParams = { } as const; let runCliAgent: typeof import("./cli-runner.js").runCliAgent; +let restoreCliRunnerTestDeps: typeof import("./cli-runner.js").restoreCliRunnerTestDeps; +let setCliRunnerTestDeps: typeof import("./cli-runner.js").setCliRunnerTestDeps; async function captureRejectedClaudeRun( params: Parameters[0], @@ -157,19 +167,184 @@ beforeEach(() => { retireSessionMcpRuntimeForSessionKeyMock.mockResolvedValue(true); retireSessionMcpRuntimeMock.mockReset(); retireSessionMcpRuntimeMock.mockResolvedValue(true); + loadAuthProfileStoreForRuntimeMock.mockReset(); + markAuthProfileFailureMock.mockReset().mockResolvedValue(undefined); + markAuthProfileSuccessMock.mockReset().mockResolvedValue(undefined); + setCliRunnerTestDeps?.({ + loadAuthProfileStoreForRuntime: loadAuthProfileStoreForRuntimeMock, + markAuthProfileFailure: markAuthProfileFailureMock, + markAuthProfileSuccess: markAuthProfileSuccessMock, + }); }); beforeAll(async () => { - ({ runCliAgent } = await import("./cli-runner.js")); + ({ runCliAgent, restoreCliRunnerTestDeps, setCliRunnerTestDeps } = + await import("./cli-runner.js")); }); afterEach(() => { + restoreCliRunnerTestDeps(); cliBackendsTesting.resetDepsForTest(); vi.clearAllMocks(); resetDiagnosticEventsForTest(); }); describe("runCliAgent before_agent_reply seam", () => { + it.each([ + ["claude-cli", "user"], + ["google-gemini-cli", "cron"], + ])("settles one exhausted %s profile failure for a %s caller", async (provider, trigger) => { + const profileId = `${provider}:selected`; + const store = { + version: 1, + profiles: { [profileId]: { type: "api_key", provider, key: "secret" } }, + } as const; + prepareCliRunContextMock.mockImplementationOnce(async (params) => ({ + ...(makeStubContext(params as typeof baseRunParams & { trigger?: string }) as object), + effectiveAuthProfileId: profileId, + authProfileStore: store, + agentDir: "/tmp/agent", + })); + executePreparedCliRunMock.mockRejectedValueOnce( + new FailoverError("selected session expired", { reason: "session_expired", provider }), + ); + + await expect( + runCliAgent({ ...baseRunParams, provider, trigger: trigger as "user" | "cron" }), + ).rejects.toMatchObject({ reason: "session_expired" }); + + expect(markAuthProfileFailureMock).toHaveBeenCalledOnce(); + expect(markAuthProfileFailureMock).toHaveBeenCalledWith( + expect.objectContaining({ + store, + profileId, + reason: "session_expired", + agentDir: "/tmp/agent", + }), + ); + expect(markAuthProfileSuccessMock).not.toHaveBeenCalled(); + }); + + it("settles a typed selected-profile preparation failure before fallback", async () => { + const profileId = "claude-cli:selected"; + const store = { + version: 1, + profiles: { [profileId]: { type: "oauth", provider: "claude-cli" } }, + }; + loadAuthProfileStoreForRuntimeMock.mockReturnValue(store); + prepareCliRunContextMock.mockRejectedValueOnce( + new CliAuthProfilePreparationError({ + message: "selected profile needs login", + profileId, + provider: "claude-cli", + agentDir: "/tmp/agent", + }), + ); + + await expect(runCliAgent({ ...baseRunParams, provider: "claude-cli" })).rejects.toMatchObject({ + name: "CliAuthProfilePreparationError", + reason: "auth", + profileId, + }); + + expect(loadAuthProfileStoreForRuntimeMock).toHaveBeenCalledWith( + "/tmp/agent", + expect.any(Object), + ); + expect(markAuthProfileFailureMock).toHaveBeenCalledWith( + expect.objectContaining({ store, profileId, reason: "auth" }), + ); + }); + + it("records only success when fresh-session recovery succeeds and clears stale health", async () => { + const profileId = "google-gemini-cli:selected"; + const store = { + version: 1, + profiles: { + [profileId]: { type: "oauth", provider: "google-gemini-cli", access: "secret" }, + }, + usageStats: { + [profileId]: { cooldownUntil: Date.now() + 60_000, cooldownReason: "session_expired" }, + }, + }; + prepareCliRunContextMock.mockImplementationOnce(async (params) => ({ + ...(makeStubContext(params as typeof baseRunParams & { trigger?: string }) as object), + effectiveAuthProfileId: profileId, + authProfileStore: store, + agentDir: "/tmp/agent", + openClawHistoryPrompt: "history", + reusableCliSession: { mode: "reuse", sessionId: "stale-session" }, + params: { + ...(params as typeof baseRunParams), + onBeforeFreshCliSessionRetry: vi.fn(async () => true), + }, + })); + executePreparedCliRunMock + .mockRejectedValueOnce( + new FailoverError("stale session", { + reason: "session_expired", + provider: "google-gemini-cli", + }), + ) + .mockResolvedValueOnce({ text: "recovered" }); + + await expect( + runCliAgent({ ...baseRunParams, provider: "google-gemini-cli" }), + ).resolves.toBeDefined(); + + expect(executePreparedCliRunMock).toHaveBeenCalledTimes(2); + expect(markAuthProfileFailureMock).not.toHaveBeenCalled(); + expect(markAuthProfileSuccessMock).toHaveBeenCalledOnce(); + expect(markAuthProfileSuccessMock).toHaveBeenCalledWith({ + store, + profileId, + provider: "google-gemini-cli", + agentDir: "/tmp/agent", + }); + }); + + it.each([ + new FailoverError("bad transcript", { reason: "format" }), + new FailoverError("context full", { reason: "context_overflow" }), + new FailoverError("pre-provider timeout", { + reason: "timeout", + cliTimeout: { + mode: "no-output", + timeoutSeconds: 30, + observedActivity: false, + activeToolCount: 0, + backgroundTaskCount: 0, + }, + }), + ])("does not settle selected-profile health for local failure %#", async (error) => { + const profileId = "claude-cli:selected"; + prepareCliRunContextMock.mockImplementationOnce(async (params) => ({ + ...(makeStubContext(params as typeof baseRunParams & { trigger?: string }) as object), + effectiveAuthProfileId: profileId, + authProfileStore: { + version: 1, + profiles: { [profileId]: { type: "api_key", provider: "claude-cli", key: "secret" } }, + }, + })); + executePreparedCliRunMock.mockRejectedValueOnce(error); + + await expect(runCliAgent({ ...baseRunParams, provider: "claude-cli" })).rejects.toBe(error); + expect(markAuthProfileFailureMock).not.toHaveBeenCalled(); + expect(markAuthProfileSuccessMock).not.toHaveBeenCalled(); + }); + + it("leaves ambient no-profile failures out of shared health", async () => { + executePreparedCliRunMock.mockRejectedValueOnce( + new FailoverError("ambient auth failed", { reason: "auth", provider: "claude-cli" }), + ); + + await expect(runCliAgent({ ...baseRunParams, provider: "claude-cli" })).rejects.toMatchObject({ + reason: "auth", + }); + expect(markAuthProfileFailureMock).not.toHaveBeenCalled(); + expect(markAuthProfileSuccessMock).not.toHaveBeenCalled(); + }); + it("adds Claude CLI harness and run ownership at the runner entrypoint", async () => { const events: DiagnosticEventPayload[] = []; const unsubscribe = onTrustedInternalDiagnosticEvent((event) => { diff --git a/src/agents/cli-runner.ts b/src/agents/cli-runner.ts index e2cfa1e42623..4cd732c7bb1f 100644 --- a/src/agents/cli-runner.ts +++ b/src/agents/cli-runner.ts @@ -27,6 +27,13 @@ import { import { resolveBlockMessage } from "../plugins/hook-decision-types.js"; import { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; import { resolveAgentIdFromSessionKey } from "../routing/session-key.js"; +import { + externalCliDiscoveryForProviderAuth, + loadAuthProfileStoreForRuntime, + markAuthProfileFailure, + markAuthProfileSuccess, + type AuthProfileStore, +} from "./auth-profiles.js"; import { isHeartbeatLifecycleRunKind } from "./bootstrap-mode.js"; import { resolveCliRuntimeArtifactFingerprint, @@ -34,6 +41,7 @@ import { } from "./cli-auth-epoch.js"; import { resolveCliBackendConfig } from "./cli-backends.js"; import type { CliOutput } from "./cli-output.js"; +import { CliAuthProfilePreparationError } from "./cli-runner/auth-profile-preparation-error.js"; import { shouldUseClaudeLiveSession } from "./cli-runner/claude-live-session.js"; import { attachCliMessagingDeliveryEvidence, @@ -58,6 +66,7 @@ import { claudeCliSessionTranscriptHasContent as claudeCliSessionTranscriptHasCo import { classifyFailoverReason, isFailoverErrorMessage } from "./embedded-agent-helpers.js"; import type { EmbeddedAgentRunResult } from "./embedded-agent-runner.js"; import { waitForDeferredTurnMaintenanceForSession } from "./embedded-agent-runner/context-engine-maintenance.js"; +import { resolveAuthProfileFailureReason } from "./embedded-agent-runner/run/auth-profile-failure-policy.js"; import { buildEmbeddedRunPayloads } from "./embedded-agent-runner/run/payloads.js"; import { FailoverError, isFailoverError, resolveFailoverStatus } from "./failover-error.js"; import { @@ -89,6 +98,9 @@ const cliRunnerDeps = { setTimeout(resolve, delayMs); }); }, + loadAuthProfileStoreForRuntime, + markAuthProfileFailure, + markAuthProfileSuccess, }; /** Overrides top-level CLI runner dependencies for tests. */ @@ -104,6 +116,60 @@ export function restoreCliRunnerTestDeps(): void { setTimeout(resolve, delayMs); }); }; + cliRunnerDeps.loadAuthProfileStoreForRuntime = loadAuthProfileStoreForRuntime; + cliRunnerDeps.markAuthProfileFailure = markAuthProfileFailure; + cliRunnerDeps.markAuthProfileSuccess = markAuthProfileSuccess; +} + +async function settleCliAuthProfile(params: { + store: AuthProfileStore; + profileId: string; + provider: string; + agentDir?: string; + terminal: + | { outcome: "success" } + | { + outcome: "failure"; + error: unknown; + config?: RunCliAgentParams["config"]; + runId: string; + modelId?: string; + }; +}): Promise { + try { + if (params.terminal.outcome === "success") { + await cliRunnerDeps.markAuthProfileSuccess({ + store: params.store, + profileId: params.profileId, + provider: params.provider, + agentDir: params.agentDir, + }); + return; + } + const error = params.terminal.error; + const reason = resolveAuthProfileFailureReason({ + failoverReason: isFailoverError(error) ? error.reason : null, + providerStarted: + isFailoverError(error) && error.reason === "timeout" + ? error.cliTimeout?.observedActivity + : undefined, + }); + if (reason) { + await cliRunnerDeps.markAuthProfileFailure({ + store: params.store, + profileId: params.profileId, + reason, + cfg: params.terminal.config, + agentDir: params.agentDir, + runId: params.terminal.runId, + modelId: params.terminal.modelId, + }); + } + } catch (error) { + log.warn( + `CLI auth-profile ${params.terminal.outcome} settlement failed: ${formatErrorMessage(error)}`, + ); + } } function isClaudeCliProvider(provider: string): boolean { @@ -563,7 +629,34 @@ async function runCliAgentInternal( }; } const { prepareCliRunContext } = await import("./cli-runner/prepare.runtime.js"); - const context = await prepareCliRunContext(params); + let context: PreparedCliRunContext; + try { + context = await prepareCliRunContext(params); + } catch (error) { + if (error instanceof CliAuthProfilePreparationError) { + const store = cliRunnerDeps.loadAuthProfileStoreForRuntime(error.agentDir, { + externalCli: externalCliDiscoveryForProviderAuth({ + cfg: params.config, + provider: error.provider, + profileId: error.profileId, + }), + }); + await settleCliAuthProfile({ + store, + profileId: error.profileId, + provider: error.provider, + agentDir: error.agentDir, + terminal: { + outcome: "failure", + error, + config: params.config, + runId: params.runId, + modelId: params.model, + }, + }); + } + throw error; + } let result: EmbeddedAgentRunResult | undefined; let runError: unknown; try { @@ -571,6 +664,7 @@ async function runCliAgentInternal( } catch (error) { runError = error; } + const terminalRunError = runError; let cleanupError: unknown; const recordCleanupError = (error: unknown) => { cleanupError ??= error; @@ -607,6 +701,36 @@ async function runCliAgentInternal( cleanupError instanceof Error ? cleanupError : new Error(formatErrorMessage(cleanupError)); } } + // Settle only after backend recovery is exhausted. Recording inside an + // attempt would quarantine a healthy profile for a recovered session fault. + if (context.effectiveAuthProfileId && context.authProfileStore) { + const profileId = context.effectiveAuthProfileId; + const authProfileStore = context.authProfileStore; + if (terminalRunError) { + await settleCliAuthProfile({ + store: authProfileStore, + profileId, + provider: authProfileStore.profiles[profileId]?.provider ?? params.provider, + agentDir: context.agentDir, + terminal: { + outcome: "failure", + error: terminalRunError, + config: params.config, + runId: params.runId, + modelId: context.modelId, + }, + }); + } else if (result && result.meta.stopReason !== "error") { + const provider = authProfileStore.profiles[profileId]?.provider ?? params.provider; + await settleCliAuthProfile({ + store: authProfileStore, + profileId, + provider, + agentDir: context.agentDir, + terminal: { outcome: "success" }, + }); + } + } if (runError) { throw runError instanceof Error ? runError : new Error(formatErrorMessage(runError)); } diff --git a/src/agents/cli-runner/auth-profile-preparation-error.ts b/src/agents/cli-runner/auth-profile-preparation-error.ts new file mode 100644 index 000000000000..8c984d64eb43 --- /dev/null +++ b/src/agents/cli-runner/auth-profile-preparation-error.ts @@ -0,0 +1,18 @@ +/** Typed terminal fact for a selected profile that fails before CLI spawn. */ +import { FailoverError } from "../failover-error.js"; + +export class CliAuthProfilePreparationError extends FailoverError { + declare readonly profileId: string; + declare readonly provider: string; + readonly agentDir: string; + + constructor(params: { message: string; profileId: string; provider: string; agentDir: string }) { + super(params.message, { + reason: "auth", + provider: params.provider, + profileId: params.profileId, + }); + this.name = "CliAuthProfilePreparationError"; + this.agentDir = params.agentDir; + } +} diff --git a/src/agents/cli-runner/prepare.test.ts b/src/agents/cli-runner/prepare.test.ts index f7f1c2c7d5fe..b3491a90cf22 100644 --- a/src/agents/cli-runner/prepare.test.ts +++ b/src/agents/cli-runner/prepare.test.ts @@ -984,6 +984,13 @@ describe("prepareCliRunContext", () => { await expect(preparation).rejects.toThrow( `could not materialize selected auth profile "${authProfileId}"`, ); + await expect(preparation).rejects.toMatchObject({ + name: "CliAuthProfilePreparationError", + reason: "auth", + profileId: authProfileId, + provider: "anthropic", + agentDir, + }); await expect(preparation).rejects.toThrow("openclaw models auth login --provider anthropic"); expect(prepareExecution).not.toHaveBeenCalled(); }); diff --git a/src/agents/cli-runner/prepare.ts b/src/agents/cli-runner/prepare.ts index 2e2fbf509b2d..614d6879dc95 100644 --- a/src/agents/cli-runner/prepare.ts +++ b/src/agents/cli-runner/prepare.ts @@ -114,6 +114,7 @@ import { DEFAULT_BOOTSTRAP_FILENAME, isWorkspaceBootstrapPending as isWorkspaceBootstrapPendingImpl, } from "../workspace.js"; +import { CliAuthProfilePreparationError } from "./auth-profile-preparation-error.js"; import { prepareCliBundleMcpConfig } from "./bundle-mcp.js"; import { getClaudeLiveSessionGenerationForOwner } from "./claude-live-session.js"; import { prepareClaudeCliSkillsPlugin } from "./claude-skills-plugin.js"; @@ -376,15 +377,19 @@ function buildCliAuthProfileResolutionError(params: { backendId: string; profileId: string; provider: string; + agentDir: string; failure: CliAuthProfileResolutionFailure; -}): Error { +}): CliAuthProfilePreparationError { const loginCommand = buildOAuthRefreshFailureLoginCommand(params.provider, { profileId: params.profileId, }); const reason = describeCliAuthProfileResolutionFailure(params.profileId, params.failure); - return new Error( - `CLI backend "${params.backendId}" ${reason}. Re-authenticate with: ${loginCommand}. OpenClaw did not start the run.`, - ); + return new CliAuthProfilePreparationError({ + message: `CLI backend "${params.backendId}" ${reason}. Re-authenticate with: ${loginCommand}. OpenClaw did not start the run.`, + profileId: params.profileId, + provider: params.provider, + agentDir: params.agentDir, + }); } /** Builds the complete context required to execute a CLI-backed agent run. */ @@ -590,6 +595,7 @@ export async function prepareCliRunContext( backendId: backendResolved.id, profileId: authProfileId, provider: nativeClaudeCliCredential.provider, + agentDir, failure: { kind: "native-login-missing" }, }); } @@ -598,6 +604,7 @@ export async function prepareCliRunContext( backendId: backendResolved.id, profileId: authProfileId, provider: nativeClaudeCliCredential.provider, + agentDir, failure: { kind: "native-login-identity-mismatch" }, }); } @@ -629,6 +636,7 @@ export async function prepareCliRunContext( backendId: backendResolved.id, profileId: authProfileId, provider: writableAuthStore.profiles[authProfileId]?.provider ?? params.provider, + agentDir, failure: { kind: "unmaterialized" }, }); } @@ -641,6 +649,7 @@ export async function prepareCliRunContext( backendId: backendResolved.id, profileId: authProfileId, provider: writableAuthStore.profiles[authProfileId]?.provider ?? params.provider, + agentDir, failure: { kind: "resolved-as-other", resolvedProfileId: resolvedAuth.profileId }, }); } @@ -656,6 +665,7 @@ export async function prepareCliRunContext( backendId: backendResolved.id, profileId: authProfileId, provider: resolvedAuth?.provider ?? params.provider, + agentDir, failure: { kind: "unmaterialized" }, }); } @@ -1597,6 +1607,7 @@ export async function prepareCliRunContext( return { params: preparedParams, effectiveAuthProfileId, + ...(authStore ? { authProfileStore: authStore } : {}), agentDir, started, workspaceDir, @@ -1669,6 +1680,7 @@ export async function prepareCliRunContext( return { params: preparedParams, effectiveAuthProfileId, + ...(authStore ? { authProfileStore: authStore } : {}), agentDir, started, workspaceDir, diff --git a/src/agents/cli-runner/types.ts b/src/agents/cli-runner/types.ts index d05d2a9271c9..fffcf3cc3d42 100644 --- a/src/agents/cli-runner/types.ts +++ b/src/agents/cli-runner/types.ts @@ -33,6 +33,7 @@ import type { SpawnSecretInput } from "../../process/supervisor/types.js"; import type { InputProvenance } from "../../sessions/input-provenance.js"; import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; import type { SkillSnapshot } from "../../skills/types.js"; +import type { AuthProfileStore } from "../auth-profiles/types.js"; import type { ExecElevatedDefaults } from "../bash-tools.exec-types.js"; import type { BootstrapContextMode } from "../bootstrap-files.js"; import type { BootstrapContextRunKind } from "../bootstrap-mode.js"; @@ -288,6 +289,8 @@ export type CliSessionBindingFacts = { export type PreparedCliRunContext = { params: RunCliAgentParams; effectiveAuthProfileId?: string; + /** Selected profile snapshot used only for terminal health settlement. */ + authProfileStore?: AuthProfileStore; agentDir?: string; started: number; workspaceDir: string; diff --git a/src/commands/doctor-auth.profile-health.test.ts b/src/commands/doctor-auth.profile-health.test.ts index e2d35dab2326..f3fcc374c1a0 100644 --- a/src/commands/doctor-auth.profile-health.test.ts +++ b/src/commands/doctor-auth.profile-health.test.ts @@ -22,7 +22,8 @@ const authProfileMocks = vi.hoisted(() => ({ resolveProfileUnusableUntilForDisplay: vi.fn(), })); -vi.mock("../agents/auth-profiles.js", () => ({ +vi.mock("../agents/auth-profiles.js", async (importOriginal) => ({ + ...(await importOriginal()), ensureAuthProfileStore: authProfileMocks.ensureAuthProfileStore, hasAnyAuthProfileStoreSource: authProfileMocks.hasAnyAuthProfileStoreSource, hasLocalAuthProfileStoreSource: authProfileMocks.hasLocalAuthProfileStoreSource, @@ -245,7 +246,10 @@ describe("noteAuthProfileHealth", () => { }); it.each([ - ["auth_permanent", "Refresh or replace credentials, then retry."], + [ + "auth_permanent", + "Re-authenticate with `openclaw models auth login --provider openai --profile-id 'openai:disabled'`.", + ], ["unknown", "Wait for cooldown or switch provider."], ] satisfies Array<[AuthProfileFailureReason, string]>)( "maps disabled %s profiles to their production health hint", @@ -257,7 +261,9 @@ describe("noteAuthProfileHealth", () => { authProfileMocks.resolveProfileUnusableUntilForDisplay.mockReturnValue(now + 5 * 60_000); authProfileMocks.ensureAuthProfileStore.mockReturnValue({ version: 1, - profiles: {}, + profiles: { + "openai:disabled": { type: "api_key", provider: "openai", key: "secret" }, + }, usageStats: { "openai:disabled": { disabledUntil: now + 5 * 60_000, @@ -276,6 +282,46 @@ describe("noteAuthProfileHealth", () => { }, ); + it("reports a session-expired Claude CLI profile with its exact re-login action", async () => { + const now = 1_700_000_000_000; + vi.spyOn(Date, "now").mockReturnValue(now); + const mainDir = path.join(tempDir, "main-agent"); + authProfileMocks.hasAnyAuthProfileStoreSource.mockReturnValue(true); + authProfileMocks.resolveProfileUnusableUntilForDisplay.mockReturnValue(now + 5 * 60_000); + authProfileMocks.ensureAuthProfileStore.mockReturnValue({ + version: 1, + profiles: { + "anthropic:claude-cli": { + type: "oauth", + provider: "claude-cli", + access: "secret", + refresh: "secret", + expires: now + 60_000, + }, + }, + usageStats: { + "anthropic:claude-cli": { + cooldownUntil: now + 5 * 60_000, + cooldownReason: "session_expired", + }, + }, + } satisfies AuthProfileStore); + + const findings = await collectAuthProfileHealthFindings({ + cfg: { + agents: { list: [{ id: "main", default: true, agentDir: mainDir }] }, + } as OpenClawConfig, + }); + + expect(findings).toEqual([ + expect.objectContaining({ + message: "Auth profile anthropic:claude-cli is cooldown:session_expired (5m).", + fixHint: + "Re-authenticate with `claude auth login && openclaw models auth login --provider anthropic --method cli --profile-id 'anthropic:claude-cli'`.", + }), + ]); + }); + it("maps cooldown profiles to cooldown guidance", async () => { const now = 1_700_000_000_000; vi.spyOn(Date, "now").mockReturnValue(now); diff --git a/src/commands/doctor-auth.ts b/src/commands/doctor-auth.ts index ee3d7caee9fd..544e48daf572 100644 --- a/src/commands/doctor-auth.ts +++ b/src/commands/doctor-auth.ts @@ -24,6 +24,7 @@ import { import { CLAUDE_CLI_PROFILE_ID } from "../agents/auth-profiles/constants.js"; import { formatAuthDoctorHint } from "../agents/auth-profiles/doctor.js"; import { + buildAuthProfileUnusableHint, buildOAuthRefreshFailureLoginCommand, classifyOAuthRefreshFailure, formatOAuthRefreshFailureLoginCommandMarkdown, @@ -201,22 +202,6 @@ function listAuthProfileHealthTargets(cfg: OpenClawConfig): AuthProfileHealthTar return [...targets.values()]; } -/** Returns the short doctor hint for disabled or cooldown auth profiles. */ -function resolveUnusableProfileHint(params: { - kind: "cooldown" | "disabled"; - reason?: string; -}): string { - if (params.kind === "disabled") { - if (params.reason === "billing") { - return "Top up credits (provider billing) or switch provider."; - } - if (params.reason === "auth_permanent" || params.reason === "auth") { - return "Refresh or replace credentials, then retry."; - } - } - return "Wait for cooldown or switch provider."; -} - function formatOAuthRefreshFailureReason(reason: OAuthRefreshFailureReason | null): string { switch (reason) { case "refresh_token_reused": @@ -386,12 +371,13 @@ async function collectAuthProfileHealthFindingsForTarget(params: { const stats = store.usageStats?.[profileId]; const remaining = formatRemainingShort(until - now); const disabledActive = typeof stats?.disabledUntil === "number" && now < stats.disabledUntil; - const kind = disabledActive - ? `disabled${stats.disabledReason ? `:${stats.disabledReason}` : ""}` - : "cooldown"; - const hint = resolveUnusableProfileHint({ + const reason = disabledActive ? stats?.disabledReason : stats?.cooldownReason; + const kind = `${disabledActive ? "disabled" : "cooldown"}${reason ? `:${reason}` : ""}`; + const hint = buildAuthProfileUnusableHint({ kind: disabledActive ? "disabled" : "cooldown", - reason: stats?.disabledReason, + reason, + provider: store.profiles[profileId]?.provider ?? profileId, + profileId, }); findings.push( authProfileCooldownToHealthFinding({ @@ -491,12 +477,13 @@ async function noteAuthProfileHealthForTarget(params: { const stats = store.usageStats?.[profileId]; const remaining = formatRemainingShort(until - now); const disabledActive = typeof stats?.disabledUntil === "number" && now < stats.disabledUntil; - const kind = disabledActive - ? `disabled${stats.disabledReason ? `:${stats.disabledReason}` : ""}` - : "cooldown"; - const hint = resolveUnusableProfileHint({ + const reason = disabledActive ? stats?.disabledReason : stats?.cooldownReason; + const kind = `${disabledActive ? "disabled" : "cooldown"}${reason ? `:${reason}` : ""}`; + const hint = buildAuthProfileUnusableHint({ kind: disabledActive ? "disabled" : "cooldown", - reason: stats?.disabledReason, + reason, + provider: store.profiles[profileId]?.provider ?? profileId, + profileId, }); out.push(`- ${profileId}: ${kind} (${remaining})${hint ? ` — ${hint}` : ""}`); } diff --git a/src/commands/models/auth-list.test.ts b/src/commands/models/auth-list.test.ts index ef2892899398..c19b254b00f9 100644 --- a/src/commands/models/auth-list.test.ts +++ b/src/commands/models/auth-list.test.ts @@ -111,6 +111,7 @@ describe("modelsAuthListCommand", () => { id: "openai:user@example.com", label: "openai:user@example.com", provider: "openai", + recoveryHint: "Wait for cooldown or switch provider.", type: "oauth", }, ], @@ -120,6 +121,47 @@ describe("modelsAuthListCommand", () => { expect(JSON.stringify(runtime.jsonPayloads[0])).not.toContain("secret"); }); + it("shows the cooldown reason and re-authentication action in text and JSON", async () => { + mocks.ensureAuthProfileStore.mockReturnValue({ + version: 1, + profiles: { + "anthropic:claude-cli": { + type: "oauth", + provider: "claude-cli", + access: "secret", + refresh: "secret", + expires: 1_900_000_000_000, + }, + }, + usageStats: { + "anthropic:claude-cli": { + cooldownUntil: 1_900_000_100_000, + cooldownReason: "session_expired", + }, + }, + } satisfies AuthProfileStore); + + const textRuntime = createRuntime(); + await modelsAuthListCommand({}, textRuntime); + expect(textRuntime.logs.at(-1)).toContain("cooldown:session_expired"); + expect(textRuntime.logs.at(-1)).toContain( + "claude auth login && openclaw models auth login --provider anthropic --method cli", + ); + + const jsonRuntime = createRuntime(); + await modelsAuthListCommand({ json: true }, jsonRuntime); + expect(jsonRuntime.jsonPayloads[0]).toMatchObject({ + profiles: [ + expect.objectContaining({ + id: "anthropic:claude-cli", + cooldownReason: "session_expired", + recoveryHint: + "Re-authenticate with `claude auth login && openclaw models auth login --provider anthropic --method cli --profile-id 'anthropic:claude-cli'`.", + }), + ], + }); + }); + it("treats the OpenAI filter as the friendly view over API-key and OAuth profiles", async () => { const store: AuthProfileStore = { version: 1, diff --git a/src/commands/models/auth-list.ts b/src/commands/models/auth-list.ts index 05c4200c8912..49400eec2275 100644 --- a/src/commands/models/auth-list.ts +++ b/src/commands/models/auth-list.ts @@ -9,6 +9,7 @@ import { type AuthProfileStore, type ProfileUsageStats, } from "../../agents/auth-profiles.js"; +import { buildAuthProfileUnusableHint } from "../../agents/auth-profiles/oauth-refresh-failure.js"; import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js"; import { type RuntimeEnv, writeRuntimeJson } from "../../runtime.js"; import { shortenHomePath } from "../../utils.js"; @@ -25,6 +26,9 @@ type AuthProfileSummary = { expiresAt?: string; cooldownUntil?: string; disabledUntil?: string; + cooldownReason?: ProfileUsageStats["cooldownReason"]; + disabledReason?: ProfileUsageStats["disabledReason"]; + recoveryHint?: string; }; function resolveProviderFilter(rawProvider: string | undefined): { @@ -65,6 +69,21 @@ function summarizeProfile(params: { const expiresAt = resolveProfileExpiry(params.profile); const cooldownUntil = formatTimestamp(params.usage?.cooldownUntil); const disabledUntil = formatTimestamp(params.usage?.disabledUntil); + const disabledActive = Boolean(disabledUntil); + const reason = disabledActive + ? params.usage?.disabledReason + : cooldownUntil + ? params.usage?.cooldownReason + : undefined; + const recoveryHint = + disabledUntil || cooldownUntil + ? buildAuthProfileUnusableHint({ + kind: disabledActive ? "disabled" : "cooldown", + reason, + provider: params.profile.provider, + profileId: params.profileId, + }) + : undefined; return { id: params.profileId, provider: resolveProviderIdForAuth(params.profile.provider), @@ -79,6 +98,9 @@ function summarizeProfile(params: { ...(expiresAt ? { expiresAt } : {}), ...(cooldownUntil ? { cooldownUntil } : {}), ...(disabledUntil ? { disabledUntil } : {}), + ...(params.usage?.cooldownReason ? { cooldownReason: params.usage.cooldownReason } : {}), + ...(params.usage?.disabledReason ? { disabledReason: params.usage.disabledReason } : {}), + ...(recoveryHint ? { recoveryHint } : {}), }; } @@ -88,12 +110,16 @@ function formatProfileLine(profile: AuthProfileSummary): string { details.push(`expires ${profile.expiresAt}`); } if (profile.cooldownUntil) { - details.push(`cooldown until ${profile.cooldownUntil}`); + details.push( + `cooldown${profile.cooldownReason ? `:${profile.cooldownReason}` : ""} until ${profile.cooldownUntil}`, + ); } if (profile.disabledUntil) { - details.push(`disabled until ${profile.disabledUntil}`); + details.push( + `disabled${profile.disabledReason ? `:${profile.disabledReason}` : ""} until ${profile.disabledUntil}`, + ); } - return `- ${profile.label} [${details.join("; ")}]`; + return `- ${profile.label} [${details.join("; ")}]${profile.recoveryHint ? ` — ${profile.recoveryHint}` : ""}`; } /** Lists auth profiles for the selected agent, optionally filtered by provider. */ diff --git a/src/commands/models/list.status-command.ts b/src/commands/models/list.status-command.ts index d86c956765a7..eb7f6720448d 100644 --- a/src/commands/models/list.status-command.ts +++ b/src/commands/models/list.status-command.ts @@ -14,6 +14,7 @@ import { DEFAULT_OAUTH_WARN_MS, formatRemainingShort, } from "../../agents/auth-health.js"; +import { buildAuthProfileUnusableHint } from "../../agents/auth-profiles/oauth-refresh-failure.js"; import { resolveAuthStorePathForDisplay } from "../../agents/auth-profiles/paths.js"; import { ensureAuthProfileStore, @@ -1254,6 +1255,7 @@ export async function modelsStatusCommand( provider?: string; kind: "cooldown" | "disabled"; reason?: string; + recoveryHint: string; until: number; remainingMs: number; }> = []; @@ -1267,11 +1269,19 @@ export async function modelsStatusCommand( typeof stats?.disabledUntil === "number" && now < stats.disabledUntil ? "disabled" : "cooldown"; + const reason = kind === "disabled" ? stats?.disabledReason : stats?.cooldownReason; + const provider = store.profiles[profileId]?.provider; out.push({ profileId, - provider: store.profiles[profileId]?.provider, + provider, kind, - reason: stats?.disabledReason, + reason, + recoveryHint: buildAuthProfileUnusableHint({ + kind, + reason, + provider: provider ?? profileId, + profileId, + }), until: unusableUntil, remainingMs: unusableUntil - now, }); @@ -1622,6 +1632,18 @@ export async function modelsStatusCommand( } } + if (unusableProfiles.length > 0) { + runtime.log(""); + runtime.log(colorize(rich, theme.heading, "Unavailable auth profiles")); + for (const profile of unusableProfiles) { + const reason = profile.reason ? `:${profile.reason}` : ""; + const provider = profile.provider ? ` (${profile.provider})` : ""; + runtime.log( + `- ${theme.heading(profile.profileId)}${provider} ${profile.kind}${reason} (${formatRemainingShort(profile.remainingMs)}) — ${profile.recoveryHint}`, + ); + } + } + runtime.log(""); runtime.log(colorize(rich, theme.heading, "OAuth/token status")); if (oauthProfiles.length === 0) { diff --git a/src/commands/models/list.status.test.ts b/src/commands/models/list.status.test.ts index 59ee591ef4e0..bf98dcb63134 100644 --- a/src/commands/models/list.status.test.ts +++ b/src/commands/models/list.status.test.ts @@ -554,6 +554,48 @@ async function withOpenAIStatusFixture( } describe("modelsStatusCommand auth overview", () => { + it("shows cooldown reasons and recovery guidance in JSON and text output", async () => { + const now = Date.now(); + const store = mocks.store as typeof mocks.store & { + usageStats?: Record; + }; + store.usageStats = { + "anthropic:default": { + cooldownUntil: now + 60_000, + cooldownReason: "session_expired", + }, + }; + mocks.resolveProfileUnusableUntilForDisplay.mockImplementation((_store, profileId) => + profileId === "anthropic:default" ? now + 60_000 : undefined, + ); + + try { + const jsonRuntime = createRuntime(); + await modelsStatusCommand({ json: true }, jsonRuntime as never); + expect(parseFirstJsonLog(jsonRuntime).auth.unusableProfiles).toEqual([ + expect.objectContaining({ + profileId: "anthropic:default", + kind: "cooldown", + reason: "session_expired", + recoveryHint: + "Re-authenticate with `openclaw models auth login --provider anthropic --profile-id 'anthropic:default'`.", + }), + ]); + + const textRuntime = createRuntime(); + await modelsStatusCommand({}, textRuntime as never); + const output = (textRuntime.log as Mock).mock.calls + .map((call: unknown[]) => String(call[0])) + .join("\n"); + expect(output).toContain("Unavailable auth profiles"); + expect(output).toContain("anthropic:default (anthropic) cooldown:session_expired"); + expect(output).toContain("openclaw models auth login --provider anthropic"); + } finally { + delete store.usageStats; + mocks.resolveProfileUnusableUntilForDisplay.mockReset().mockReturnValue(undefined); + } + }); + it("does not restore over plugin metadata published while status is running", async () => { const originalLoadModelCatalog = mocks.loadModelCatalog.getMockImplementation(); const config = mocks.loadConfig();