From 1a27e7f3ec2febf4fc36c7b5bd2a984630e05449 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 15 Jul 2026 04:10:57 -0700 Subject: [PATCH] fix(codex): model-scoped usage-limit blocks, structural 429 classification, no silent API-key billing (#108254) * test(codex): use allowlisted placeholder for auth-bridge api-key fixture * fix(codex): model-scoped usage-limit blocks, structural 429 classification, no silent API-key billing - usage-limit blocks written from Codex rate-limit resets are model-scoped via a persisted blockedScope marker: healthy sibling models on the same auth profile stay usable; a different/unknown model failing widens the block profile-wide and never narrows back; legacy rows without the marker stay profile-wide until they expire (#100556) - Codex usage-limit failures now surface as status-429 Error objects at both ingress paths (turn-start and streamed turn failure, wrapped at the event projector), so core failover classification is structural instead of matching message wording; profile blocking uses only rate-limit data whose revision advanced during the turn - usage-limit detection requires the structured codexErrorInfo signal; the over-broad "usage limit" substring match that misclassified unrelated errors as subscription limits is gone (#96815) - subscription/OAuth routes can no longer silently fall back to env/auth.json API keys: ambient key fallback is restricted to explicit api-key routes, native-auth subscription routes verify the account is chatgpt-backed via account/read, and shared app-server clients are partitioned by auth requirement so pooled clients cannot cross billing modes (#106375) - integrated e2e regression: usage-limit promptError -> same-model sibling profile rotation -> model fallback with reason rate_limit * chore(codex): keep CodexUsageLimitErrorResult type local --- .../codex/src/app-server/attempt-results.ts | 3 +- .../codex/src/app-server/attempt-startup.ts | 2 + .../codex/src/app-server/auth-bridge.test.ts | 117 +++++++++++- .../codex/src/app-server/auth-bridge.ts | 100 +++++++++-- .../src/app-server/event-projector.test.ts | 31 ++-- .../codex/src/app-server/event-projector.ts | 34 ++-- extensions/codex/src/app-server/models.ts | 8 +- .../codex/src/app-server/rate-limits.test.ts | 22 +++ .../codex/src/app-server/rate-limits.ts | 17 +- .../src/app-server/run-attempt-connection.ts | 1 + .../src/app-server/run-attempt-finalize.ts | 34 +++- .../codex/src/app-server/run-attempt-start.ts | 2 + .../src/app-server/run-attempt-turn-start.ts | 2 + .../run-attempt.usage-limits.test.ts | 168 ++++++++++++++++-- .../src/app-server/shared-client.test.ts | 53 +++++- .../codex/src/app-server/shared-client.ts | 40 ++++- .../codex/src/app-server/side-question.ts | 1 + .../codex/src/app-server/usage-limit-error.ts | 32 ++-- src/agents/auth-profiles/order.test.ts | 50 ++++++ src/agents/auth-profiles/order.ts | 5 +- src/agents/auth-profiles/state.ts | 1 + src/agents/auth-profiles/types.ts | 1 + src/agents/auth-profiles/usage-state.ts | 52 +++++- src/agents/auth-profiles/usage.test.ts | 113 +++++++++++- src/agents/auth-profiles/usage.ts | 15 +- src/agents/failover-error.test.ts | 12 ++ .../model-fallback.run-embedded.e2e.test.ts | 47 ++++- 27 files changed, 850 insertions(+), 113 deletions(-) diff --git a/extensions/codex/src/app-server/attempt-results.ts b/extensions/codex/src/app-server/attempt-results.ts index 05d2e04d5e5b..5b5631b9d347 100644 --- a/extensions/codex/src/app-server/attempt-results.ts +++ b/extensions/codex/src/app-server/attempt-results.ts @@ -104,6 +104,7 @@ export function resolveCodexAppServerReplayBlockedReason( export function buildCodexTurnStartFailureResult(params: { params: EmbeddedRunAttemptParams; message: string; + promptError?: unknown; messagesSnapshot: AgentMessage[]; systemPromptReport: CodexSystemPromptReport; }): EmbeddedRunAttemptResult { @@ -114,7 +115,7 @@ export function buildCodexTurnStartFailureResult(params: { idleTimedOut: false, timedOutDuringCompaction: false, timedOutDuringToolExecution: false, - promptError: params.message, + promptError: params.promptError ?? params.message, promptErrorSource: "prompt", sessionIdUsed: params.params.sessionId, messagesSnapshot: params.messagesSnapshot, diff --git a/extensions/codex/src/app-server/attempt-startup.ts b/extensions/codex/src/app-server/attempt-startup.ts index ec2c5481e309..29490822cc40 100644 --- a/extensions/codex/src/app-server/attempt-startup.ts +++ b/extensions/codex/src/app-server/attempt-startup.ts @@ -125,6 +125,7 @@ export async function startCodexAttemptThread(params: { pluginConfig: CodexPluginConfig; computerUseConfig: ResolvedCodexComputerUseConfig; startupAuthProfileId: string | null | undefined; + startupAuthRequirement?: CodexAppServerClientOptions["authRequirement"]; startupAuthBindingFingerprint: string | undefined; runtimeArtifactRequest?: Readonly<{ expected?: AgentHarnessRuntimeArtifactBinding; @@ -228,6 +229,7 @@ export async function startCodexAttemptThread(params: { ...(params.startupPreparedAuth ? { preparedAuth: params.startupPreparedAuth } : { authProfileId: params.startupAuthProfileId }), + authRequirement: params.startupAuthRequirement, authProfileStore: attemptParams.authProfileStore, authBindingFingerprint: params.startupAuthBindingFingerprint, ...(params.runtimeArtifactRequest diff --git a/extensions/codex/src/app-server/auth-bridge.test.ts b/extensions/codex/src/app-server/auth-bridge.test.ts index 3276b287ed67..2cb4de56ce3e 100644 --- a/extensions/codex/src/app-server/auth-bridge.test.ts +++ b/extensions/codex/src/app-server/auth-bridge.test.ts @@ -1965,6 +1965,112 @@ describe("bridgeCodexAppServerStartOptions", () => { } }); + it("fails subscription auth instead of falling back to an API key", async () => { + const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-app-server-")); + const request = vi.fn(async () => ({ type: "apiKey" })); + vi.stubEnv("CODEX_API_KEY", "placeholder"); + let rejection: unknown; + try { + await applyCodexAppServerAuthProfile({ + client: { request } as never, + agentDir, + authProfileId: "openai:work", + authProfileStore: { + version: 1, + profiles: {}, + }, + authRequirement: "subscription", + startOptions: createStartOptions({ + env: { CODEX_API_KEY: "placeholder" }, + }), + }); + } catch (error) { + rejection = error; + } finally { + await fs.rm(agentDir, { recursive: true, force: true }); + } + + expect(rejection).toBeInstanceOf(Error); + expect((rejection as { status?: unknown }).status).toBe(401); + expect(request).not.toHaveBeenCalled(); + }); + + it("preserves transient subscription credential resolution errors", async () => { + const transientError = Object.assign(new Error("temporary refresh failure"), { status: 503 }); + oauthMocks.refreshOpenAICodexToken.mockRejectedValueOnce(transientError); + const request = vi.fn(); + + await expect( + applyCodexAppServerAuthProfile({ + client: { request } as never, + agentDir: "/tmp/openclaw-agent", + authProfileId: "openai:work", + authProfileStore: { + version: 1, + profiles: { + "openai:work": { + type: "oauth", + provider: "openai", + access: "placeholder", + refresh: "placeholder", + expires: Date.now() - 60_000, + }, + }, + }, + authRequirement: "subscription", + }), + ).rejects.toBe(transientError); + expect(request).not.toHaveBeenCalled(); + }); + + it("accepts native ChatGPT auth for subscription routes", async () => { + const request = vi.fn(async () => ({ + account: { type: "chatgpt", email: null, planType: "plus" }, + requiresOpenaiAuth: true, + })); + + await applyCodexAppServerAuthProfile({ + client: { request } as never, + agentDir: "/tmp/openclaw-agent", + authProfileId: null, + authRequirement: "subscription", + }); + + expect(request).toHaveBeenCalledOnce(); + expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false }); + }); + + it("rejects native API-key auth for subscription routes", async () => { + const request = vi.fn(async () => ({ + account: { type: "apiKey" }, + requiresOpenaiAuth: false, + })); + + await expect( + applyCodexAppServerAuthProfile({ + client: { request } as never, + agentDir: "/tmp/openclaw-agent", + authProfileId: null, + authRequirement: "subscription", + }), + ).rejects.toMatchObject({ status: 401 }); + expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false }); + }); + + it("rejects missing native auth for subscription routes", async () => { + const request = vi.fn(async () => ({ account: null, requiresOpenaiAuth: true })); + + await expect( + applyCodexAppServerAuthProfile({ + client: { request } as never, + agentDir: "/tmp/openclaw-agent", + authProfileId: null, + authRequirement: "subscription", + }), + ).rejects.toMatchObject({ status: 401 }); + expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false }); + }); + it("falls back to CODEX_API_KEY when no auth profile and no Codex account is available", async () => { const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-app-server-")); const request = vi.fn(async (method: string) => { @@ -1979,15 +2085,16 @@ describe("bridgeCodexAppServerStartOptions", () => { await applyCodexAppServerAuthProfile({ client: { request } as never, agentDir, + authRequirement: "api-key", startOptions: createStartOptions({ - env: { CODEX_API_KEY: "configured-codex-api-key" }, + env: { CODEX_API_KEY: "test-token-placeholder" }, }), }); expect(request).toHaveBeenNthCalledWith(1, "account/read", { refreshToken: false }); expect(request).toHaveBeenNthCalledWith(2, "account/login/start", { type: "apiKey", - apiKey: "configured-codex-api-key", + apiKey: "test-token-placeholder", }); } finally { await fs.rm(agentDir, { recursive: true, force: true }); @@ -2008,6 +2115,7 @@ describe("bridgeCodexAppServerStartOptions", () => { await applyCodexAppServerAuthProfile({ client: { request } as never, agentDir, + authRequirement: "api-key", startOptions: createStartOptions(), }); @@ -2037,6 +2145,7 @@ describe("bridgeCodexAppServerStartOptions", () => { await applyCodexAppServerAuthProfile({ client: { request } as never, agentDir, + authRequirement: "api-key", startOptions: createStartOptions(), }); @@ -2060,6 +2169,7 @@ describe("bridgeCodexAppServerStartOptions", () => { await applyCodexAppServerAuthProfile({ client: { request } as never, agentDir, + authRequirement: "api-key", startOptions: createStartOptions(), }); @@ -2092,6 +2202,7 @@ describe("bridgeCodexAppServerStartOptions", () => { await applyCodexAppServerAuthProfile({ client: { request } as never, agentDir, + authRequirement: "api-key", startOptions: createStartOptions({ env: { CODEX_HOME: path.join(root, "isolated-codex-home") }, }), @@ -2173,6 +2284,7 @@ describe("bridgeCodexAppServerStartOptions", () => { await applyCodexAppServerAuthProfile({ client: { request } as never, agentDir, + authRequirement: "api-key", startOptions: createStartOptions({ clearEnv: ["CODEX_API_KEY", "OPENAI_API_KEY"], }), @@ -2198,6 +2310,7 @@ describe("bridgeCodexAppServerStartOptions", () => { await applyCodexAppServerAuthProfile({ client: { request } as never, agentDir, + authRequirement: "api-key", startOptions: createStartOptions({ transport: "websocket", url: "ws://127.0.0.1:1455", diff --git a/extensions/codex/src/app-server/auth-bridge.ts b/extensions/codex/src/app-server/auth-bridge.ts index e599f7c23a6d..62f0191ac8aa 100644 --- a/extensions/codex/src/app-server/auth-bridge.ts +++ b/extensions/codex/src/app-server/auth-bridge.ts @@ -29,10 +29,11 @@ import { resolveCodexComputerUseConfig, type CodexAppServerStartOptions, } from "./config.js"; -import type { - CodexChatgptAuthTokensRefreshResponse, - CodexGetAccountResponse, - CodexLoginAccountParams, +import { + isJsonObject, + type CodexChatgptAuthTokensRefreshResponse, + type CodexGetAccountResponse, + type CodexLoginAccountParams, } from "./protocol.js"; import { isCodexAppServerNativeAuthProfile } from "./session-binding.js"; import { resolveCodexAppServerSpawnEnv } from "./transport-stdio.js"; @@ -61,6 +62,7 @@ const CODEX_APP_SERVER_HOME_ENV_VARS = [CODEX_HOME_ENV_VAR, HOME_ENV_VAR]; const CODEX_AUTH_JSON_FILENAME = "auth.json"; const CODEX_HOME_DIRNAME = ".codex"; type AuthProfileOrderConfig = Parameters[0]["cfg"]; +export type CodexAppServerAuthRequirement = "api-key" | "subscription"; const scopedOAuthRefreshQueues = new WeakMap< AuthProfileStore, Map> @@ -250,7 +252,7 @@ export async function resolveCodexAppServerPreparedAuthProfileSnapshot(params: { /** Maps one prepared route to one mutually exclusive app-server auth handoff. */ export async function resolveCodexAppServerPreparedAuthHandoff(params: { - authRequirement?: "api-key" | "subscription"; + authRequirement?: CodexAppServerAuthRequirement; resolvedApiKey?: string; authProfileId?: string; authProfileStore: AuthProfileStore; @@ -281,7 +283,7 @@ export async function resolveCodexAppServerPreparedAuthHandoff(params: { return { authProfileId, nativeAuthProfile }; } if (!authProfileId || !nativeAuthProfile) { - throw new Error(params.subscriptionProfileRequiredError); + throw createCodexAppServerAuthError(params.subscriptionProfileRequiredError); } const snapshot = await resolveCodexAppServerPreparedAuthProfileSnapshot({ @@ -291,7 +293,7 @@ export async function resolveCodexAppServerPreparedAuthHandoff(params: { config: params.config, }); if (!snapshot) { - throw new Error(params.subscriptionProfileUnusableError); + throw createCodexAppServerAuthError(params.subscriptionProfileUnusableError); } return { authProfileId, @@ -474,6 +476,7 @@ export async function applyCodexAppServerAuthProfile(params: { authProfileId?: string | null; authProfileStore?: AuthProfileStore; preparedAuth?: CodexAppServerResolvedPreparedAuth; + authRequirement?: CodexAppServerAuthRequirement; startOptions?: CodexAppServerStartOptions; config?: AuthProfileOrderConfig; }): Promise { @@ -489,16 +492,52 @@ export async function applyCodexAppServerAuthProfile(params: { return; } if (params.authProfileId === null) { + if (params.authRequirement === "subscription") { + const response = await params.client.request("account/read", { + refreshToken: false, + }); + if (!isJsonObject(response.account) || response.account.type !== "chatgpt") { + throw createCodexAppServerAuthError( + "Codex subscription auth profile could not produce login credentials.", + ); + } + } return; } - const loginParams = await resolveCodexAppServerAuthProfileLoginParams({ - agentDir: params.agentDir, - authProfileId: params.authProfileId, - authProfileStore: params.authProfileStore, - config: params.config, - }); + let loginParams: CodexLoginAccountParams | undefined; + try { + loginParams = await resolveCodexAppServerAuthProfileLoginParams({ + agentDir: params.agentDir, + authProfileId: params.authProfileId, + authProfileStore: params.authProfileStore, + config: params.config, + }); + } catch (error) { + if ( + params.authRequirement === "subscription" && + error instanceof CodexAppServerAuthProfileUnavailableError + ) { + throw createCodexAppServerAuthError( + "Codex subscription auth profile could not produce login credentials.", + error, + ); + } + throw error; + } + if (params.authRequirement === "subscription" && loginParams?.type !== "chatgptAuthTokens") { + throw createCodexAppServerAuthError( + "Codex subscription auth profile could not produce login credentials.", + ); + } if (!loginParams) { - if (params.startOptions?.transport !== "stdio") { + // Observe native state only for explicit API-key routes. A subscription + // route must fail here so profile rotation can run before billing changes. + if (params.authRequirement === "subscription") { + throw createCodexAppServerAuthError( + "Codex subscription auth profile could not produce login credentials.", + ); + } + if (params.authRequirement !== "api-key" || params.startOptions?.transport !== "stdio") { return; } const env = resolveCodexAppServerSpawnEnv(params.startOptions, process.env); @@ -515,13 +554,40 @@ export async function applyCodexAppServerAuthProfile(params: { await params.client.request("account/login/start", loginParams); } -function resolveCodexAppServerAuthProfileLoginParams(params: { +function createCodexAppServerAuthError(message: string, cause?: unknown): Error & { status: 401 } { + const error = cause === undefined ? new Error(message) : new Error(message, { cause }); + return Object.assign(error, { status: 401 as const }); +} + +class CodexAppServerAuthProfileUnavailableError extends Error {} + +async function resolveCodexAppServerAuthProfileLoginParams(params: { agentDir: string; authProfileId?: string; authProfileStore?: AuthProfileStore; config?: AuthProfileOrderConfig; }): Promise { - return resolveCodexAppServerAuthProfileLoginParamsInternal(params); + const store = resolveCodexAppServerAuthProfileStore(params); + const profileId = resolveCodexAppServerAuthProfileId({ + authProfileId: params.authProfileId, + store, + config: params.config, + }); + const profile = profileId ? store.profiles[profileId] : undefined; + if (profileId && !profile) { + throw new CodexAppServerAuthProfileUnavailableError( + `Codex app-server auth profile "${profileId}" was not found.`, + ); + } + if (profileId && profile && !isCodexAppServerAuthProfileCredential(profile, params.config)) { + throw new CodexAppServerAuthProfileUnavailableError( + `Codex app-server auth profile "${profileId}" must be OpenAI Codex auth or an OpenAI API-key backup.`, + ); + } + return await resolveCodexAppServerAuthProfileLoginParamsInternal({ + ...params, + authProfileStore: store, + }); } export async function refreshCodexAppServerAuthTokens(params: { @@ -582,7 +648,7 @@ async function resolveCodexAppServerAuthProfileLoginParamsInternal(params: { config: params.config, }); if (!loginParams) { - throw new Error( + throw new CodexAppServerAuthProfileUnavailableError( `Codex app-server auth profile "${profileId}" does not contain usable credentials.`, ); } diff --git a/extensions/codex/src/app-server/event-projector.test.ts b/extensions/codex/src/app-server/event-projector.test.ts index dae0e9ace8f0..646187520b08 100644 --- a/extensions/codex/src/app-server/event-projector.test.ts +++ b/extensions/codex/src/app-server/event-projector.test.ts @@ -140,6 +140,13 @@ function buildEmptyToolTelemetry(): CodexAppServerToolTelemetry { }; } +function expectUsageLimitPromptError(value: unknown): Error & { status: 429 } { + expect(value).toBeInstanceOf(Error); + const error = value as Error & { status?: unknown }; + expect(error.status).toBe(429); + return error as Error & { status: 429 }; +} + function requireRecord(value: unknown, label: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new Error(`Expected ${label}`); @@ -1851,9 +1858,10 @@ describe("CodexAppServerEventProjector", () => { const result = projector.buildResult(buildEmptyToolTelemetry()); - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Next reset in"); - expect(result.promptError).toContain("Wait until the reset time"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Next reset in"); + expect(promptError.message).toContain("Wait until the reset time"); expect(result.promptErrorSource).toBe("prompt"); }); @@ -1880,8 +1888,9 @@ describe("CodexAppServerEventProjector", () => { const result = projector.buildResult(buildEmptyToolTelemetry()); - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Next reset in"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Next reset in"); expect(result.promptErrorSource).toBe("prompt"); }); @@ -1920,8 +1929,9 @@ describe("CodexAppServerEventProjector", () => { const result = projector.buildResult(buildEmptyToolTelemetry()); - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Next reset in"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Next reset in"); expect(result.promptErrorSource).toBe("prompt"); }); @@ -1946,9 +1956,10 @@ describe("CodexAppServerEventProjector", () => { const result = projector.buildResult(buildEmptyToolTelemetry()); - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Codex says to try again at May 11th, 2026 9:00 AM."); - expect(result.promptError).not.toContain("Codex did not return a reset time"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Codex says to try again at May 11th, 2026 9:00 AM."); + expect(promptError.message).not.toContain("Codex did not return a reset time"); expect(result.promptErrorSource).toBe("prompt"); }); diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index 19f194974401..2fe83c2b23a1 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -50,6 +50,7 @@ import { import { formatCodexUsageLimitErrorMessage } from "./rate-limits.js"; import type { CodexTrajectoryRecorder } from "./trajectory.js"; import { attachCodexMirrorIdentity } from "./upstream-prompt-provenance.js"; +import { createCodexUsageLimitPromptError } from "./usage-limit-error.js"; import { promptSnapshot } from "./user-prompt-message.js"; export { CodexNativeToolLifecycleProjector }; @@ -575,14 +576,14 @@ export class CodexAppServerEventProjector { } this.completedTurn = turn; if (turn.status === "failed") { - this.promptError = - formatCodexUsageLimitErrorMessage({ - message: turn.error?.message, - codexErrorInfo: turn.error?.codexErrorInfo as JsonValue | null | undefined, - rateLimits: this.options.readRecentRateLimits?.(), - }) ?? - turn.error?.message ?? - "codex app-server turn failed"; + const usageLimitMessage = formatCodexUsageLimitErrorMessage({ + message: turn.error?.message, + codexErrorInfo: turn.error?.codexErrorInfo as JsonValue | null | undefined, + rateLimits: this.options.readRecentRateLimits?.(), + }); + this.promptError = usageLimitMessage + ? createCodexUsageLimitPromptError(usageLimitMessage) + : (turn.error?.message ?? "codex app-server turn failed"); this.promptErrorSource = "prompt"; } const turnItems = turn.items ?? []; @@ -674,15 +675,16 @@ export class CodexAppServerEventProjector { }); } - private formatCodexErrorMessage(params: JsonObject): string | undefined { + private formatCodexErrorMessage(params: JsonObject): string | Error | undefined { const error = isJsonObject(params.error) ? params.error : undefined; - return ( - formatCodexUsageLimitErrorMessage({ - message: error ? readString(error, "message") : undefined, - codexErrorInfo: error?.codexErrorInfo, - rateLimits: this.options.readRecentRateLimits?.(), - }) ?? readCodexErrorNotificationMessage(params) - ); + const usageLimitMessage = formatCodexUsageLimitErrorMessage({ + message: error ? readString(error, "message") : undefined, + codexErrorInfo: error?.codexErrorInfo, + rateLimits: this.options.readRecentRateLimits?.(), + }); + return usageLimitMessage + ? createCodexUsageLimitPromptError(usageLimitMessage) + : readCodexErrorNotificationMessage(params); } private emitAgentEvent( diff --git a/extensions/codex/src/app-server/models.ts b/extensions/codex/src/app-server/models.ts index 47ec8109c606..2222494174b5 100644 --- a/extensions/codex/src/app-server/models.ts +++ b/extensions/codex/src/app-server/models.ts @@ -3,7 +3,10 @@ * endpoint, including pagination and shared-client lease handling. */ import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; -import type { resolveCodexAppServerAuthProfileIdForAgent } from "./auth-bridge.js"; +import type { + CodexAppServerAuthRequirement, + resolveCodexAppServerAuthProfileIdForAgent, +} from "./auth-bridge.js"; import type { CodexAppServerClient } from "./client.js"; import type { CodexAppServerStartOptions } from "./config.js"; import { readCodexModelListResponse } from "./protocol-validators.js"; @@ -37,6 +40,7 @@ export type CodexAppServerListModelsOptions = { timeoutMs?: number; startOptions?: CodexAppServerStartOptions; authProfileId?: string; + authRequirement?: CodexAppServerAuthRequirement; agentDir?: string; config?: Parameters[0]["config"]; sharedClient?: boolean; @@ -93,6 +97,7 @@ async function withCodexAppServerModelClient( startOptions: options.startOptions, timeoutMs, authProfileId: options.authProfileId, + authRequirement: options.authRequirement, agentDir: options.agentDir, config: options.config, }) @@ -100,6 +105,7 @@ async function withCodexAppServerModelClient( startOptions: options.startOptions, timeoutMs, authProfileId: options.authProfileId, + authRequirement: options.authRequirement, agentDir: options.agentDir, config: options.config, }); diff --git a/extensions/codex/src/app-server/rate-limits.test.ts b/extensions/codex/src/app-server/rate-limits.test.ts index 935e6dd2ad9c..f1e37be202b4 100644 --- a/extensions/codex/src/app-server/rate-limits.test.ts +++ b/extensions/codex/src/app-server/rate-limits.test.ts @@ -4,11 +4,33 @@ import { buildCodexAppServerUsageSnapshot, formatCodexUsageLimitErrorMessage, resolveCodexUsageLimitResetAtMs, + shouldRefreshCodexRateLimitsForUsageLimitMessage, summarizeCodexAccountUsage, summarizeCodexRateLimits, } from "./rate-limits.js"; describe("formatCodexUsageLimitErrorMessage", () => { + it("does not infer a Codex usage limit from unrelated prose", () => { + expect( + formatCodexUsageLimitErrorMessage({ + message: "The workspace usage limit setting could not be loaded.", + }), + ).toBeUndefined(); + expect(shouldRefreshCodexRateLimitsForUsageLimitMessage("temporary usage limit warning")).toBe( + false, + ); + }); + + it("accepts normalized structured Codex usage-limit error info", () => { + const message = formatCodexUsageLimitErrorMessage({ + message: "quota exhausted", + codexErrorInfo: "usage_limit-exceeded", + }); + + expect(message?.startsWith("You've reached your Codex subscription usage limit.")).toBe(true); + expect(shouldRefreshCodexRateLimitsForUsageLimitMessage(message)).toBe(true); + }); + it("gives actionable guidance when Codex omits reset details", () => { const message = formatCodexUsageLimitErrorMessage({ message: "You've reached your usage limit.", diff --git a/extensions/codex/src/app-server/rate-limits.ts b/extensions/codex/src/app-server/rate-limits.ts index 9b87e3013112..ce58b7fc77c6 100644 --- a/extensions/codex/src/app-server/rate-limits.ts +++ b/extensions/codex/src/app-server/rate-limits.ts @@ -25,6 +25,7 @@ const ONE_DAY_MS = 24 * ONE_HOUR_MS; const DAY_WINDOW_MINUTES = 24 * 60; const WEEKLY_WINDOW_MINUTES = 7 * DAY_WINDOW_MINUTES; const WEEKLY_RESET_GAP_MS = 3 * ONE_DAY_MS; +const CODEX_USAGE_LIMIT_MESSAGE_PREFIX = "You've reached your Codex subscription usage limit."; type LimitWindowKey = (typeof LIMIT_WINDOW_KEYS)[number]; @@ -58,7 +59,7 @@ export function formatCodexUsageLimitErrorMessage(params: { nowMs?: number; }): string | undefined { const message = normalizeText(params.message); - if (!isCodexUsageLimitError(params.codexErrorInfo, message)) { + if (!isCodexUsageLimitError(params.codexErrorInfo)) { return undefined; } const nowMs = params.nowMs ?? Date.now(); @@ -67,7 +68,7 @@ export function formatCodexUsageLimitErrorMessage(params: { const nextReset = blockingReset ?? (usageSummary?.blocked ? undefined : selectNextRateLimitReset(params.rateLimits, nowMs)); - const parts = ["You've reached your Codex subscription usage limit."]; + const parts = [CODEX_USAGE_LIMIT_MESSAGE_PREFIX]; let recoveryAction = "Wait until Codex becomes available"; if (nextReset) { parts.push(`Next reset ${formatResetTime(nextReset.resetsAtMs, nowMs)}.`); @@ -95,9 +96,10 @@ export function shouldRefreshCodexRateLimitsForUsageLimitMessage( message: string | null | undefined, ): boolean { const text = normalizeText(message); + // Only our formatted prefix is a refresh contract. Provider prose alone is + // not structural evidence of a Codex usage-limit failure. return Boolean( - text?.includes("You've reached your Codex subscription usage limit.") && - !text.includes("Next reset "), + text?.startsWith(CODEX_USAGE_LIMIT_MESSAGE_PREFIX) && !text.includes("Next reset "), ); } @@ -210,10 +212,7 @@ export function buildCodexAppServerUsageSnapshot(value: unknown): ProviderUsageS }; } -function isCodexUsageLimitError( - codexErrorInfo: JsonValue | null | undefined, - message: string | undefined, -): boolean { +function isCodexUsageLimitError(codexErrorInfo: JsonValue | null | undefined): boolean { if (codexErrorInfo === "usageLimitExceeded") { return true; } @@ -223,7 +222,7 @@ function isCodexUsageLimitError( return true; } } - return Boolean(message?.toLowerCase().includes("usage limit")); + return false; } function selectNextRateLimitReset( diff --git a/extensions/codex/src/app-server/run-attempt-connection.ts b/extensions/codex/src/app-server/run-attempt-connection.ts index 4045a6bb2ece..f847062f2f18 100644 --- a/extensions/codex/src/app-server/run-attempt-connection.ts +++ b/extensions/codex/src/app-server/run-attempt-connection.ts @@ -365,6 +365,7 @@ export async function prepareCodexAttemptConnection({ params, options }: CodexRu isInactiveThreadBootstrapBinding, usesSupervisionConnection, startupAuthProfileId, + startupAuthRequirement: preparedAuthRoute?.authRequirement, startupPreparedAuth, startupClientAuthProfileId, effectiveWorkspace, diff --git a/extensions/codex/src/app-server/run-attempt-finalize.ts b/extensions/codex/src/app-server/run-attempt-finalize.ts index 6a17d79324f0..174d5624afce 100644 --- a/extensions/codex/src/app-server/run-attempt-finalize.ts +++ b/extensions/codex/src/app-server/run-attempt-finalize.ts @@ -17,6 +17,7 @@ import { isInvalidCodexImagePayloadError, resolveCodexAppServerReplayBlockedReason, } from "./attempt-results.js"; +import { readCodexRateLimitsRevision, readRecentCodexRateLimits } from "./rate-limit-cache.js"; import type { CodexAttemptActiveTurn } from "./run-attempt-active-turn.js"; import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js"; import { @@ -36,7 +37,12 @@ import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request. import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; import { normalizeCodexTrajectoryError, recordCodexTrajectoryCompletion } from "./trajectory.js"; import { codexTranscriptMirrorRuntime } from "./transcript-mirror.js"; -import { refreshCodexUsageLimitPromptError } from "./usage-limit-error.js"; +import { + createCodexUsageLimitPromptError, + isCodexUsageLimitPromptError, + markCodexAuthProfileBlockedFromRateLimits, + refreshCodexUsageLimitPromptError, +} from "./usage-limit-error.js"; export async function finalizeCodexAttempt( resources: CodexAttemptResources, @@ -72,6 +78,7 @@ export async function finalizeCodexAttempt( effectiveWorkspace, agentDir, attemptStartedAt, + startupAuthProfileId, } = connection; const { toolBridge, toolState } = attemptTools; const { @@ -158,9 +165,11 @@ export async function finalizeCodexAttempt( const finalPromptErrorMessage = typeof finalPromptError === "string" ? finalPromptError - : finalPromptError - ? formatErrorMessage(finalPromptError) - : undefined; + : finalPromptError instanceof Error + ? finalPromptError.message + : finalPromptError + ? formatErrorMessage(finalPromptError) + : undefined; if (isInvalidCodexImagePayloadError(finalPromptErrorMessage)) { await clearCodexBindingAfterInvalidImagePayload(bindingStore, bindingIdentity, { phase: "turn_completed", @@ -197,7 +206,22 @@ export async function finalizeCodexAttempt( signal: runAbortController.signal, }); if (refreshedUsageLimitPromptError) { - finalPromptError = refreshedUsageLimitPromptError; + await markCodexAuthProfileBlockedFromRateLimits({ + params, + authProfileId: startupAuthProfileId, + rateLimits: refreshedUsageLimitPromptError.rateLimitsForProfile, + }); + finalPromptError = createCodexUsageLimitPromptError(refreshedUsageLimitPromptError.message); + } else if ( + isCodexUsageLimitPromptError(finalPromptError) && + state.rateLimitsRevisionBeforeLastTurnStart !== undefined && + readCodexRateLimitsRevision(resourceState.client) > state.rateLimitsRevisionBeforeLastTurnStart + ) { + await markCodexAuthProfileBlockedFromRateLimits({ + params, + authProfileId: startupAuthProfileId, + rateLimits: readRecentCodexRateLimits(resourceState.client), + }); } const finalPromptErrorSource = effectiveTimedOut || clientClosedPromptErrorForFinal ? "prompt" : result.promptErrorSource; diff --git a/extensions/codex/src/app-server/run-attempt-start.ts b/extensions/codex/src/app-server/run-attempt-start.ts index cb0bbc08007c..872c310ab3b5 100644 --- a/extensions/codex/src/app-server/run-attempt-start.ts +++ b/extensions/codex/src/app-server/run-attempt-start.ts @@ -62,6 +62,7 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources) resolveReviewerPolicyContext, resolveRuntimeOptionsForCurrentBinding, startupAuthProfileId, + startupAuthRequirement, abortFromUpstream, } = connection; let pluginAppServer = withCodexAppServerFastModeServiceTier(appServer, runtimeParams); @@ -77,6 +78,7 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources) pluginConfig, computerUseConfig, startupAuthProfileId: startupClientAuthProfileId, + startupAuthRequirement, startupAuthBindingFingerprint: preparedAuthBinding?.fingerprint, ...(runtimeArtifactRequest ? { runtimeArtifactRequest } : {}), startupPreparedAuth, diff --git a/extensions/codex/src/app-server/run-attempt-turn-start.ts b/extensions/codex/src/app-server/run-attempt-turn-start.ts index 168d5f172ecd..5439d51dd545 100644 --- a/extensions/codex/src/app-server/run-attempt-turn-start.ts +++ b/extensions/codex/src/app-server/run-attempt-turn-start.ts @@ -29,6 +29,7 @@ import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request. import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; import { buildCodexUserPromptMessage } from "./transcript-mirror.js"; import { + createCodexUsageLimitPromptError, formatCodexTurnStartUsageLimitError, markCodexAuthProfileBlockedFromRateLimits, } from "./usage-limit-error.js"; @@ -262,6 +263,7 @@ export async function startCodexAttemptTurn( result: buildCodexTurnStartFailureResult({ params, message: usageLimitError.message, + promptError: createCodexUsageLimitPromptError(usageLimitError.message), messagesSnapshot, systemPromptReport, }), diff --git a/extensions/codex/src/app-server/run-attempt.usage-limits.test.ts b/extensions/codex/src/app-server/run-attempt.usage-limits.test.ts index 24dea8773fe0..779d7d7ffef6 100644 --- a/extensions/codex/src/app-server/run-attempt.usage-limits.test.ts +++ b/extensions/codex/src/app-server/run-attempt.usage-limits.test.ts @@ -1,5 +1,6 @@ // Codex tests cover run attempt.usage limits plugin behavior. import path from "node:path"; +import { saveAuthProfileStore } from "openclaw/plugin-sdk/agent-runtime"; import { describe, expect, it } from "vitest"; import { readCodexRateLimitsRevision, rememberCodexRateLimitsRead } from "./rate-limit-cache.js"; import { @@ -13,6 +14,13 @@ import { setupRunAttemptTestHooks(); +function expectUsageLimitPromptError(value: unknown): Error & { status: 429 } { + expect(value).toBeInstanceOf(Error); + const error = value as Error & { status?: unknown }; + expect(error.status).toBe(429); + return error as Error & { status: 429 }; +} + describe("runCodexAppServerAttempt usage limits", () => { it("preserves Codex usage-limit reset details when turn/start fails", async () => { const sessionFile = path.join(tempDir, "session.jsonl"); @@ -39,6 +47,7 @@ describe("runCodexAppServerAttempt usage limits", () => { harnessRef.current = harness; const params = createParams(sessionFile, workspaceDir); + params.agentDir = path.join(tempDir, "agent"); params.authProfileId = authProfileId; params.authProfileStore = { version: 1, @@ -46,8 +55,8 @@ describe("runCodexAppServerAttempt usage limits", () => { [authProfileId]: { type: "oauth", provider: "openai", - access: "access", - refresh: "refresh", + access: "placeholder", + refresh: "placeholder", expires: Date.now() + 60_000, }, }, @@ -55,8 +64,9 @@ describe("runCodexAppServerAttempt usage limits", () => { const result = await runCodexAppServerAttempt(params); expect(result.promptErrorSource).toBe("prompt"); - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Next reset in"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Next reset in"); }); it("uses a recent Codex rate-limit snapshot when turn/start omits reset details", async () => { @@ -93,8 +103,8 @@ describe("runCodexAppServerAttempt usage limits", () => { [authProfileId]: { type: "oauth", provider: "openai", - access: "access", - refresh: "refresh", + access: "placeholder", + refresh: "placeholder", expires: Date.now() + 60_000, }, }, @@ -105,8 +115,9 @@ describe("runCodexAppServerAttempt usage limits", () => { const result = await run; expect(result.promptErrorSource).toBe("prompt"); - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Next reset in"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Next reset in"); expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined(); }); @@ -153,8 +164,8 @@ describe("runCodexAppServerAttempt usage limits", () => { [authProfileId]: { type: "oauth", provider: "openai", - access: "access", - refresh: "refresh", + access: "placeholder", + refresh: "placeholder", expires: Date.now() + 60_000, }, }, @@ -162,7 +173,7 @@ describe("runCodexAppServerAttempt usage limits", () => { const result = await runCodexAppServerAttempt(params); - expect(result.promptError).toContain("Next reset in"); + expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in"); expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined(); }); @@ -187,15 +198,17 @@ describe("runCodexAppServerAttempt usage limits", () => { const result = await run; expect(result.promptErrorSource).toBe("prompt"); - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Next reset in"); - expect(result.promptError).not.toContain("Codex did not return a reset time"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Next reset in"); + expect(promptError.message).not.toContain("Codex did not return a reset time"); }); it("refreshes Codex account rate limits when a failed turn omits reset details", async () => { const sessionFile = path.join(tempDir, "session.jsonl"); const workspaceDir = path.join(tempDir, "workspace"); const resetsAt = Math.ceil(Date.now() / 1000) + 120; + const authProfileId = "openai:work"; const harness = createStartedThreadHarness(async (method) => { if (method === "account/rateLimits/read") { return rateLimitsUpdated(resetsAt).params; @@ -203,7 +216,23 @@ describe("runCodexAppServerAttempt usage limits", () => { return undefined; }); - const run = runCodexAppServerAttempt(createParams(sessionFile, workspaceDir)); + const params = createParams(sessionFile, workspaceDir); + params.agentDir = path.join(tempDir, "streamed-usage-limit-agent"); + params.authProfileId = authProfileId; + params.authProfileStore = { + version: 1, + profiles: { + [authProfileId]: { + type: "oauth", + provider: "openai", + access: "placeholder", + refresh: "placeholder", + expires: Date.now() + 60_000, + }, + }, + }; + saveAuthProfileStore(params.authProfileStore, params.agentDir); + const run = runCodexAppServerAttempt(params); await harness.waitForMethod("turn/start"); await harness.notify({ method: "turn/completed", @@ -223,11 +252,114 @@ describe("runCodexAppServerAttempt usage limits", () => { const result = await run; - expect(result.promptError).toContain("You've reached your Codex subscription usage limit."); - expect(result.promptError).toContain("Next reset in"); - expect(result.promptError).not.toContain("Codex did not return a reset time"); + const promptError = expectUsageLimitPromptError(result.promptError); + expect(promptError.message).toContain("You've reached your Codex subscription usage limit."); + expect(promptError.message).toContain("Next reset in"); + expect(promptError.message).not.toContain("Codex did not return a reset time"); + expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBe(resetsAt * 1000); expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe( true, ); }); + + it("blocks after a streamed usage-limit failure with trusted in-turn limits", async () => { + const sessionFile = path.join(tempDir, "session.jsonl"); + const workspaceDir = path.join(tempDir, "workspace"); + const resetsAt = Math.ceil(Date.now() / 1000) + 120; + const authProfileId = "openai:work"; + const harness = createStartedThreadHarness(async () => undefined); + const params = createParams(sessionFile, workspaceDir); + params.agentDir = path.join(tempDir, "trusted-streamed-usage-limit-agent"); + params.authProfileId = authProfileId; + params.authProfileStore = { + version: 1, + profiles: { + [authProfileId]: { + type: "oauth", + provider: "openai", + access: "placeholder", + refresh: "placeholder", + expires: Date.now() + 60_000, + }, + }, + }; + saveAuthProfileStore(params.authProfileStore, params.agentDir); + + const run = runCodexAppServerAttempt(params); + await harness.waitForMethod("turn/start"); + await harness.notify(rateLimitsUpdated(resetsAt)); + await harness.notify({ + method: "turn/completed", + params: { + threadId: "thread-1", + turnId: "turn-1", + turn: { + id: "turn-1", + status: "failed", + error: { + message: "You've reached your usage limit.", + codexErrorInfo: "usageLimitExceeded", + }, + }, + }, + }); + + const result = await run; + + expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in"); + expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBe(resetsAt * 1000); + expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe( + false, + ); + }); + + it("does not block after a streamed usage-limit failure with only stale limits", async () => { + const sessionFile = path.join(tempDir, "session.jsonl"); + const workspaceDir = path.join(tempDir, "workspace"); + const resetsAt = Math.ceil(Date.now() / 1000) + 120; + const authProfileId = "openai:work"; + const harness = createStartedThreadHarness(async () => undefined); + rememberCodexRateLimitsRead(harness.client, rateLimitsUpdated(resetsAt).params); + const params = createParams(sessionFile, workspaceDir); + params.agentDir = path.join(tempDir, "stale-streamed-usage-limit-agent"); + params.authProfileId = authProfileId; + params.authProfileStore = { + version: 1, + profiles: { + [authProfileId]: { + type: "oauth", + provider: "openai", + access: "placeholder", + refresh: "placeholder", + expires: Date.now() + 60_000, + }, + }, + }; + + const run = runCodexAppServerAttempt(params); + await harness.waitForMethod("turn/start"); + await harness.notify({ + method: "turn/completed", + params: { + threadId: "thread-1", + turnId: "turn-1", + turn: { + id: "turn-1", + status: "failed", + error: { + message: "You've reached your usage limit.", + codexErrorInfo: "usageLimitExceeded", + }, + }, + }, + }); + + const result = await run; + + expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in"); + expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined(); + expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe( + false, + ); + }); }); diff --git a/extensions/codex/src/app-server/shared-client.test.ts b/extensions/codex/src/app-server/shared-client.test.ts index 13fc1f683171..2d6cec025045 100644 --- a/extensions/codex/src/app-server/shared-client.test.ts +++ b/extensions/codex/src/app-server/shared-client.test.ts @@ -1414,12 +1414,18 @@ describe("shared Codex app-server client", () => { .mockReturnValueOnce("api-key:first") .mockReturnValueOnce("api-key:second"); - const firstList = listCodexAppServerModels({ timeoutMs: 1000 }); + const firstList = listCodexAppServerModels({ + timeoutMs: 1000, + authRequirement: "api-key", + }); await sendInitializeResult(first, "openclaw/0.143.0 (macOS; test)"); await sendEmptyModelList(first); await expect(firstList).resolves.toEqual({ models: [] }); - const secondList = listCodexAppServerModels({ timeoutMs: 1000 }); + const secondList = listCodexAppServerModels({ + timeoutMs: 1000, + authRequirement: "api-key", + }); await sendInitializeResult(second, "openclaw/0.143.0 (macOS; test)"); await sendEmptyModelList(second); await expect(secondList).resolves.toEqual({ models: [] }); @@ -1429,6 +1435,49 @@ describe("shared Codex app-server client", () => { expect(second.process.stdin.destroyed).toBe(false); }); + it("does not share a client across auth requirements", async () => { + const first = createClientHarness(); + const second = createClientHarness(); + const startSpy = vi + .spyOn(CodexAppServerClient, "start") + .mockReturnValueOnce(first.client) + .mockReturnValueOnce(second.client); + + const firstList = listCodexAppServerModels({ + timeoutMs: 1000, + authProfileId: "openai:work", + authRequirement: "api-key", + }); + await sendInitializeResult(first, "openclaw/0.143.0 (macOS; test)"); + await sendEmptyModelList(first); + await expect(firstList).resolves.toEqual({ models: [] }); + + const secondList = listCodexAppServerModels({ + timeoutMs: 1000, + authProfileId: "openai:work", + authRequirement: "subscription", + }); + await sendInitializeResult(second, "openclaw/0.143.0 (macOS; test)"); + await sendEmptyModelList(second); + await expect(secondList).resolves.toEqual({ models: [] }); + + expect(startSpy).toHaveBeenCalledTimes(2); + expect(first.process.stdin.destroyed).toBe(false); + expect(second.process.stdin.destroyed).toBe(false); + }); + + it("rejects prepared auth that conflicts with the auth requirement", async () => { + const startSpy = vi.spyOn(CodexAppServerClient, "start"); + + await expect( + getSharedCodexAppServerClient({ + authRequirement: "subscription", + preparedAuth: { kind: "api-key", apiKey: "placeholder" }, + }), + ).rejects.toThrow("Prepared Codex auth does not satisfy the requested auth requirement."); + expect(startSpy).not.toHaveBeenCalled(); + }); + it("does not let one shared-client failure tear down another keyed client", async () => { const first = createClientHarness(); const second = createClientHarness(); diff --git a/extensions/codex/src/app-server/shared-client.ts b/extensions/codex/src/app-server/shared-client.ts index a2d58fd82110..9d566a8c1807 100644 --- a/extensions/codex/src/app-server/shared-client.ts +++ b/extensions/codex/src/app-server/shared-client.ts @@ -17,6 +17,7 @@ import { resolveCodexAppServerPreparedAuthProfileSnapshot, resolveCodexAppServerPreparedApiKeyCacheKey, type CodexAppServerPreparedAuth, + type CodexAppServerAuthRequirement, type CodexAppServerResolvedPreparedAuth, } from "./auth-bridge.js"; import { ensureCodexAppServerClientRuntime } from "./client-runtime.js"; @@ -246,6 +247,7 @@ export type CodexAppServerClientOptions = { /** Previously minted exact runtime required before the process may start. */ expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding; preparedAuth?: CodexAppServerPreparedAuth; + authRequirement?: CodexAppServerAuthRequirement; agentDir?: string; config?: Parameters[0]["config"]; onStartedClient?: (client: CodexAppServerClient) => void; @@ -263,10 +265,20 @@ type ResolvedCodexAppServerClientStartContext = { authProfileId: string | undefined; authProfileStore: AuthProfileStore | undefined; preparedAuth: CodexAppServerResolvedPreparedAuth | undefined; + authRequirement: CodexAppServerAuthRequirement | undefined; requestedStartOptions: CodexAppServerStartOptions; startOptions: CodexAppServerStartOptions; }; +function inferAuthRequirement( + preparedAuth: CodexAppServerPreparedAuth | undefined, +): CodexAppServerAuthRequirement | undefined { + if (preparedAuth?.kind === "api-key") { + return "api-key"; + } + return preparedAuth?.kind === "profile" ? "subscription" : undefined; +} + async function resolveCodexAppServerClientStartContext( options?: CodexAppServerClientOptions, ): Promise { @@ -287,6 +299,15 @@ async function resolveCodexAppServerClientStartContext( if (preparedAuth && requestedStartOptions.homeScope === "user") { throw new Error("Prepared Codex auth requires an isolated app-server home."); } + const preparedAuthRequirement = inferAuthRequirement(preparedAuth); + if ( + options?.authRequirement && + preparedAuthRequirement && + options.authRequirement !== preparedAuthRequirement + ) { + throw new Error("Prepared Codex auth does not satisfy the requested auth requirement."); + } + const authRequirement = options?.authRequirement ?? preparedAuthRequirement; const usesNativeAuth = !preparedAuth && (options?.authProfileId === null || requestedStartOptions.homeScope === "user"); @@ -361,6 +382,7 @@ async function resolveCodexAppServerClientStartContext( authProfileStore, requestedStartOptions, preparedAuth: resolvedPreparedAuth, + authRequirement, startOptions, }; } @@ -500,6 +522,7 @@ async function acquireSharedCodexAppServerClient( authProfileId, authProfileStore, preparedAuth, + authRequirement, requestedStartOptions, startOptions, } = context; @@ -508,15 +531,15 @@ async function acquireSharedCodexAppServerClient( preparedAuth?.kind === "api-key" ? resolveCodexAppServerPreparedApiKeyCacheKey(preparedAuth.apiKey) : (preparedAuth?.snapshot.secretFreeCacheKey ?? - (authProfileId - ? undefined - : resolveCodexAppServerFallbackApiKeyCacheKey({ startOptions }))); - const baseKey = codexAppServerStartOptionsKey(startOptions, { + (authRequirement === "api-key" && !authProfileId + ? resolveCodexAppServerFallbackApiKeyCacheKey({ startOptions }) + : undefined)); + const baseKey = `${codexAppServerStartOptionsKey(startOptions, { authProfileId, authBindingFingerprint: options?.authBindingFingerprint, agentDir: usesNativeAuth ? undefined : agentDir, fallbackApiKeyCacheKey: authIdentityCacheKey, - }); + })}\0auth-requirement:${authRequirement ?? "native"}`; // Capture turns cannot inherit a normal client whose loaded bytes predate the // filesystem snapshot. Keep their physical process generation separate. const runtimeArtifactMode = @@ -576,6 +599,7 @@ async function acquireSharedCodexAppServerClient( authProfileId: usesNativeAuth || preparedAuth?.kind === "api-key" ? null : authProfileId, authProfileStore, preparedAuth, + authRequirement, runtimeArtifactMode, ...(options?.expectedRuntimeArtifact ? { expectedRuntimeArtifact: options.expectedRuntimeArtifact } @@ -665,6 +689,7 @@ function createSharedCodexAppServerClientStartup(params: { expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding; runtimeArtifactSignal?: AbortSignal; preparedAuth?: CodexAppServerResolvedPreparedAuth; + authRequirement?: CodexAppServerAuthRequirement; config?: CodexAppServerClientOptions["config"]; }): SharedCodexAppServerClientStartup { const initialized = createDeferred(); @@ -675,6 +700,7 @@ function createSharedCodexAppServerClientStartup(params: { authProfileId: params.authProfileId, authProfileStore: params.authProfileStore, preparedAuth: params.preparedAuth, + authRequirement: params.authRequirement, runtimeArtifactMode: params.runtimeArtifactMode, ...(params.expectedRuntimeArtifact ? { expectedRuntimeArtifact: params.expectedRuntimeArtifact } @@ -722,6 +748,7 @@ export async function createIsolatedCodexAppServerClient( authProfileId, authProfileStore, preparedAuth, + authRequirement, requestedStartOptions, startOptions, } = await withCodexAppServerAcquireDeadline( @@ -736,6 +763,7 @@ export async function createIsolatedCodexAppServerClient( authProfileId: usesNativeAuth || preparedAuth?.kind === "api-key" ? null : authProfileId, authProfileStore, preparedAuth, + authRequirement, runtimeArtifactMode: options?.runtimeArtifactMode ?? (options?.expectedRuntimeArtifact ? "capture" : undefined), ...(options?.expectedRuntimeArtifact @@ -759,6 +787,7 @@ async function startInitializedCodexAppServerClient(params: { expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding; runtimeArtifactSignal?: AbortSignal; preparedAuth?: CodexAppServerResolvedPreparedAuth; + authRequirement?: CodexAppServerAuthRequirement; config?: CodexAppServerClientOptions["config"]; timeoutMs?: number; abandonSignal?: AbortSignal; @@ -859,6 +888,7 @@ async function startInitializedCodexAppServerClient(params: { agentDir: params.agentDir, authProfileId: params.authProfileId, preparedAuth: params.preparedAuth, + authRequirement: params.authRequirement, startOptions, config: params.config, ...(params.authProfileStore ? { authProfileStore: params.authProfileStore } : {}), diff --git a/extensions/codex/src/app-server/side-question.ts b/extensions/codex/src/app-server/side-question.ts index 68c25cd3122a..bff134024af4 100644 --- a/extensions/codex/src/app-server/side-question.ts +++ b/extensions/codex/src/app-server/side-question.ts @@ -288,6 +288,7 @@ export async function runCodexAppServerSideQuestion( const clientOptions = { startOptions: appServer.start, timeoutMs: appServer.requestTimeoutMs, + authRequirement: preparedRuntimeAuth.plan.modelRoute?.authRequirement, ...(startupPreparedAuth ? { preparedAuth: startupPreparedAuth } : { authProfileId: connection.clientAuthProfileId }), diff --git a/extensions/codex/src/app-server/usage-limit-error.ts b/extensions/codex/src/app-server/usage-limit-error.ts index c038ace08b89..44bab51b6dad 100644 --- a/extensions/codex/src/app-server/usage-limit-error.ts +++ b/extensions/codex/src/app-server/usage-limit-error.ts @@ -41,6 +41,14 @@ type CodexUsageLimitErrorResult = { rateLimitsForProfile?: JsonValue; }; +export function createCodexUsageLimitPromptError(message: string): Error & { status: 429 } { + return Object.assign(new Error(message), { status: 429 as const }); +} + +export function isCodexUsageLimitPromptError(error: unknown): error is Error & { status: 429 } { + return error instanceof Error && "status" in error && error.status === 429; +} + /** Marks a Codex auth profile blocked until the reset time advertised by rate limits. */ export async function markCodexAuthProfileBlockedFromRateLimits(params: { params: EmbeddedRunAttemptParams; @@ -101,22 +109,20 @@ export async function refreshCodexUsageLimitPromptError(params: { message: string | undefined; timeoutMs?: number; signal?: AbortSignal; -}): Promise { +}): Promise { if (!shouldRefreshCodexRateLimitsForUsageLimitMessage(params.message)) { return undefined; } - return ( - await refreshCodexUsageLimitError({ - client: params.client, - source: { - message: params.message, - codexErrorInfo: "usageLimitExceeded", - rateLimits: readRecentCodexRateLimits(params.client), - }, - timeoutMs: params.timeoutMs, - signal: params.signal, - }) - )?.message; + return refreshCodexUsageLimitError({ + client: params.client, + source: { + message: params.message, + codexErrorInfo: "usageLimitExceeded", + rateLimits: readRecentCodexRateLimits(params.client), + }, + timeoutMs: params.timeoutMs, + signal: params.signal, + }); } async function refreshCodexUsageLimitError(params: { diff --git a/src/agents/auth-profiles/order.test.ts b/src/agents/auth-profiles/order.test.ts index d7714822adba..1469d01ce43e 100644 --- a/src/agents/auth-profiles/order.test.ts +++ b/src/agents/auth-profiles/order.test.ts @@ -368,6 +368,56 @@ describe("resolveAuthProfileOrder", () => { ).toStrictEqual(["fixture-provider:backup", "fixture-provider:primary"]); }); + it("does not apply a block scoped to another model when ordering profiles", () => { + const store: AuthProfileStore = { + version: 1, + profiles: { + "fixture-provider:primary": { + type: "api_key", + provider: "fixture-provider", + key: "placeholder", + }, + "fixture-provider:backup": { + type: "api_key", + provider: "fixture-provider", + key: "placeholder", + }, + }, + usageStats: { + "fixture-provider:primary": { + blockedUntil: Date.now() + 60_000, + blockedReason: "subscription_limit", + blockedModel: "model-a", + blockedScope: "model", + }, + }, + }; + const cfg = { + auth: { + order: { + "fixture-provider": ["fixture-provider:primary", "fixture-provider:backup"], + }, + }, + } satisfies OpenClawConfig; + + expect( + resolveAuthProfileOrder({ + cfg, + store, + provider: "fixture-provider", + forModel: "model-b", + }), + ).toStrictEqual(["fixture-provider:primary", "fixture-provider:backup"]); + expect( + resolveAuthProfileOrder({ + cfg, + store, + provider: "fixture-provider", + forModel: "model-a", + }), + ).toStrictEqual(["fixture-provider:backup", "fixture-provider:primary"]); + }); + it("keeps unresolved OAuth refs only in read-only profile ordering", () => { const store: AuthProfileStore = { version: 1, diff --git a/src/agents/auth-profiles/order.ts b/src/agents/auth-profiles/order.ts index 4dbc95c09e74..d1845946f7a6 100644 --- a/src/agents/auth-profiles/order.ts +++ b/src/agents/auth-profiles/order.ts @@ -374,7 +374,7 @@ export function resolveAuthProfileOrderWithMetadata( for (const profileId of deduped) { if (isProfileInCooldown(store, profileId, now, forModel)) { const cooldownUntil = - resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}) ?? now; + resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}, forModel) ?? now; inCooldown.push({ profileId, cooldownUntil }); } else { available.push(profileId); @@ -493,7 +493,8 @@ function orderProfilesByMode( const cooldownSorted = inCooldown .map((profileId) => ({ profileId, - cooldownUntil: resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}) ?? now, + cooldownUntil: + resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}, forModel) ?? now, })) .toSorted((a, b) => a.cooldownUntil - b.cooldownUntil) .map((entry) => entry.profileId); diff --git a/src/agents/auth-profiles/state.ts b/src/agents/auth-profiles/state.ts index 42153e2a238a..f246840fdb15 100644 --- a/src/agents/auth-profiles/state.ts +++ b/src/agents/auth-profiles/state.ts @@ -118,6 +118,7 @@ function normalizeUsageStatsEntry(raw: unknown): ProfileUsageStats | undefined { blockedReason: normalizeEnumValue(raw.blockedReason, AUTH_BLOCKED_REASONS), blockedSource: normalizeEnumValue(raw.blockedSource, AUTH_BLOCKED_SOURCES), blockedModel: normalizeOptionalString(raw.blockedModel), + blockedScope: raw.blockedScope === "model" ? "model" : undefined, cooldownUntil: normalizeFiniteNumber(raw.cooldownUntil), cooldownReason: normalizeEnumValue(raw.cooldownReason, AUTH_FAILURE_REASONS), cooldownModel: normalizeOptionalString(raw.cooldownModel), diff --git a/src/agents/auth-profiles/types.ts b/src/agents/auth-profiles/types.ts index 75ea45c1feb3..281072c9e037 100644 --- a/src/agents/auth-profiles/types.ts +++ b/src/agents/auth-profiles/types.ts @@ -106,6 +106,7 @@ export type ProfileUsageStats = { blockedReason?: AuthProfileBlockedReason; blockedSource?: AuthProfileBlockedSource; blockedModel?: string; + blockedScope?: "model"; cooldownUntil?: number; cooldownReason?: AuthProfileFailureReason; cooldownModel?: string; diff --git a/src/agents/auth-profiles/usage-state.ts b/src/agents/auth-profiles/usage-state.ts index 11485c4aac4d..4c4b429c6b80 100644 --- a/src/agents/auth-profiles/usage-state.ts +++ b/src/agents/auth-profiles/usage-state.ts @@ -23,9 +23,16 @@ export function isModelScopedCooldownReason(reason: AuthProfileFailureReason | u /** Resolves the latest active blocked/cooldown/disabled timestamp for a profile. */ export function resolveProfileUnusableUntil( - stats: Pick, + stats: Pick< + ProfileUsageStats, + "blockedUntil" | "blockedModel" | "blockedScope" | "cooldownUntil" | "disabledUntil" + >, + forModel?: string, ): number | null { - const values = [stats.blockedUntil, stats.cooldownUntil, stats.disabledUntil] + const blockedUntil = isBlockScopedToDifferentModel(stats, forModel) + ? undefined + : stats.blockedUntil; + const values = [blockedUntil, stats.cooldownUntil, stats.disabledUntil] .map((value) => asDateTimestampMs(value)) .filter((value): value is number => value !== undefined && value > 0); if (values.length === 0) { @@ -40,10 +47,40 @@ export function isActiveUnusableWindow(until: number | undefined, now: number): return timestamp !== undefined && timestamp > 0 && now < timestamp; } +function isBlockedWindowActiveForModel( + stats: Pick, + now: number, + forModel?: string, +): boolean { + return ( + !isBlockScopedToDifferentModel(stats, forModel) && + isActiveUnusableWindow(stats.blockedUntil, now) + ); +} + +function isBlockScopedToDifferentModel( + stats: Pick, + forModel?: string, +): boolean { + // Legacy rows carried blockedModel for profile-wide blocks without a scope marker. + // Only explicit model scope narrows them; unmarked rows stay wide until expiry. + return Boolean( + forModel && + stats.blockedScope === "model" && + stats.blockedModel && + stats.blockedModel !== forModel, + ); +} + function shouldBypassModelScopedCooldown( stats: Pick< ProfileUsageStats, - "blockedUntil" | "cooldownReason" | "cooldownModel" | "disabledUntil" + | "blockedUntil" + | "blockedModel" + | "blockedScope" + | "cooldownReason" + | "cooldownModel" + | "disabledUntil" >, now: number, forModel?: string, @@ -53,7 +90,7 @@ function shouldBypassModelScopedCooldown( isModelScopedCooldownReason(stats.cooldownReason) && stats.cooldownModel && stats.cooldownModel !== forModel && - !isActiveUnusableWindow(stats.blockedUntil, now) && + !isBlockedWindowActiveForModel(stats, now, forModel) && !isActiveUnusableWindow(stats.disabledUntil, now), ); } @@ -82,7 +119,7 @@ export function isProfileInCooldown( if (shouldBypassModelScopedCooldown(stats, ts, forModel)) { return false; } - const unusableUntil = resolveProfileUnusableUntil(stats); + const unusableUntil = resolveProfileUnusableUntil(stats, forModel); return unusableUntil ? ts < unusableUntil : false; } @@ -107,7 +144,7 @@ export function getSoonestCooldownExpiry( if (shouldBypassModelScopedCooldown(stats, ts, options?.forModel)) { continue; } - const until = resolveProfileUnusableUntil(stats); + const until = resolveProfileUnusableUntil(stats, options?.forModel); if (typeof until !== "number" || !Number.isFinite(until) || until <= 0) { continue; } @@ -115,7 +152,7 @@ export function getSoonestCooldownExpiry( options?.forModel && stats.cooldownReason === "rate_limit" && stats.cooldownModel === options.forModel && - !isActiveUnusableWindow(stats.blockedUntil, ts) && + !isBlockedWindowActiveForModel(stats, ts, options.forModel) && !isActiveUnusableWindow(stats.disabledUntil, ts); if (matchingModelScopedCooldown) { latestMatchingModelCooldown = @@ -195,6 +232,7 @@ export function clearExpiredCooldowns(store: AuthProfileStore, now?: number): bo stats.blockedReason = undefined; stats.blockedSource = undefined; stats.blockedModel = undefined; + stats.blockedScope = undefined; profileMutated = true; } if (disabledExpired) { diff --git a/src/agents/auth-profiles/usage.test.ts b/src/agents/auth-profiles/usage.test.ts index 11cec1e2f561..1eb978c9c32b 100644 --- a/src/agents/auth-profiles/usage.test.ts +++ b/src/agents/auth-profiles/usage.test.ts @@ -83,6 +83,7 @@ function expectProfileErrorStateCleared( ) { expect(stats?.blockedUntil).toBeUndefined(); expect(stats?.blockedReason).toBeUndefined(); + expect(stats?.blockedScope).toBeUndefined(); expect(stats?.cooldownUntil).toBeUndefined(); expect(stats?.disabledUntil).toBeUndefined(); expect(stats?.disabledReason).toBeUndefined(); @@ -103,6 +104,18 @@ describe("resolveProfileUnusableUntil", () => { ).toBe(300); expect(resolveProfileUnusableUntil({ cooldownUntil: 300 })).toBe(300); }); + + it("keeps legacy blockedModel rows profile-wide", () => { + expect( + resolveProfileUnusableUntil({ blockedUntil: 300, blockedModel: "model-a" }, "model-b"), + ).toBe(300); + }); + + it("applies explicitly model-scoped blocks only to that model", () => { + const stats = { blockedUntil: 300, blockedModel: "model-a", blockedScope: "model" as const }; + expect(resolveProfileUnusableUntil(stats, "model-a")).toBe(300); + expect(resolveProfileUnusableUntil(stats, "model-b")).toBeNull(); + }); }); describe("resolveProfileUnusableUntilForDisplay", () => { @@ -285,17 +298,32 @@ describe("isProfileInCooldown", () => { expect(isProfileInCooldown(store, "github-copilot:github", undefined, "gpt-4.1")).toBe(true); }); - it("does not bypass model-scoped cooldown when blockedUntil is active", () => { + it("bypasses model-scoped blocks and cooldowns for sibling models", () => { const now = Date.now(); const store = makeStore({ "google:default": { blockedUntil: now + 120_000, blockedReason: "subscription_limit", + blockedModel: "gemini-3-flash-preview", + blockedScope: "model", cooldownUntil: now + 60_000, cooldownReason: "timeout", cooldownModel: "gemini-3-flash-preview", }, }); + expect(isProfileInCooldown(store, "google:default", now, "gemini-3-flash-preview")).toBe(true); + expect(isProfileInCooldown(store, "google:default", now, "gemini-3.1-flash-lite")).toBe(false); + }); + + it("keeps legacy blockedModel rows active for sibling models", () => { + const now = Date.now(); + const store = makeStore({ + "google:default": { + blockedUntil: now + 120_000, + blockedModel: "gemini-3-flash-preview", + }, + }); + expect(isProfileInCooldown(store, "google:default", now, "gemini-3.1-flash-lite")).toBe(true); }); }); @@ -917,6 +945,89 @@ describe("markAuthProfileFailure — active windows do not extend on retry", () }); describe("markAuthProfileBlockedUntil", () => { + it("keeps repeated same-model blocks scoped to that model", async () => { + const now = Date.parse("2026-05-30T18:00:00.000Z"); + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now); + const store = makeStore({ + "openai:default": { + blockedUntil: now + 60_000, + blockedModel: "gpt-5.4", + blockedScope: "model", + }, + }); + mockLockedUpdateForStore(store); + try { + await markAuthProfileBlockedUntil({ + store, + profileId: "openai:default", + blockedUntil: now + 120_000, + source: "codex_rate_limits", + modelId: "gpt-5.4", + }); + } finally { + nowSpy.mockRestore(); + } + + expect(store.usageStats?.["openai:default"]?.blockedModel).toBe("gpt-5.4"); + expect(store.usageStats?.["openai:default"]?.blockedScope).toBe("model"); + expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4")).toBe(true); + expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(false); + }); + + it("widens an active block after a different model fails", async () => { + const now = Date.parse("2026-05-30T18:00:00.000Z"); + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now); + const store = makeStore({ + "openai:default": { + blockedUntil: now + 60_000, + blockedModel: "gpt-5.4", + blockedScope: "model", + }, + }); + mockLockedUpdateForStore(store); + try { + await markAuthProfileBlockedUntil({ + store, + profileId: "openai:default", + blockedUntil: now + 120_000, + source: "codex_rate_limits", + modelId: "gpt-5.4-mini", + }); + } finally { + nowSpy.mockRestore(); + } + + expect(store.usageStats?.["openai:default"]?.blockedModel).toBeUndefined(); + expect(store.usageStats?.["openai:default"]?.blockedScope).toBeUndefined(); + expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(true); + }); + + it("never narrows an active profile-wide block", async () => { + const now = Date.parse("2026-05-30T18:00:00.000Z"); + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now); + const store = makeStore({ + "openai:default": { + blockedUntil: now + 60_000, + }, + }); + mockLockedUpdateForStore(store); + try { + await markAuthProfileBlockedUntil({ + store, + profileId: "openai:default", + blockedUntil: now + 120_000, + source: "codex_rate_limits", + modelId: "gpt-5.4", + }); + } finally { + nowSpy.mockRestore(); + } + + expect(store.usageStats?.["openai:default"]?.blockedModel).toBeUndefined(); + expect(store.usageStats?.["openai:default"]?.blockedScope).toBeUndefined(); + expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(true); + }); + it("keeps a later active blocked-until timestamp", async () => { const nowSpy = vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-05-30T18:00:00.000Z")); const laterBlockedUntil = Date.parse("2031-01-01T00:00:00.000Z"); diff --git a/src/agents/auth-profiles/usage.ts b/src/agents/auth-profiles/usage.ts index 00e232376253..7afb5c29a8c4 100644 --- a/src/agents/auth-profiles/usage.ts +++ b/src/agents/auth-profiles/usage.ts @@ -224,6 +224,7 @@ function applyWhamCooldownResult(params: { blockedReason: "subscription_limit", blockedSource: params.whamResult.blockedSource ?? "wham", blockedModel: undefined, + blockedScope: undefined, cooldownUntil: undefined, cooldownReason: undefined, cooldownModel: undefined, @@ -586,6 +587,7 @@ function resetUsageStats( blockedReason: undefined, blockedSource: undefined, blockedModel: undefined, + blockedScope: undefined, cooldownUntil: undefined, cooldownReason: undefined, cooldownModel: undefined, @@ -842,12 +844,23 @@ function buildBlockedProfileUsageStats(params: { params.previousStats?.blockedUntil, params.now, ); + // One active block can stay model-scoped only while every observation names + // that same model. Mixed or unknown observations widen the profile. + const blockedModel = + activeBlockedUntil === 0 + ? params.modelId + : params.previousStats?.blockedScope === "model" && + params.previousStats.blockedModel === params.modelId && + params.modelId + ? params.modelId + : undefined; return { ...params.previousStats, blockedUntil: Math.max(activeBlockedUntil, params.blockedUntil), blockedReason: "subscription_limit", blockedSource: params.source, - blockedModel: params.modelId, + blockedModel, + blockedScope: blockedModel ? "model" : undefined, cooldownUntil: undefined, cooldownReason: undefined, cooldownModel: undefined, diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts index 144ca4e577c1..25f3d7c17822 100644 --- a/src/agents/failover-error.test.ts +++ b/src/agents/failover-error.test.ts @@ -995,6 +995,18 @@ describe("failover-error", () => { expect(coerceToFailoverError(err)?.status).toBe(429); }); + it("classifies a structured prompt error independently of its wording", () => { + const promptError = Object.assign(new Error("quota exhausted"), { status: 429 as const }); + const failoverError = coerceToFailoverError(promptError, { + provider: "openai", + model: "gpt-5.4", + }); + + expect(failoverError?.reason).toBe("rate_limit"); + expect(failoverError?.status).toBe(429); + expect(failoverError?.message).toBe("quota exhausted"); + }); + it("lets wrapped causes override parent context-overflow classifications", () => { const err = new Error("INVALID_ARGUMENT: input exceeds the maximum number of tokens", { cause: { code: "RESOURCE_EXHAUSTED" }, diff --git a/src/agents/model-fallback.run-embedded.e2e.test.ts b/src/agents/model-fallback.run-embedded.e2e.test.ts index 9770b775207d..4451f1fe1f55 100644 --- a/src/agents/model-fallback.run-embedded.e2e.test.ts +++ b/src/agents/model-fallback.run-embedded.e2e.test.ts @@ -210,20 +210,26 @@ function expectFailureCount( expect(failureCounts?.[reason]).toBe(expected); } -async function writeMultiProfileAuthStore(agentDir: string) { +async function writeMultiProfileAuthStore( + agentDir: string, + options?: { openAiProfileCount?: 2 | 3 }, +) { + const includeThirdOpenAiProfile = options?.openAiProfileCount !== 2; saveAuthProfileStore( { version: 1, profiles: { "openai:p1": { type: "api_key", provider: "openai", key: "sk-openai-1" }, "openai:p2": { type: "api_key", provider: "openai", key: "sk-openai-2" }, - "openai:p3": { type: "api_key", provider: "openai", key: "sk-openai-3" }, + ...(includeThirdOpenAiProfile + ? { "openai:p3": { type: "api_key" as const, provider: "openai", key: "placeholder" } } + : {}), "groq:p1": { type: "api_key", provider: "groq", key: "sk-groq" }, }, usageStats: { "openai:p1": { lastUsed: 1 }, "openai:p2": { lastUsed: 2 }, - "openai:p3": { lastUsed: 3 }, + ...(includeThirdOpenAiProfile ? { "openai:p3": { lastUsed: 3 } } : {}), "groq:p1": { lastUsed: 4 }, }, }, @@ -983,6 +989,41 @@ describe("runWithModelFallback + runEmbeddedAgent failover behavior", () => { }); }); + it("rotates Codex profiles on structured prompt rate limits before model fallback", async () => { + await withAgentWorkspace(async ({ agentDir, workspaceDir }) => { + await writeMultiProfileAuthStore(agentDir, { openAiProfileCount: 2 }); + mockPrimaryFailureThenFallbackSuccess(() => { + return makeEmbeddedRunnerAttempt({ + promptError: Object.assign( + new Error("You've reached your Codex subscription usage limit."), + { status: 429 as const }, + ), + promptErrorSource: "prompt", + }); + }); + + const result = await runEmbeddedFallback({ + agentDir, + workspaceDir, + sessionKey: "agent:test:codex-structured-prompt-rate-limit", + runId: "run:codex-structured-prompt-rate-limit", + }); + + expect(result.provider).toBe("groq"); + expect(result.model).toBe("mock-2"); + expect(result.attempts[0]?.reason).toBe("rate_limit"); + expectProviderAttemptCounts({ openai: 2, groq: 1 }); + const primaryCalls = runEmbeddedAttemptMock.mock.calls + .map(([params]) => params as EmbeddedAttemptParams) + .filter((params) => params.provider === "openai"); + expect(primaryCalls.map((params) => params.authProfileId)).toStrictEqual([ + "openai:p1", + "openai:p2", + ]); + expect(primaryCalls.map((params) => params.modelId)).toStrictEqual(["mock-1", "mock-1"]); + }); + }); + it("respects prompt-side rateLimitedProfileRotations=0 and falls back immediately", async () => { await withAgentWorkspace(async ({ agentDir, workspaceDir }) => { await writeMultiProfileAuthStore(agentDir);