fix(codex): model-scoped usage-limit blocks, structural 429 classification, no silent API-key billing (#108254)

* test(codex): use allowlisted placeholder for auth-bridge api-key fixture

* fix(codex): model-scoped usage-limit blocks, structural 429 classification, no silent API-key billing

- usage-limit blocks written from Codex rate-limit resets are model-scoped via
  a persisted blockedScope marker: healthy sibling models on the same auth
  profile stay usable; a different/unknown model failing widens the block
  profile-wide and never narrows back; legacy rows without the marker stay
  profile-wide until they expire (#100556)
- Codex usage-limit failures now surface as status-429 Error objects at both
  ingress paths (turn-start and streamed turn failure, wrapped at the event
  projector), so core failover classification is structural instead of
  matching message wording; profile blocking uses only rate-limit data whose
  revision advanced during the turn
- usage-limit detection requires the structured codexErrorInfo signal; the
  over-broad "usage limit" substring match that misclassified unrelated
  errors as subscription limits is gone (#96815)
- subscription/OAuth routes can no longer silently fall back to env/auth.json
  API keys: ambient key fallback is restricted to explicit api-key routes,
  native-auth subscription routes verify the account is chatgpt-backed via
  account/read, and shared app-server clients are partitioned by auth
  requirement so pooled clients cannot cross billing modes (#106375)
- integrated e2e regression: usage-limit promptError -> same-model sibling
  profile rotation -> model fallback with reason rate_limit

* chore(codex): keep CodexUsageLimitErrorResult type local
This commit is contained in:
Peter Steinberger
2026-07-15 04:10:57 -07:00
committed by GitHub
parent 4e756adaaf
commit 1a27e7f3ec
27 changed files with 850 additions and 113 deletions
@@ -104,6 +104,7 @@ export function resolveCodexAppServerReplayBlockedReason(
export function buildCodexTurnStartFailureResult(params: {
params: EmbeddedRunAttemptParams;
message: string;
promptError?: unknown;
messagesSnapshot: AgentMessage[];
systemPromptReport: CodexSystemPromptReport;
}): EmbeddedRunAttemptResult {
@@ -114,7 +115,7 @@ export function buildCodexTurnStartFailureResult(params: {
idleTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
promptError: params.message,
promptError: params.promptError ?? params.message,
promptErrorSource: "prompt",
sessionIdUsed: params.params.sessionId,
messagesSnapshot: params.messagesSnapshot,
@@ -125,6 +125,7 @@ export async function startCodexAttemptThread(params: {
pluginConfig: CodexPluginConfig;
computerUseConfig: ResolvedCodexComputerUseConfig;
startupAuthProfileId: string | null | undefined;
startupAuthRequirement?: CodexAppServerClientOptions["authRequirement"];
startupAuthBindingFingerprint: string | undefined;
runtimeArtifactRequest?: Readonly<{
expected?: AgentHarnessRuntimeArtifactBinding;
@@ -228,6 +229,7 @@ export async function startCodexAttemptThread(params: {
...(params.startupPreparedAuth
? { preparedAuth: params.startupPreparedAuth }
: { authProfileId: params.startupAuthProfileId }),
authRequirement: params.startupAuthRequirement,
authProfileStore: attemptParams.authProfileStore,
authBindingFingerprint: params.startupAuthBindingFingerprint,
...(params.runtimeArtifactRequest
@@ -1965,6 +1965,112 @@ describe("bridgeCodexAppServerStartOptions", () => {
}
});
it("fails subscription auth instead of falling back to an API key", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-app-server-"));
const request = vi.fn(async () => ({ type: "apiKey" }));
vi.stubEnv("CODEX_API_KEY", "placeholder");
let rejection: unknown;
try {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authProfileId: "openai:work",
authProfileStore: {
version: 1,
profiles: {},
},
authRequirement: "subscription",
startOptions: createStartOptions({
env: { CODEX_API_KEY: "placeholder" },
}),
});
} catch (error) {
rejection = error;
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
expect(rejection).toBeInstanceOf(Error);
expect((rejection as { status?: unknown }).status).toBe(401);
expect(request).not.toHaveBeenCalled();
});
it("preserves transient subscription credential resolution errors", async () => {
const transientError = Object.assign(new Error("temporary refresh failure"), { status: 503 });
oauthMocks.refreshOpenAICodexToken.mockRejectedValueOnce(transientError);
const request = vi.fn();
await expect(
applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir: "/tmp/openclaw-agent",
authProfileId: "openai:work",
authProfileStore: {
version: 1,
profiles: {
"openai:work": {
type: "oauth",
provider: "openai",
access: "placeholder",
refresh: "placeholder",
expires: Date.now() - 60_000,
},
},
},
authRequirement: "subscription",
}),
).rejects.toBe(transientError);
expect(request).not.toHaveBeenCalled();
});
it("accepts native ChatGPT auth for subscription routes", async () => {
const request = vi.fn(async () => ({
account: { type: "chatgpt", email: null, planType: "plus" },
requiresOpenaiAuth: true,
}));
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir: "/tmp/openclaw-agent",
authProfileId: null,
authRequirement: "subscription",
});
expect(request).toHaveBeenCalledOnce();
expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false });
});
it("rejects native API-key auth for subscription routes", async () => {
const request = vi.fn(async () => ({
account: { type: "apiKey" },
requiresOpenaiAuth: false,
}));
await expect(
applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir: "/tmp/openclaw-agent",
authProfileId: null,
authRequirement: "subscription",
}),
).rejects.toMatchObject({ status: 401 });
expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false });
});
it("rejects missing native auth for subscription routes", async () => {
const request = vi.fn(async () => ({ account: null, requiresOpenaiAuth: true }));
await expect(
applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir: "/tmp/openclaw-agent",
authProfileId: null,
authRequirement: "subscription",
}),
).rejects.toMatchObject({ status: 401 });
expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false });
});
it("falls back to CODEX_API_KEY when no auth profile and no Codex account is available", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-app-server-"));
const request = vi.fn(async (method: string) => {
@@ -1979,15 +2085,16 @@ describe("bridgeCodexAppServerStartOptions", () => {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authRequirement: "api-key",
startOptions: createStartOptions({
env: { CODEX_API_KEY: "configured-codex-api-key" },
env: { CODEX_API_KEY: "test-token-placeholder" },
}),
});
expect(request).toHaveBeenNthCalledWith(1, "account/read", { refreshToken: false });
expect(request).toHaveBeenNthCalledWith(2, "account/login/start", {
type: "apiKey",
apiKey: "configured-codex-api-key",
apiKey: "test-token-placeholder",
});
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
@@ -2008,6 +2115,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authRequirement: "api-key",
startOptions: createStartOptions(),
});
@@ -2037,6 +2145,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authRequirement: "api-key",
startOptions: createStartOptions(),
});
@@ -2060,6 +2169,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authRequirement: "api-key",
startOptions: createStartOptions(),
});
@@ -2092,6 +2202,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authRequirement: "api-key",
startOptions: createStartOptions({
env: { CODEX_HOME: path.join(root, "isolated-codex-home") },
}),
@@ -2173,6 +2284,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authRequirement: "api-key",
startOptions: createStartOptions({
clearEnv: ["CODEX_API_KEY", "OPENAI_API_KEY"],
}),
@@ -2198,6 +2310,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
await applyCodexAppServerAuthProfile({
client: { request } as never,
agentDir,
authRequirement: "api-key",
startOptions: createStartOptions({
transport: "websocket",
url: "ws://127.0.0.1:1455",
+83 -17
View File
@@ -29,10 +29,11 @@ import {
resolveCodexComputerUseConfig,
type CodexAppServerStartOptions,
} from "./config.js";
import type {
CodexChatgptAuthTokensRefreshResponse,
CodexGetAccountResponse,
CodexLoginAccountParams,
import {
isJsonObject,
type CodexChatgptAuthTokensRefreshResponse,
type CodexGetAccountResponse,
type CodexLoginAccountParams,
} from "./protocol.js";
import { isCodexAppServerNativeAuthProfile } from "./session-binding.js";
import { resolveCodexAppServerSpawnEnv } from "./transport-stdio.js";
@@ -61,6 +62,7 @@ const CODEX_APP_SERVER_HOME_ENV_VARS = [CODEX_HOME_ENV_VAR, HOME_ENV_VAR];
const CODEX_AUTH_JSON_FILENAME = "auth.json";
const CODEX_HOME_DIRNAME = ".codex";
type AuthProfileOrderConfig = Parameters<typeof resolveAuthProfileOrder>[0]["cfg"];
export type CodexAppServerAuthRequirement = "api-key" | "subscription";
const scopedOAuthRefreshQueues = new WeakMap<
AuthProfileStore,
Map<string, Promise<OAuthCredential>>
@@ -250,7 +252,7 @@ export async function resolveCodexAppServerPreparedAuthProfileSnapshot(params: {
/** Maps one prepared route to one mutually exclusive app-server auth handoff. */
export async function resolveCodexAppServerPreparedAuthHandoff(params: {
authRequirement?: "api-key" | "subscription";
authRequirement?: CodexAppServerAuthRequirement;
resolvedApiKey?: string;
authProfileId?: string;
authProfileStore: AuthProfileStore;
@@ -281,7 +283,7 @@ export async function resolveCodexAppServerPreparedAuthHandoff(params: {
return { authProfileId, nativeAuthProfile };
}
if (!authProfileId || !nativeAuthProfile) {
throw new Error(params.subscriptionProfileRequiredError);
throw createCodexAppServerAuthError(params.subscriptionProfileRequiredError);
}
const snapshot = await resolveCodexAppServerPreparedAuthProfileSnapshot({
@@ -291,7 +293,7 @@ export async function resolveCodexAppServerPreparedAuthHandoff(params: {
config: params.config,
});
if (!snapshot) {
throw new Error(params.subscriptionProfileUnusableError);
throw createCodexAppServerAuthError(params.subscriptionProfileUnusableError);
}
return {
authProfileId,
@@ -474,6 +476,7 @@ export async function applyCodexAppServerAuthProfile(params: {
authProfileId?: string | null;
authProfileStore?: AuthProfileStore;
preparedAuth?: CodexAppServerResolvedPreparedAuth;
authRequirement?: CodexAppServerAuthRequirement;
startOptions?: CodexAppServerStartOptions;
config?: AuthProfileOrderConfig;
}): Promise<void> {
@@ -489,16 +492,52 @@ export async function applyCodexAppServerAuthProfile(params: {
return;
}
if (params.authProfileId === null) {
if (params.authRequirement === "subscription") {
const response = await params.client.request<CodexGetAccountResponse>("account/read", {
refreshToken: false,
});
if (!isJsonObject(response.account) || response.account.type !== "chatgpt") {
throw createCodexAppServerAuthError(
"Codex subscription auth profile could not produce login credentials.",
);
}
}
return;
}
const loginParams = await resolveCodexAppServerAuthProfileLoginParams({
agentDir: params.agentDir,
authProfileId: params.authProfileId,
authProfileStore: params.authProfileStore,
config: params.config,
});
let loginParams: CodexLoginAccountParams | undefined;
try {
loginParams = await resolveCodexAppServerAuthProfileLoginParams({
agentDir: params.agentDir,
authProfileId: params.authProfileId,
authProfileStore: params.authProfileStore,
config: params.config,
});
} catch (error) {
if (
params.authRequirement === "subscription" &&
error instanceof CodexAppServerAuthProfileUnavailableError
) {
throw createCodexAppServerAuthError(
"Codex subscription auth profile could not produce login credentials.",
error,
);
}
throw error;
}
if (params.authRequirement === "subscription" && loginParams?.type !== "chatgptAuthTokens") {
throw createCodexAppServerAuthError(
"Codex subscription auth profile could not produce login credentials.",
);
}
if (!loginParams) {
if (params.startOptions?.transport !== "stdio") {
// Observe native state only for explicit API-key routes. A subscription
// route must fail here so profile rotation can run before billing changes.
if (params.authRequirement === "subscription") {
throw createCodexAppServerAuthError(
"Codex subscription auth profile could not produce login credentials.",
);
}
if (params.authRequirement !== "api-key" || params.startOptions?.transport !== "stdio") {
return;
}
const env = resolveCodexAppServerSpawnEnv(params.startOptions, process.env);
@@ -515,13 +554,40 @@ export async function applyCodexAppServerAuthProfile(params: {
await params.client.request("account/login/start", loginParams);
}
function resolveCodexAppServerAuthProfileLoginParams(params: {
function createCodexAppServerAuthError(message: string, cause?: unknown): Error & { status: 401 } {
const error = cause === undefined ? new Error(message) : new Error(message, { cause });
return Object.assign(error, { status: 401 as const });
}
class CodexAppServerAuthProfileUnavailableError extends Error {}
async function resolveCodexAppServerAuthProfileLoginParams(params: {
agentDir: string;
authProfileId?: string;
authProfileStore?: AuthProfileStore;
config?: AuthProfileOrderConfig;
}): Promise<CodexLoginAccountParams | undefined> {
return resolveCodexAppServerAuthProfileLoginParamsInternal(params);
const store = resolveCodexAppServerAuthProfileStore(params);
const profileId = resolveCodexAppServerAuthProfileId({
authProfileId: params.authProfileId,
store,
config: params.config,
});
const profile = profileId ? store.profiles[profileId] : undefined;
if (profileId && !profile) {
throw new CodexAppServerAuthProfileUnavailableError(
`Codex app-server auth profile "${profileId}" was not found.`,
);
}
if (profileId && profile && !isCodexAppServerAuthProfileCredential(profile, params.config)) {
throw new CodexAppServerAuthProfileUnavailableError(
`Codex app-server auth profile "${profileId}" must be OpenAI Codex auth or an OpenAI API-key backup.`,
);
}
return await resolveCodexAppServerAuthProfileLoginParamsInternal({
...params,
authProfileStore: store,
});
}
export async function refreshCodexAppServerAuthTokens(params: {
@@ -582,7 +648,7 @@ async function resolveCodexAppServerAuthProfileLoginParamsInternal(params: {
config: params.config,
});
if (!loginParams) {
throw new Error(
throw new CodexAppServerAuthProfileUnavailableError(
`Codex app-server auth profile "${profileId}" does not contain usable credentials.`,
);
}
@@ -140,6 +140,13 @@ function buildEmptyToolTelemetry(): CodexAppServerToolTelemetry {
};
}
function expectUsageLimitPromptError(value: unknown): Error & { status: 429 } {
expect(value).toBeInstanceOf(Error);
const error = value as Error & { status?: unknown };
expect(error.status).toBe(429);
return error as Error & { status: 429 };
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new Error(`Expected ${label}`);
@@ -1851,9 +1858,10 @@ describe("CodexAppServerEventProjector", () => {
const result = projector.buildResult(buildEmptyToolTelemetry());
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Next reset in");
expect(result.promptError).toContain("Wait until the reset time");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Next reset in");
expect(promptError.message).toContain("Wait until the reset time");
expect(result.promptErrorSource).toBe("prompt");
});
@@ -1880,8 +1888,9 @@ describe("CodexAppServerEventProjector", () => {
const result = projector.buildResult(buildEmptyToolTelemetry());
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Next reset in");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Next reset in");
expect(result.promptErrorSource).toBe("prompt");
});
@@ -1920,8 +1929,9 @@ describe("CodexAppServerEventProjector", () => {
const result = projector.buildResult(buildEmptyToolTelemetry());
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Next reset in");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Next reset in");
expect(result.promptErrorSource).toBe("prompt");
});
@@ -1946,9 +1956,10 @@ describe("CodexAppServerEventProjector", () => {
const result = projector.buildResult(buildEmptyToolTelemetry());
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Codex says to try again at May 11th, 2026 9:00 AM.");
expect(result.promptError).not.toContain("Codex did not return a reset time");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Codex says to try again at May 11th, 2026 9:00 AM.");
expect(promptError.message).not.toContain("Codex did not return a reset time");
expect(result.promptErrorSource).toBe("prompt");
});
@@ -50,6 +50,7 @@ import {
import { formatCodexUsageLimitErrorMessage } from "./rate-limits.js";
import type { CodexTrajectoryRecorder } from "./trajectory.js";
import { attachCodexMirrorIdentity } from "./upstream-prompt-provenance.js";
import { createCodexUsageLimitPromptError } from "./usage-limit-error.js";
import { promptSnapshot } from "./user-prompt-message.js";
export { CodexNativeToolLifecycleProjector };
@@ -575,14 +576,14 @@ export class CodexAppServerEventProjector {
}
this.completedTurn = turn;
if (turn.status === "failed") {
this.promptError =
formatCodexUsageLimitErrorMessage({
message: turn.error?.message,
codexErrorInfo: turn.error?.codexErrorInfo as JsonValue | null | undefined,
rateLimits: this.options.readRecentRateLimits?.(),
}) ??
turn.error?.message ??
"codex app-server turn failed";
const usageLimitMessage = formatCodexUsageLimitErrorMessage({
message: turn.error?.message,
codexErrorInfo: turn.error?.codexErrorInfo as JsonValue | null | undefined,
rateLimits: this.options.readRecentRateLimits?.(),
});
this.promptError = usageLimitMessage
? createCodexUsageLimitPromptError(usageLimitMessage)
: (turn.error?.message ?? "codex app-server turn failed");
this.promptErrorSource = "prompt";
}
const turnItems = turn.items ?? [];
@@ -674,15 +675,16 @@ export class CodexAppServerEventProjector {
});
}
private formatCodexErrorMessage(params: JsonObject): string | undefined {
private formatCodexErrorMessage(params: JsonObject): string | Error | undefined {
const error = isJsonObject(params.error) ? params.error : undefined;
return (
formatCodexUsageLimitErrorMessage({
message: error ? readString(error, "message") : undefined,
codexErrorInfo: error?.codexErrorInfo,
rateLimits: this.options.readRecentRateLimits?.(),
}) ?? readCodexErrorNotificationMessage(params)
);
const usageLimitMessage = formatCodexUsageLimitErrorMessage({
message: error ? readString(error, "message") : undefined,
codexErrorInfo: error?.codexErrorInfo,
rateLimits: this.options.readRecentRateLimits?.(),
});
return usageLimitMessage
? createCodexUsageLimitPromptError(usageLimitMessage)
: readCodexErrorNotificationMessage(params);
}
private emitAgentEvent(
+7 -1
View File
@@ -3,7 +3,10 @@
* endpoint, including pagination and shared-client lease handling.
*/
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import type { resolveCodexAppServerAuthProfileIdForAgent } from "./auth-bridge.js";
import type {
CodexAppServerAuthRequirement,
resolveCodexAppServerAuthProfileIdForAgent,
} from "./auth-bridge.js";
import type { CodexAppServerClient } from "./client.js";
import type { CodexAppServerStartOptions } from "./config.js";
import { readCodexModelListResponse } from "./protocol-validators.js";
@@ -37,6 +40,7 @@ export type CodexAppServerListModelsOptions = {
timeoutMs?: number;
startOptions?: CodexAppServerStartOptions;
authProfileId?: string;
authRequirement?: CodexAppServerAuthRequirement;
agentDir?: string;
config?: Parameters<typeof resolveCodexAppServerAuthProfileIdForAgent>[0]["config"];
sharedClient?: boolean;
@@ -93,6 +97,7 @@ async function withCodexAppServerModelClient<T>(
startOptions: options.startOptions,
timeoutMs,
authProfileId: options.authProfileId,
authRequirement: options.authRequirement,
agentDir: options.agentDir,
config: options.config,
})
@@ -100,6 +105,7 @@ async function withCodexAppServerModelClient<T>(
startOptions: options.startOptions,
timeoutMs,
authProfileId: options.authProfileId,
authRequirement: options.authRequirement,
agentDir: options.agentDir,
config: options.config,
});
@@ -4,11 +4,33 @@ import {
buildCodexAppServerUsageSnapshot,
formatCodexUsageLimitErrorMessage,
resolveCodexUsageLimitResetAtMs,
shouldRefreshCodexRateLimitsForUsageLimitMessage,
summarizeCodexAccountUsage,
summarizeCodexRateLimits,
} from "./rate-limits.js";
describe("formatCodexUsageLimitErrorMessage", () => {
it("does not infer a Codex usage limit from unrelated prose", () => {
expect(
formatCodexUsageLimitErrorMessage({
message: "The workspace usage limit setting could not be loaded.",
}),
).toBeUndefined();
expect(shouldRefreshCodexRateLimitsForUsageLimitMessage("temporary usage limit warning")).toBe(
false,
);
});
it("accepts normalized structured Codex usage-limit error info", () => {
const message = formatCodexUsageLimitErrorMessage({
message: "quota exhausted",
codexErrorInfo: "usage_limit-exceeded",
});
expect(message?.startsWith("You've reached your Codex subscription usage limit.")).toBe(true);
expect(shouldRefreshCodexRateLimitsForUsageLimitMessage(message)).toBe(true);
});
it("gives actionable guidance when Codex omits reset details", () => {
const message = formatCodexUsageLimitErrorMessage({
message: "You've reached your usage limit.",
@@ -25,6 +25,7 @@ const ONE_DAY_MS = 24 * ONE_HOUR_MS;
const DAY_WINDOW_MINUTES = 24 * 60;
const WEEKLY_WINDOW_MINUTES = 7 * DAY_WINDOW_MINUTES;
const WEEKLY_RESET_GAP_MS = 3 * ONE_DAY_MS;
const CODEX_USAGE_LIMIT_MESSAGE_PREFIX = "You've reached your Codex subscription usage limit.";
type LimitWindowKey = (typeof LIMIT_WINDOW_KEYS)[number];
@@ -58,7 +59,7 @@ export function formatCodexUsageLimitErrorMessage(params: {
nowMs?: number;
}): string | undefined {
const message = normalizeText(params.message);
if (!isCodexUsageLimitError(params.codexErrorInfo, message)) {
if (!isCodexUsageLimitError(params.codexErrorInfo)) {
return undefined;
}
const nowMs = params.nowMs ?? Date.now();
@@ -67,7 +68,7 @@ export function formatCodexUsageLimitErrorMessage(params: {
const nextReset =
blockingReset ??
(usageSummary?.blocked ? undefined : selectNextRateLimitReset(params.rateLimits, nowMs));
const parts = ["You've reached your Codex subscription usage limit."];
const parts = [CODEX_USAGE_LIMIT_MESSAGE_PREFIX];
let recoveryAction = "Wait until Codex becomes available";
if (nextReset) {
parts.push(`Next reset ${formatResetTime(nextReset.resetsAtMs, nowMs)}.`);
@@ -95,9 +96,10 @@ export function shouldRefreshCodexRateLimitsForUsageLimitMessage(
message: string | null | undefined,
): boolean {
const text = normalizeText(message);
// Only our formatted prefix is a refresh contract. Provider prose alone is
// not structural evidence of a Codex usage-limit failure.
return Boolean(
text?.includes("You've reached your Codex subscription usage limit.") &&
!text.includes("Next reset "),
text?.startsWith(CODEX_USAGE_LIMIT_MESSAGE_PREFIX) && !text.includes("Next reset "),
);
}
@@ -210,10 +212,7 @@ export function buildCodexAppServerUsageSnapshot(value: unknown): ProviderUsageS
};
}
function isCodexUsageLimitError(
codexErrorInfo: JsonValue | null | undefined,
message: string | undefined,
): boolean {
function isCodexUsageLimitError(codexErrorInfo: JsonValue | null | undefined): boolean {
if (codexErrorInfo === "usageLimitExceeded") {
return true;
}
@@ -223,7 +222,7 @@ function isCodexUsageLimitError(
return true;
}
}
return Boolean(message?.toLowerCase().includes("usage limit"));
return false;
}
function selectNextRateLimitReset(
@@ -365,6 +365,7 @@ export async function prepareCodexAttemptConnection({ params, options }: CodexRu
isInactiveThreadBootstrapBinding,
usesSupervisionConnection,
startupAuthProfileId,
startupAuthRequirement: preparedAuthRoute?.authRequirement,
startupPreparedAuth,
startupClientAuthProfileId,
effectiveWorkspace,
@@ -17,6 +17,7 @@ import {
isInvalidCodexImagePayloadError,
resolveCodexAppServerReplayBlockedReason,
} from "./attempt-results.js";
import { readCodexRateLimitsRevision, readRecentCodexRateLimits } from "./rate-limit-cache.js";
import type { CodexAttemptActiveTurn } from "./run-attempt-active-turn.js";
import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js";
import {
@@ -36,7 +37,12 @@ import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.
import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js";
import { normalizeCodexTrajectoryError, recordCodexTrajectoryCompletion } from "./trajectory.js";
import { codexTranscriptMirrorRuntime } from "./transcript-mirror.js";
import { refreshCodexUsageLimitPromptError } from "./usage-limit-error.js";
import {
createCodexUsageLimitPromptError,
isCodexUsageLimitPromptError,
markCodexAuthProfileBlockedFromRateLimits,
refreshCodexUsageLimitPromptError,
} from "./usage-limit-error.js";
export async function finalizeCodexAttempt(
resources: CodexAttemptResources,
@@ -72,6 +78,7 @@ export async function finalizeCodexAttempt(
effectiveWorkspace,
agentDir,
attemptStartedAt,
startupAuthProfileId,
} = connection;
const { toolBridge, toolState } = attemptTools;
const {
@@ -158,9 +165,11 @@ export async function finalizeCodexAttempt(
const finalPromptErrorMessage =
typeof finalPromptError === "string"
? finalPromptError
: finalPromptError
? formatErrorMessage(finalPromptError)
: undefined;
: finalPromptError instanceof Error
? finalPromptError.message
: finalPromptError
? formatErrorMessage(finalPromptError)
: undefined;
if (isInvalidCodexImagePayloadError(finalPromptErrorMessage)) {
await clearCodexBindingAfterInvalidImagePayload(bindingStore, bindingIdentity, {
phase: "turn_completed",
@@ -197,7 +206,22 @@ export async function finalizeCodexAttempt(
signal: runAbortController.signal,
});
if (refreshedUsageLimitPromptError) {
finalPromptError = refreshedUsageLimitPromptError;
await markCodexAuthProfileBlockedFromRateLimits({
params,
authProfileId: startupAuthProfileId,
rateLimits: refreshedUsageLimitPromptError.rateLimitsForProfile,
});
finalPromptError = createCodexUsageLimitPromptError(refreshedUsageLimitPromptError.message);
} else if (
isCodexUsageLimitPromptError(finalPromptError) &&
state.rateLimitsRevisionBeforeLastTurnStart !== undefined &&
readCodexRateLimitsRevision(resourceState.client) > state.rateLimitsRevisionBeforeLastTurnStart
) {
await markCodexAuthProfileBlockedFromRateLimits({
params,
authProfileId: startupAuthProfileId,
rateLimits: readRecentCodexRateLimits(resourceState.client),
});
}
const finalPromptErrorSource =
effectiveTimedOut || clientClosedPromptErrorForFinal ? "prompt" : result.promptErrorSource;
@@ -62,6 +62,7 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources)
resolveReviewerPolicyContext,
resolveRuntimeOptionsForCurrentBinding,
startupAuthProfileId,
startupAuthRequirement,
abortFromUpstream,
} = connection;
let pluginAppServer = withCodexAppServerFastModeServiceTier(appServer, runtimeParams);
@@ -77,6 +78,7 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources)
pluginConfig,
computerUseConfig,
startupAuthProfileId: startupClientAuthProfileId,
startupAuthRequirement,
startupAuthBindingFingerprint: preparedAuthBinding?.fingerprint,
...(runtimeArtifactRequest ? { runtimeArtifactRequest } : {}),
startupPreparedAuth,
@@ -29,6 +29,7 @@ import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.
import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js";
import { buildCodexUserPromptMessage } from "./transcript-mirror.js";
import {
createCodexUsageLimitPromptError,
formatCodexTurnStartUsageLimitError,
markCodexAuthProfileBlockedFromRateLimits,
} from "./usage-limit-error.js";
@@ -262,6 +263,7 @@ export async function startCodexAttemptTurn(
result: buildCodexTurnStartFailureResult({
params,
message: usageLimitError.message,
promptError: createCodexUsageLimitPromptError(usageLimitError.message),
messagesSnapshot,
systemPromptReport,
}),
@@ -1,5 +1,6 @@
// Codex tests cover run attempt.usage limits plugin behavior.
import path from "node:path";
import { saveAuthProfileStore } from "openclaw/plugin-sdk/agent-runtime";
import { describe, expect, it } from "vitest";
import { readCodexRateLimitsRevision, rememberCodexRateLimitsRead } from "./rate-limit-cache.js";
import {
@@ -13,6 +14,13 @@ import {
setupRunAttemptTestHooks();
function expectUsageLimitPromptError(value: unknown): Error & { status: 429 } {
expect(value).toBeInstanceOf(Error);
const error = value as Error & { status?: unknown };
expect(error.status).toBe(429);
return error as Error & { status: 429 };
}
describe("runCodexAppServerAttempt usage limits", () => {
it("preserves Codex usage-limit reset details when turn/start fails", async () => {
const sessionFile = path.join(tempDir, "session.jsonl");
@@ -39,6 +47,7 @@ describe("runCodexAppServerAttempt usage limits", () => {
harnessRef.current = harness;
const params = createParams(sessionFile, workspaceDir);
params.agentDir = path.join(tempDir, "agent");
params.authProfileId = authProfileId;
params.authProfileStore = {
version: 1,
@@ -46,8 +55,8 @@ describe("runCodexAppServerAttempt usage limits", () => {
[authProfileId]: {
type: "oauth",
provider: "openai",
access: "access",
refresh: "refresh",
access: "placeholder",
refresh: "placeholder",
expires: Date.now() + 60_000,
},
},
@@ -55,8 +64,9 @@ describe("runCodexAppServerAttempt usage limits", () => {
const result = await runCodexAppServerAttempt(params);
expect(result.promptErrorSource).toBe("prompt");
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Next reset in");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Next reset in");
});
it("uses a recent Codex rate-limit snapshot when turn/start omits reset details", async () => {
@@ -93,8 +103,8 @@ describe("runCodexAppServerAttempt usage limits", () => {
[authProfileId]: {
type: "oauth",
provider: "openai",
access: "access",
refresh: "refresh",
access: "placeholder",
refresh: "placeholder",
expires: Date.now() + 60_000,
},
},
@@ -105,8 +115,9 @@ describe("runCodexAppServerAttempt usage limits", () => {
const result = await run;
expect(result.promptErrorSource).toBe("prompt");
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Next reset in");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Next reset in");
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined();
});
@@ -153,8 +164,8 @@ describe("runCodexAppServerAttempt usage limits", () => {
[authProfileId]: {
type: "oauth",
provider: "openai",
access: "access",
refresh: "refresh",
access: "placeholder",
refresh: "placeholder",
expires: Date.now() + 60_000,
},
},
@@ -162,7 +173,7 @@ describe("runCodexAppServerAttempt usage limits", () => {
const result = await runCodexAppServerAttempt(params);
expect(result.promptError).toContain("Next reset in");
expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in");
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined();
});
@@ -187,15 +198,17 @@ describe("runCodexAppServerAttempt usage limits", () => {
const result = await run;
expect(result.promptErrorSource).toBe("prompt");
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Next reset in");
expect(result.promptError).not.toContain("Codex did not return a reset time");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Next reset in");
expect(promptError.message).not.toContain("Codex did not return a reset time");
});
it("refreshes Codex account rate limits when a failed turn omits reset details", async () => {
const sessionFile = path.join(tempDir, "session.jsonl");
const workspaceDir = path.join(tempDir, "workspace");
const resetsAt = Math.ceil(Date.now() / 1000) + 120;
const authProfileId = "openai:work";
const harness = createStartedThreadHarness(async (method) => {
if (method === "account/rateLimits/read") {
return rateLimitsUpdated(resetsAt).params;
@@ -203,7 +216,23 @@ describe("runCodexAppServerAttempt usage limits", () => {
return undefined;
});
const run = runCodexAppServerAttempt(createParams(sessionFile, workspaceDir));
const params = createParams(sessionFile, workspaceDir);
params.agentDir = path.join(tempDir, "streamed-usage-limit-agent");
params.authProfileId = authProfileId;
params.authProfileStore = {
version: 1,
profiles: {
[authProfileId]: {
type: "oauth",
provider: "openai",
access: "placeholder",
refresh: "placeholder",
expires: Date.now() + 60_000,
},
},
};
saveAuthProfileStore(params.authProfileStore, params.agentDir);
const run = runCodexAppServerAttempt(params);
await harness.waitForMethod("turn/start");
await harness.notify({
method: "turn/completed",
@@ -223,11 +252,114 @@ describe("runCodexAppServerAttempt usage limits", () => {
const result = await run;
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
expect(result.promptError).toContain("Next reset in");
expect(result.promptError).not.toContain("Codex did not return a reset time");
const promptError = expectUsageLimitPromptError(result.promptError);
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
expect(promptError.message).toContain("Next reset in");
expect(promptError.message).not.toContain("Codex did not return a reset time");
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBe(resetsAt * 1000);
expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe(
true,
);
});
it("blocks after a streamed usage-limit failure with trusted in-turn limits", async () => {
const sessionFile = path.join(tempDir, "session.jsonl");
const workspaceDir = path.join(tempDir, "workspace");
const resetsAt = Math.ceil(Date.now() / 1000) + 120;
const authProfileId = "openai:work";
const harness = createStartedThreadHarness(async () => undefined);
const params = createParams(sessionFile, workspaceDir);
params.agentDir = path.join(tempDir, "trusted-streamed-usage-limit-agent");
params.authProfileId = authProfileId;
params.authProfileStore = {
version: 1,
profiles: {
[authProfileId]: {
type: "oauth",
provider: "openai",
access: "placeholder",
refresh: "placeholder",
expires: Date.now() + 60_000,
},
},
};
saveAuthProfileStore(params.authProfileStore, params.agentDir);
const run = runCodexAppServerAttempt(params);
await harness.waitForMethod("turn/start");
await harness.notify(rateLimitsUpdated(resetsAt));
await harness.notify({
method: "turn/completed",
params: {
threadId: "thread-1",
turnId: "turn-1",
turn: {
id: "turn-1",
status: "failed",
error: {
message: "You've reached your usage limit.",
codexErrorInfo: "usageLimitExceeded",
},
},
},
});
const result = await run;
expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in");
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBe(resetsAt * 1000);
expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe(
false,
);
});
it("does not block after a streamed usage-limit failure with only stale limits", async () => {
const sessionFile = path.join(tempDir, "session.jsonl");
const workspaceDir = path.join(tempDir, "workspace");
const resetsAt = Math.ceil(Date.now() / 1000) + 120;
const authProfileId = "openai:work";
const harness = createStartedThreadHarness(async () => undefined);
rememberCodexRateLimitsRead(harness.client, rateLimitsUpdated(resetsAt).params);
const params = createParams(sessionFile, workspaceDir);
params.agentDir = path.join(tempDir, "stale-streamed-usage-limit-agent");
params.authProfileId = authProfileId;
params.authProfileStore = {
version: 1,
profiles: {
[authProfileId]: {
type: "oauth",
provider: "openai",
access: "placeholder",
refresh: "placeholder",
expires: Date.now() + 60_000,
},
},
};
const run = runCodexAppServerAttempt(params);
await harness.waitForMethod("turn/start");
await harness.notify({
method: "turn/completed",
params: {
threadId: "thread-1",
turnId: "turn-1",
turn: {
id: "turn-1",
status: "failed",
error: {
message: "You've reached your usage limit.",
codexErrorInfo: "usageLimitExceeded",
},
},
},
});
const result = await run;
expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in");
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined();
expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe(
false,
);
});
});
@@ -1414,12 +1414,18 @@ describe("shared Codex app-server client", () => {
.mockReturnValueOnce("api-key:first")
.mockReturnValueOnce("api-key:second");
const firstList = listCodexAppServerModels({ timeoutMs: 1000 });
const firstList = listCodexAppServerModels({
timeoutMs: 1000,
authRequirement: "api-key",
});
await sendInitializeResult(first, "openclaw/0.143.0 (macOS; test)");
await sendEmptyModelList(first);
await expect(firstList).resolves.toEqual({ models: [] });
const secondList = listCodexAppServerModels({ timeoutMs: 1000 });
const secondList = listCodexAppServerModels({
timeoutMs: 1000,
authRequirement: "api-key",
});
await sendInitializeResult(second, "openclaw/0.143.0 (macOS; test)");
await sendEmptyModelList(second);
await expect(secondList).resolves.toEqual({ models: [] });
@@ -1429,6 +1435,49 @@ describe("shared Codex app-server client", () => {
expect(second.process.stdin.destroyed).toBe(false);
});
it("does not share a client across auth requirements", async () => {
const first = createClientHarness();
const second = createClientHarness();
const startSpy = vi
.spyOn(CodexAppServerClient, "start")
.mockReturnValueOnce(first.client)
.mockReturnValueOnce(second.client);
const firstList = listCodexAppServerModels({
timeoutMs: 1000,
authProfileId: "openai:work",
authRequirement: "api-key",
});
await sendInitializeResult(first, "openclaw/0.143.0 (macOS; test)");
await sendEmptyModelList(first);
await expect(firstList).resolves.toEqual({ models: [] });
const secondList = listCodexAppServerModels({
timeoutMs: 1000,
authProfileId: "openai:work",
authRequirement: "subscription",
});
await sendInitializeResult(second, "openclaw/0.143.0 (macOS; test)");
await sendEmptyModelList(second);
await expect(secondList).resolves.toEqual({ models: [] });
expect(startSpy).toHaveBeenCalledTimes(2);
expect(first.process.stdin.destroyed).toBe(false);
expect(second.process.stdin.destroyed).toBe(false);
});
it("rejects prepared auth that conflicts with the auth requirement", async () => {
const startSpy = vi.spyOn(CodexAppServerClient, "start");
await expect(
getSharedCodexAppServerClient({
authRequirement: "subscription",
preparedAuth: { kind: "api-key", apiKey: "placeholder" },
}),
).rejects.toThrow("Prepared Codex auth does not satisfy the requested auth requirement.");
expect(startSpy).not.toHaveBeenCalled();
});
it("does not let one shared-client failure tear down another keyed client", async () => {
const first = createClientHarness();
const second = createClientHarness();
@@ -17,6 +17,7 @@ import {
resolveCodexAppServerPreparedAuthProfileSnapshot,
resolveCodexAppServerPreparedApiKeyCacheKey,
type CodexAppServerPreparedAuth,
type CodexAppServerAuthRequirement,
type CodexAppServerResolvedPreparedAuth,
} from "./auth-bridge.js";
import { ensureCodexAppServerClientRuntime } from "./client-runtime.js";
@@ -246,6 +247,7 @@ export type CodexAppServerClientOptions = {
/** Previously minted exact runtime required before the process may start. */
expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding;
preparedAuth?: CodexAppServerPreparedAuth;
authRequirement?: CodexAppServerAuthRequirement;
agentDir?: string;
config?: Parameters<typeof resolveCodexAppServerAuthProfileIdForAgent>[0]["config"];
onStartedClient?: (client: CodexAppServerClient) => void;
@@ -263,10 +265,20 @@ type ResolvedCodexAppServerClientStartContext = {
authProfileId: string | undefined;
authProfileStore: AuthProfileStore | undefined;
preparedAuth: CodexAppServerResolvedPreparedAuth | undefined;
authRequirement: CodexAppServerAuthRequirement | undefined;
requestedStartOptions: CodexAppServerStartOptions;
startOptions: CodexAppServerStartOptions;
};
function inferAuthRequirement(
preparedAuth: CodexAppServerPreparedAuth | undefined,
): CodexAppServerAuthRequirement | undefined {
if (preparedAuth?.kind === "api-key") {
return "api-key";
}
return preparedAuth?.kind === "profile" ? "subscription" : undefined;
}
async function resolveCodexAppServerClientStartContext(
options?: CodexAppServerClientOptions,
): Promise<ResolvedCodexAppServerClientStartContext> {
@@ -287,6 +299,15 @@ async function resolveCodexAppServerClientStartContext(
if (preparedAuth && requestedStartOptions.homeScope === "user") {
throw new Error("Prepared Codex auth requires an isolated app-server home.");
}
const preparedAuthRequirement = inferAuthRequirement(preparedAuth);
if (
options?.authRequirement &&
preparedAuthRequirement &&
options.authRequirement !== preparedAuthRequirement
) {
throw new Error("Prepared Codex auth does not satisfy the requested auth requirement.");
}
const authRequirement = options?.authRequirement ?? preparedAuthRequirement;
const usesNativeAuth =
!preparedAuth &&
(options?.authProfileId === null || requestedStartOptions.homeScope === "user");
@@ -361,6 +382,7 @@ async function resolveCodexAppServerClientStartContext(
authProfileStore,
requestedStartOptions,
preparedAuth: resolvedPreparedAuth,
authRequirement,
startOptions,
};
}
@@ -500,6 +522,7 @@ async function acquireSharedCodexAppServerClient(
authProfileId,
authProfileStore,
preparedAuth,
authRequirement,
requestedStartOptions,
startOptions,
} = context;
@@ -508,15 +531,15 @@ async function acquireSharedCodexAppServerClient(
preparedAuth?.kind === "api-key"
? resolveCodexAppServerPreparedApiKeyCacheKey(preparedAuth.apiKey)
: (preparedAuth?.snapshot.secretFreeCacheKey ??
(authProfileId
? undefined
: resolveCodexAppServerFallbackApiKeyCacheKey({ startOptions })));
const baseKey = codexAppServerStartOptionsKey(startOptions, {
(authRequirement === "api-key" && !authProfileId
? resolveCodexAppServerFallbackApiKeyCacheKey({ startOptions })
: undefined));
const baseKey = `${codexAppServerStartOptionsKey(startOptions, {
authProfileId,
authBindingFingerprint: options?.authBindingFingerprint,
agentDir: usesNativeAuth ? undefined : agentDir,
fallbackApiKeyCacheKey: authIdentityCacheKey,
});
})}\0auth-requirement:${authRequirement ?? "native"}`;
// Capture turns cannot inherit a normal client whose loaded bytes predate the
// filesystem snapshot. Keep their physical process generation separate.
const runtimeArtifactMode =
@@ -576,6 +599,7 @@ async function acquireSharedCodexAppServerClient(
authProfileId: usesNativeAuth || preparedAuth?.kind === "api-key" ? null : authProfileId,
authProfileStore,
preparedAuth,
authRequirement,
runtimeArtifactMode,
...(options?.expectedRuntimeArtifact
? { expectedRuntimeArtifact: options.expectedRuntimeArtifact }
@@ -665,6 +689,7 @@ function createSharedCodexAppServerClientStartup(params: {
expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding;
runtimeArtifactSignal?: AbortSignal;
preparedAuth?: CodexAppServerResolvedPreparedAuth;
authRequirement?: CodexAppServerAuthRequirement;
config?: CodexAppServerClientOptions["config"];
}): SharedCodexAppServerClientStartup {
const initialized = createDeferred<void>();
@@ -675,6 +700,7 @@ function createSharedCodexAppServerClientStartup(params: {
authProfileId: params.authProfileId,
authProfileStore: params.authProfileStore,
preparedAuth: params.preparedAuth,
authRequirement: params.authRequirement,
runtimeArtifactMode: params.runtimeArtifactMode,
...(params.expectedRuntimeArtifact
? { expectedRuntimeArtifact: params.expectedRuntimeArtifact }
@@ -722,6 +748,7 @@ export async function createIsolatedCodexAppServerClient(
authProfileId,
authProfileStore,
preparedAuth,
authRequirement,
requestedStartOptions,
startOptions,
} = await withCodexAppServerAcquireDeadline(
@@ -736,6 +763,7 @@ export async function createIsolatedCodexAppServerClient(
authProfileId: usesNativeAuth || preparedAuth?.kind === "api-key" ? null : authProfileId,
authProfileStore,
preparedAuth,
authRequirement,
runtimeArtifactMode:
options?.runtimeArtifactMode ?? (options?.expectedRuntimeArtifact ? "capture" : undefined),
...(options?.expectedRuntimeArtifact
@@ -759,6 +787,7 @@ async function startInitializedCodexAppServerClient(params: {
expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding;
runtimeArtifactSignal?: AbortSignal;
preparedAuth?: CodexAppServerResolvedPreparedAuth;
authRequirement?: CodexAppServerAuthRequirement;
config?: CodexAppServerClientOptions["config"];
timeoutMs?: number;
abandonSignal?: AbortSignal;
@@ -859,6 +888,7 @@ async function startInitializedCodexAppServerClient(params: {
agentDir: params.agentDir,
authProfileId: params.authProfileId,
preparedAuth: params.preparedAuth,
authRequirement: params.authRequirement,
startOptions,
config: params.config,
...(params.authProfileStore ? { authProfileStore: params.authProfileStore } : {}),
@@ -288,6 +288,7 @@ export async function runCodexAppServerSideQuestion(
const clientOptions = {
startOptions: appServer.start,
timeoutMs: appServer.requestTimeoutMs,
authRequirement: preparedRuntimeAuth.plan.modelRoute?.authRequirement,
...(startupPreparedAuth
? { preparedAuth: startupPreparedAuth }
: { authProfileId: connection.clientAuthProfileId }),
@@ -41,6 +41,14 @@ type CodexUsageLimitErrorResult = {
rateLimitsForProfile?: JsonValue;
};
export function createCodexUsageLimitPromptError(message: string): Error & { status: 429 } {
return Object.assign(new Error(message), { status: 429 as const });
}
export function isCodexUsageLimitPromptError(error: unknown): error is Error & { status: 429 } {
return error instanceof Error && "status" in error && error.status === 429;
}
/** Marks a Codex auth profile blocked until the reset time advertised by rate limits. */
export async function markCodexAuthProfileBlockedFromRateLimits(params: {
params: EmbeddedRunAttemptParams;
@@ -101,22 +109,20 @@ export async function refreshCodexUsageLimitPromptError(params: {
message: string | undefined;
timeoutMs?: number;
signal?: AbortSignal;
}): Promise<string | undefined> {
}): Promise<CodexUsageLimitErrorResult | undefined> {
if (!shouldRefreshCodexRateLimitsForUsageLimitMessage(params.message)) {
return undefined;
}
return (
await refreshCodexUsageLimitError({
client: params.client,
source: {
message: params.message,
codexErrorInfo: "usageLimitExceeded",
rateLimits: readRecentCodexRateLimits(params.client),
},
timeoutMs: params.timeoutMs,
signal: params.signal,
})
)?.message;
return refreshCodexUsageLimitError({
client: params.client,
source: {
message: params.message,
codexErrorInfo: "usageLimitExceeded",
rateLimits: readRecentCodexRateLimits(params.client),
},
timeoutMs: params.timeoutMs,
signal: params.signal,
});
}
async function refreshCodexUsageLimitError(params: {
+50
View File
@@ -368,6 +368,56 @@ describe("resolveAuthProfileOrder", () => {
).toStrictEqual(["fixture-provider:backup", "fixture-provider:primary"]);
});
it("does not apply a block scoped to another model when ordering profiles", () => {
const store: AuthProfileStore = {
version: 1,
profiles: {
"fixture-provider:primary": {
type: "api_key",
provider: "fixture-provider",
key: "placeholder",
},
"fixture-provider:backup": {
type: "api_key",
provider: "fixture-provider",
key: "placeholder",
},
},
usageStats: {
"fixture-provider:primary": {
blockedUntil: Date.now() + 60_000,
blockedReason: "subscription_limit",
blockedModel: "model-a",
blockedScope: "model",
},
},
};
const cfg = {
auth: {
order: {
"fixture-provider": ["fixture-provider:primary", "fixture-provider:backup"],
},
},
} satisfies OpenClawConfig;
expect(
resolveAuthProfileOrder({
cfg,
store,
provider: "fixture-provider",
forModel: "model-b",
}),
).toStrictEqual(["fixture-provider:primary", "fixture-provider:backup"]);
expect(
resolveAuthProfileOrder({
cfg,
store,
provider: "fixture-provider",
forModel: "model-a",
}),
).toStrictEqual(["fixture-provider:backup", "fixture-provider:primary"]);
});
it("keeps unresolved OAuth refs only in read-only profile ordering", () => {
const store: AuthProfileStore = {
version: 1,
+3 -2
View File
@@ -374,7 +374,7 @@ export function resolveAuthProfileOrderWithMetadata(
for (const profileId of deduped) {
if (isProfileInCooldown(store, profileId, now, forModel)) {
const cooldownUntil =
resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}) ?? now;
resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}, forModel) ?? now;
inCooldown.push({ profileId, cooldownUntil });
} else {
available.push(profileId);
@@ -493,7 +493,8 @@ function orderProfilesByMode(
const cooldownSorted = inCooldown
.map((profileId) => ({
profileId,
cooldownUntil: resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}) ?? now,
cooldownUntil:
resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}, forModel) ?? now,
}))
.toSorted((a, b) => a.cooldownUntil - b.cooldownUntil)
.map((entry) => entry.profileId);
+1
View File
@@ -118,6 +118,7 @@ function normalizeUsageStatsEntry(raw: unknown): ProfileUsageStats | undefined {
blockedReason: normalizeEnumValue(raw.blockedReason, AUTH_BLOCKED_REASONS),
blockedSource: normalizeEnumValue(raw.blockedSource, AUTH_BLOCKED_SOURCES),
blockedModel: normalizeOptionalString(raw.blockedModel),
blockedScope: raw.blockedScope === "model" ? "model" : undefined,
cooldownUntil: normalizeFiniteNumber(raw.cooldownUntil),
cooldownReason: normalizeEnumValue(raw.cooldownReason, AUTH_FAILURE_REASONS),
cooldownModel: normalizeOptionalString(raw.cooldownModel),
+1
View File
@@ -106,6 +106,7 @@ export type ProfileUsageStats = {
blockedReason?: AuthProfileBlockedReason;
blockedSource?: AuthProfileBlockedSource;
blockedModel?: string;
blockedScope?: "model";
cooldownUntil?: number;
cooldownReason?: AuthProfileFailureReason;
cooldownModel?: string;
+45 -7
View File
@@ -23,9 +23,16 @@ export function isModelScopedCooldownReason(reason: AuthProfileFailureReason | u
/** Resolves the latest active blocked/cooldown/disabled timestamp for a profile. */
export function resolveProfileUnusableUntil(
stats: Pick<ProfileUsageStats, "blockedUntil" | "cooldownUntil" | "disabledUntil">,
stats: Pick<
ProfileUsageStats,
"blockedUntil" | "blockedModel" | "blockedScope" | "cooldownUntil" | "disabledUntil"
>,
forModel?: string,
): number | null {
const values = [stats.blockedUntil, stats.cooldownUntil, stats.disabledUntil]
const blockedUntil = isBlockScopedToDifferentModel(stats, forModel)
? undefined
: stats.blockedUntil;
const values = [blockedUntil, stats.cooldownUntil, stats.disabledUntil]
.map((value) => asDateTimestampMs(value))
.filter((value): value is number => value !== undefined && value > 0);
if (values.length === 0) {
@@ -40,10 +47,40 @@ export function isActiveUnusableWindow(until: number | undefined, now: number):
return timestamp !== undefined && timestamp > 0 && now < timestamp;
}
function isBlockedWindowActiveForModel(
stats: Pick<ProfileUsageStats, "blockedUntil" | "blockedModel" | "blockedScope">,
now: number,
forModel?: string,
): boolean {
return (
!isBlockScopedToDifferentModel(stats, forModel) &&
isActiveUnusableWindow(stats.blockedUntil, now)
);
}
function isBlockScopedToDifferentModel(
stats: Pick<ProfileUsageStats, "blockedModel" | "blockedScope">,
forModel?: string,
): boolean {
// Legacy rows carried blockedModel for profile-wide blocks without a scope marker.
// Only explicit model scope narrows them; unmarked rows stay wide until expiry.
return Boolean(
forModel &&
stats.blockedScope === "model" &&
stats.blockedModel &&
stats.blockedModel !== forModel,
);
}
function shouldBypassModelScopedCooldown(
stats: Pick<
ProfileUsageStats,
"blockedUntil" | "cooldownReason" | "cooldownModel" | "disabledUntil"
| "blockedUntil"
| "blockedModel"
| "blockedScope"
| "cooldownReason"
| "cooldownModel"
| "disabledUntil"
>,
now: number,
forModel?: string,
@@ -53,7 +90,7 @@ function shouldBypassModelScopedCooldown(
isModelScopedCooldownReason(stats.cooldownReason) &&
stats.cooldownModel &&
stats.cooldownModel !== forModel &&
!isActiveUnusableWindow(stats.blockedUntil, now) &&
!isBlockedWindowActiveForModel(stats, now, forModel) &&
!isActiveUnusableWindow(stats.disabledUntil, now),
);
}
@@ -82,7 +119,7 @@ export function isProfileInCooldown(
if (shouldBypassModelScopedCooldown(stats, ts, forModel)) {
return false;
}
const unusableUntil = resolveProfileUnusableUntil(stats);
const unusableUntil = resolveProfileUnusableUntil(stats, forModel);
return unusableUntil ? ts < unusableUntil : false;
}
@@ -107,7 +144,7 @@ export function getSoonestCooldownExpiry(
if (shouldBypassModelScopedCooldown(stats, ts, options?.forModel)) {
continue;
}
const until = resolveProfileUnusableUntil(stats);
const until = resolveProfileUnusableUntil(stats, options?.forModel);
if (typeof until !== "number" || !Number.isFinite(until) || until <= 0) {
continue;
}
@@ -115,7 +152,7 @@ export function getSoonestCooldownExpiry(
options?.forModel &&
stats.cooldownReason === "rate_limit" &&
stats.cooldownModel === options.forModel &&
!isActiveUnusableWindow(stats.blockedUntil, ts) &&
!isBlockedWindowActiveForModel(stats, ts, options.forModel) &&
!isActiveUnusableWindow(stats.disabledUntil, ts);
if (matchingModelScopedCooldown) {
latestMatchingModelCooldown =
@@ -195,6 +232,7 @@ export function clearExpiredCooldowns(store: AuthProfileStore, now?: number): bo
stats.blockedReason = undefined;
stats.blockedSource = undefined;
stats.blockedModel = undefined;
stats.blockedScope = undefined;
profileMutated = true;
}
if (disabledExpired) {
+112 -1
View File
@@ -83,6 +83,7 @@ function expectProfileErrorStateCleared(
) {
expect(stats?.blockedUntil).toBeUndefined();
expect(stats?.blockedReason).toBeUndefined();
expect(stats?.blockedScope).toBeUndefined();
expect(stats?.cooldownUntil).toBeUndefined();
expect(stats?.disabledUntil).toBeUndefined();
expect(stats?.disabledReason).toBeUndefined();
@@ -103,6 +104,18 @@ describe("resolveProfileUnusableUntil", () => {
).toBe(300);
expect(resolveProfileUnusableUntil({ cooldownUntil: 300 })).toBe(300);
});
it("keeps legacy blockedModel rows profile-wide", () => {
expect(
resolveProfileUnusableUntil({ blockedUntil: 300, blockedModel: "model-a" }, "model-b"),
).toBe(300);
});
it("applies explicitly model-scoped blocks only to that model", () => {
const stats = { blockedUntil: 300, blockedModel: "model-a", blockedScope: "model" as const };
expect(resolveProfileUnusableUntil(stats, "model-a")).toBe(300);
expect(resolveProfileUnusableUntil(stats, "model-b")).toBeNull();
});
});
describe("resolveProfileUnusableUntilForDisplay", () => {
@@ -285,17 +298,32 @@ describe("isProfileInCooldown", () => {
expect(isProfileInCooldown(store, "github-copilot:github", undefined, "gpt-4.1")).toBe(true);
});
it("does not bypass model-scoped cooldown when blockedUntil is active", () => {
it("bypasses model-scoped blocks and cooldowns for sibling models", () => {
const now = Date.now();
const store = makeStore({
"google:default": {
blockedUntil: now + 120_000,
blockedReason: "subscription_limit",
blockedModel: "gemini-3-flash-preview",
blockedScope: "model",
cooldownUntil: now + 60_000,
cooldownReason: "timeout",
cooldownModel: "gemini-3-flash-preview",
},
});
expect(isProfileInCooldown(store, "google:default", now, "gemini-3-flash-preview")).toBe(true);
expect(isProfileInCooldown(store, "google:default", now, "gemini-3.1-flash-lite")).toBe(false);
});
it("keeps legacy blockedModel rows active for sibling models", () => {
const now = Date.now();
const store = makeStore({
"google:default": {
blockedUntil: now + 120_000,
blockedModel: "gemini-3-flash-preview",
},
});
expect(isProfileInCooldown(store, "google:default", now, "gemini-3.1-flash-lite")).toBe(true);
});
});
@@ -917,6 +945,89 @@ describe("markAuthProfileFailure — active windows do not extend on retry", ()
});
describe("markAuthProfileBlockedUntil", () => {
it("keeps repeated same-model blocks scoped to that model", async () => {
const now = Date.parse("2026-05-30T18:00:00.000Z");
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now);
const store = makeStore({
"openai:default": {
blockedUntil: now + 60_000,
blockedModel: "gpt-5.4",
blockedScope: "model",
},
});
mockLockedUpdateForStore(store);
try {
await markAuthProfileBlockedUntil({
store,
profileId: "openai:default",
blockedUntil: now + 120_000,
source: "codex_rate_limits",
modelId: "gpt-5.4",
});
} finally {
nowSpy.mockRestore();
}
expect(store.usageStats?.["openai:default"]?.blockedModel).toBe("gpt-5.4");
expect(store.usageStats?.["openai:default"]?.blockedScope).toBe("model");
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4")).toBe(true);
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(false);
});
it("widens an active block after a different model fails", async () => {
const now = Date.parse("2026-05-30T18:00:00.000Z");
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now);
const store = makeStore({
"openai:default": {
blockedUntil: now + 60_000,
blockedModel: "gpt-5.4",
blockedScope: "model",
},
});
mockLockedUpdateForStore(store);
try {
await markAuthProfileBlockedUntil({
store,
profileId: "openai:default",
blockedUntil: now + 120_000,
source: "codex_rate_limits",
modelId: "gpt-5.4-mini",
});
} finally {
nowSpy.mockRestore();
}
expect(store.usageStats?.["openai:default"]?.blockedModel).toBeUndefined();
expect(store.usageStats?.["openai:default"]?.blockedScope).toBeUndefined();
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(true);
});
it("never narrows an active profile-wide block", async () => {
const now = Date.parse("2026-05-30T18:00:00.000Z");
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now);
const store = makeStore({
"openai:default": {
blockedUntil: now + 60_000,
},
});
mockLockedUpdateForStore(store);
try {
await markAuthProfileBlockedUntil({
store,
profileId: "openai:default",
blockedUntil: now + 120_000,
source: "codex_rate_limits",
modelId: "gpt-5.4",
});
} finally {
nowSpy.mockRestore();
}
expect(store.usageStats?.["openai:default"]?.blockedModel).toBeUndefined();
expect(store.usageStats?.["openai:default"]?.blockedScope).toBeUndefined();
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(true);
});
it("keeps a later active blocked-until timestamp", async () => {
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-05-30T18:00:00.000Z"));
const laterBlockedUntil = Date.parse("2031-01-01T00:00:00.000Z");
+14 -1
View File
@@ -224,6 +224,7 @@ function applyWhamCooldownResult(params: {
blockedReason: "subscription_limit",
blockedSource: params.whamResult.blockedSource ?? "wham",
blockedModel: undefined,
blockedScope: undefined,
cooldownUntil: undefined,
cooldownReason: undefined,
cooldownModel: undefined,
@@ -586,6 +587,7 @@ function resetUsageStats(
blockedReason: undefined,
blockedSource: undefined,
blockedModel: undefined,
blockedScope: undefined,
cooldownUntil: undefined,
cooldownReason: undefined,
cooldownModel: undefined,
@@ -842,12 +844,23 @@ function buildBlockedProfileUsageStats(params: {
params.previousStats?.blockedUntil,
params.now,
);
// One active block can stay model-scoped only while every observation names
// that same model. Mixed or unknown observations widen the profile.
const blockedModel =
activeBlockedUntil === 0
? params.modelId
: params.previousStats?.blockedScope === "model" &&
params.previousStats.blockedModel === params.modelId &&
params.modelId
? params.modelId
: undefined;
return {
...params.previousStats,
blockedUntil: Math.max(activeBlockedUntil, params.blockedUntil),
blockedReason: "subscription_limit",
blockedSource: params.source,
blockedModel: params.modelId,
blockedModel,
blockedScope: blockedModel ? "model" : undefined,
cooldownUntil: undefined,
cooldownReason: undefined,
cooldownModel: undefined,
+12
View File
@@ -995,6 +995,18 @@ describe("failover-error", () => {
expect(coerceToFailoverError(err)?.status).toBe(429);
});
it("classifies a structured prompt error independently of its wording", () => {
const promptError = Object.assign(new Error("quota exhausted"), { status: 429 as const });
const failoverError = coerceToFailoverError(promptError, {
provider: "openai",
model: "gpt-5.4",
});
expect(failoverError?.reason).toBe("rate_limit");
expect(failoverError?.status).toBe(429);
expect(failoverError?.message).toBe("quota exhausted");
});
it("lets wrapped causes override parent context-overflow classifications", () => {
const err = new Error("INVALID_ARGUMENT: input exceeds the maximum number of tokens", {
cause: { code: "RESOURCE_EXHAUSTED" },
@@ -210,20 +210,26 @@ function expectFailureCount(
expect(failureCounts?.[reason]).toBe(expected);
}
async function writeMultiProfileAuthStore(agentDir: string) {
async function writeMultiProfileAuthStore(
agentDir: string,
options?: { openAiProfileCount?: 2 | 3 },
) {
const includeThirdOpenAiProfile = options?.openAiProfileCount !== 2;
saveAuthProfileStore(
{
version: 1,
profiles: {
"openai:p1": { type: "api_key", provider: "openai", key: "sk-openai-1" },
"openai:p2": { type: "api_key", provider: "openai", key: "sk-openai-2" },
"openai:p3": { type: "api_key", provider: "openai", key: "sk-openai-3" },
...(includeThirdOpenAiProfile
? { "openai:p3": { type: "api_key" as const, provider: "openai", key: "placeholder" } }
: {}),
"groq:p1": { type: "api_key", provider: "groq", key: "sk-groq" },
},
usageStats: {
"openai:p1": { lastUsed: 1 },
"openai:p2": { lastUsed: 2 },
"openai:p3": { lastUsed: 3 },
...(includeThirdOpenAiProfile ? { "openai:p3": { lastUsed: 3 } } : {}),
"groq:p1": { lastUsed: 4 },
},
},
@@ -983,6 +989,41 @@ describe("runWithModelFallback + runEmbeddedAgent failover behavior", () => {
});
});
it("rotates Codex profiles on structured prompt rate limits before model fallback", async () => {
await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
await writeMultiProfileAuthStore(agentDir, { openAiProfileCount: 2 });
mockPrimaryFailureThenFallbackSuccess(() => {
return makeEmbeddedRunnerAttempt({
promptError: Object.assign(
new Error("You've reached your Codex subscription usage limit."),
{ status: 429 as const },
),
promptErrorSource: "prompt",
});
});
const result = await runEmbeddedFallback({
agentDir,
workspaceDir,
sessionKey: "agent:test:codex-structured-prompt-rate-limit",
runId: "run:codex-structured-prompt-rate-limit",
});
expect(result.provider).toBe("groq");
expect(result.model).toBe("mock-2");
expect(result.attempts[0]?.reason).toBe("rate_limit");
expectProviderAttemptCounts({ openai: 2, groq: 1 });
const primaryCalls = runEmbeddedAttemptMock.mock.calls
.map(([params]) => params as EmbeddedAttemptParams)
.filter((params) => params.provider === "openai");
expect(primaryCalls.map((params) => params.authProfileId)).toStrictEqual([
"openai:p1",
"openai:p2",
]);
expect(primaryCalls.map((params) => params.modelId)).toStrictEqual(["mock-1", "mock-1"]);
});
});
it("respects prompt-side rateLimitedProfileRotations=0 and falls back immediately", async () => {
await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
await writeMultiProfileAuthStore(agentDir);