mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
fix(codex): model-scoped usage-limit blocks, structural 429 classification, no silent API-key billing (#108254)
* test(codex): use allowlisted placeholder for auth-bridge api-key fixture * fix(codex): model-scoped usage-limit blocks, structural 429 classification, no silent API-key billing - usage-limit blocks written from Codex rate-limit resets are model-scoped via a persisted blockedScope marker: healthy sibling models on the same auth profile stay usable; a different/unknown model failing widens the block profile-wide and never narrows back; legacy rows without the marker stay profile-wide until they expire (#100556) - Codex usage-limit failures now surface as status-429 Error objects at both ingress paths (turn-start and streamed turn failure, wrapped at the event projector), so core failover classification is structural instead of matching message wording; profile blocking uses only rate-limit data whose revision advanced during the turn - usage-limit detection requires the structured codexErrorInfo signal; the over-broad "usage limit" substring match that misclassified unrelated errors as subscription limits is gone (#96815) - subscription/OAuth routes can no longer silently fall back to env/auth.json API keys: ambient key fallback is restricted to explicit api-key routes, native-auth subscription routes verify the account is chatgpt-backed via account/read, and shared app-server clients are partitioned by auth requirement so pooled clients cannot cross billing modes (#106375) - integrated e2e regression: usage-limit promptError -> same-model sibling profile rotation -> model fallback with reason rate_limit * chore(codex): keep CodexUsageLimitErrorResult type local
This commit is contained in:
committed by
GitHub
parent
4e756adaaf
commit
1a27e7f3ec
@@ -104,6 +104,7 @@ export function resolveCodexAppServerReplayBlockedReason(
|
||||
export function buildCodexTurnStartFailureResult(params: {
|
||||
params: EmbeddedRunAttemptParams;
|
||||
message: string;
|
||||
promptError?: unknown;
|
||||
messagesSnapshot: AgentMessage[];
|
||||
systemPromptReport: CodexSystemPromptReport;
|
||||
}): EmbeddedRunAttemptResult {
|
||||
@@ -114,7 +115,7 @@ export function buildCodexTurnStartFailureResult(params: {
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
promptError: params.message,
|
||||
promptError: params.promptError ?? params.message,
|
||||
promptErrorSource: "prompt",
|
||||
sessionIdUsed: params.params.sessionId,
|
||||
messagesSnapshot: params.messagesSnapshot,
|
||||
|
||||
@@ -125,6 +125,7 @@ export async function startCodexAttemptThread(params: {
|
||||
pluginConfig: CodexPluginConfig;
|
||||
computerUseConfig: ResolvedCodexComputerUseConfig;
|
||||
startupAuthProfileId: string | null | undefined;
|
||||
startupAuthRequirement?: CodexAppServerClientOptions["authRequirement"];
|
||||
startupAuthBindingFingerprint: string | undefined;
|
||||
runtimeArtifactRequest?: Readonly<{
|
||||
expected?: AgentHarnessRuntimeArtifactBinding;
|
||||
@@ -228,6 +229,7 @@ export async function startCodexAttemptThread(params: {
|
||||
...(params.startupPreparedAuth
|
||||
? { preparedAuth: params.startupPreparedAuth }
|
||||
: { authProfileId: params.startupAuthProfileId }),
|
||||
authRequirement: params.startupAuthRequirement,
|
||||
authProfileStore: attemptParams.authProfileStore,
|
||||
authBindingFingerprint: params.startupAuthBindingFingerprint,
|
||||
...(params.runtimeArtifactRequest
|
||||
|
||||
@@ -1965,6 +1965,112 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("fails subscription auth instead of falling back to an API key", async () => {
|
||||
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-app-server-"));
|
||||
const request = vi.fn(async () => ({ type: "apiKey" }));
|
||||
vi.stubEnv("CODEX_API_KEY", "placeholder");
|
||||
let rejection: unknown;
|
||||
try {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authProfileId: "openai:work",
|
||||
authProfileStore: {
|
||||
version: 1,
|
||||
profiles: {},
|
||||
},
|
||||
authRequirement: "subscription",
|
||||
startOptions: createStartOptions({
|
||||
env: { CODEX_API_KEY: "placeholder" },
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
rejection = error;
|
||||
} finally {
|
||||
await fs.rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
expect(rejection).toBeInstanceOf(Error);
|
||||
expect((rejection as { status?: unknown }).status).toBe(401);
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves transient subscription credential resolution errors", async () => {
|
||||
const transientError = Object.assign(new Error("temporary refresh failure"), { status: 503 });
|
||||
oauthMocks.refreshOpenAICodexToken.mockRejectedValueOnce(transientError);
|
||||
const request = vi.fn();
|
||||
|
||||
await expect(
|
||||
applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
authProfileId: "openai:work",
|
||||
authProfileStore: {
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:work": {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "placeholder",
|
||||
refresh: "placeholder",
|
||||
expires: Date.now() - 60_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
authRequirement: "subscription",
|
||||
}),
|
||||
).rejects.toBe(transientError);
|
||||
expect(request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("accepts native ChatGPT auth for subscription routes", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
account: { type: "chatgpt", email: null, planType: "plus" },
|
||||
requiresOpenaiAuth: true,
|
||||
}));
|
||||
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
authProfileId: null,
|
||||
authRequirement: "subscription",
|
||||
});
|
||||
|
||||
expect(request).toHaveBeenCalledOnce();
|
||||
expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false });
|
||||
});
|
||||
|
||||
it("rejects native API-key auth for subscription routes", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
account: { type: "apiKey" },
|
||||
requiresOpenaiAuth: false,
|
||||
}));
|
||||
|
||||
await expect(
|
||||
applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
authProfileId: null,
|
||||
authRequirement: "subscription",
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 401 });
|
||||
expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false });
|
||||
});
|
||||
|
||||
it("rejects missing native auth for subscription routes", async () => {
|
||||
const request = vi.fn(async () => ({ account: null, requiresOpenaiAuth: true }));
|
||||
|
||||
await expect(
|
||||
applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
authProfileId: null,
|
||||
authRequirement: "subscription",
|
||||
}),
|
||||
).rejects.toMatchObject({ status: 401 });
|
||||
expect(request).toHaveBeenCalledWith("account/read", { refreshToken: false });
|
||||
});
|
||||
|
||||
it("falls back to CODEX_API_KEY when no auth profile and no Codex account is available", async () => {
|
||||
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-codex-app-server-"));
|
||||
const request = vi.fn(async (method: string) => {
|
||||
@@ -1979,15 +2085,16 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authRequirement: "api-key",
|
||||
startOptions: createStartOptions({
|
||||
env: { CODEX_API_KEY: "configured-codex-api-key" },
|
||||
env: { CODEX_API_KEY: "test-token-placeholder" },
|
||||
}),
|
||||
});
|
||||
|
||||
expect(request).toHaveBeenNthCalledWith(1, "account/read", { refreshToken: false });
|
||||
expect(request).toHaveBeenNthCalledWith(2, "account/login/start", {
|
||||
type: "apiKey",
|
||||
apiKey: "configured-codex-api-key",
|
||||
apiKey: "test-token-placeholder",
|
||||
});
|
||||
} finally {
|
||||
await fs.rm(agentDir, { recursive: true, force: true });
|
||||
@@ -2008,6 +2115,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authRequirement: "api-key",
|
||||
startOptions: createStartOptions(),
|
||||
});
|
||||
|
||||
@@ -2037,6 +2145,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authRequirement: "api-key",
|
||||
startOptions: createStartOptions(),
|
||||
});
|
||||
|
||||
@@ -2060,6 +2169,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authRequirement: "api-key",
|
||||
startOptions: createStartOptions(),
|
||||
});
|
||||
|
||||
@@ -2092,6 +2202,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authRequirement: "api-key",
|
||||
startOptions: createStartOptions({
|
||||
env: { CODEX_HOME: path.join(root, "isolated-codex-home") },
|
||||
}),
|
||||
@@ -2173,6 +2284,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authRequirement: "api-key",
|
||||
startOptions: createStartOptions({
|
||||
clearEnv: ["CODEX_API_KEY", "OPENAI_API_KEY"],
|
||||
}),
|
||||
@@ -2198,6 +2310,7 @@ describe("bridgeCodexAppServerStartOptions", () => {
|
||||
await applyCodexAppServerAuthProfile({
|
||||
client: { request } as never,
|
||||
agentDir,
|
||||
authRequirement: "api-key",
|
||||
startOptions: createStartOptions({
|
||||
transport: "websocket",
|
||||
url: "ws://127.0.0.1:1455",
|
||||
|
||||
@@ -29,10 +29,11 @@ import {
|
||||
resolveCodexComputerUseConfig,
|
||||
type CodexAppServerStartOptions,
|
||||
} from "./config.js";
|
||||
import type {
|
||||
CodexChatgptAuthTokensRefreshResponse,
|
||||
CodexGetAccountResponse,
|
||||
CodexLoginAccountParams,
|
||||
import {
|
||||
isJsonObject,
|
||||
type CodexChatgptAuthTokensRefreshResponse,
|
||||
type CodexGetAccountResponse,
|
||||
type CodexLoginAccountParams,
|
||||
} from "./protocol.js";
|
||||
import { isCodexAppServerNativeAuthProfile } from "./session-binding.js";
|
||||
import { resolveCodexAppServerSpawnEnv } from "./transport-stdio.js";
|
||||
@@ -61,6 +62,7 @@ const CODEX_APP_SERVER_HOME_ENV_VARS = [CODEX_HOME_ENV_VAR, HOME_ENV_VAR];
|
||||
const CODEX_AUTH_JSON_FILENAME = "auth.json";
|
||||
const CODEX_HOME_DIRNAME = ".codex";
|
||||
type AuthProfileOrderConfig = Parameters<typeof resolveAuthProfileOrder>[0]["cfg"];
|
||||
export type CodexAppServerAuthRequirement = "api-key" | "subscription";
|
||||
const scopedOAuthRefreshQueues = new WeakMap<
|
||||
AuthProfileStore,
|
||||
Map<string, Promise<OAuthCredential>>
|
||||
@@ -250,7 +252,7 @@ export async function resolveCodexAppServerPreparedAuthProfileSnapshot(params: {
|
||||
|
||||
/** Maps one prepared route to one mutually exclusive app-server auth handoff. */
|
||||
export async function resolveCodexAppServerPreparedAuthHandoff(params: {
|
||||
authRequirement?: "api-key" | "subscription";
|
||||
authRequirement?: CodexAppServerAuthRequirement;
|
||||
resolvedApiKey?: string;
|
||||
authProfileId?: string;
|
||||
authProfileStore: AuthProfileStore;
|
||||
@@ -281,7 +283,7 @@ export async function resolveCodexAppServerPreparedAuthHandoff(params: {
|
||||
return { authProfileId, nativeAuthProfile };
|
||||
}
|
||||
if (!authProfileId || !nativeAuthProfile) {
|
||||
throw new Error(params.subscriptionProfileRequiredError);
|
||||
throw createCodexAppServerAuthError(params.subscriptionProfileRequiredError);
|
||||
}
|
||||
|
||||
const snapshot = await resolveCodexAppServerPreparedAuthProfileSnapshot({
|
||||
@@ -291,7 +293,7 @@ export async function resolveCodexAppServerPreparedAuthHandoff(params: {
|
||||
config: params.config,
|
||||
});
|
||||
if (!snapshot) {
|
||||
throw new Error(params.subscriptionProfileUnusableError);
|
||||
throw createCodexAppServerAuthError(params.subscriptionProfileUnusableError);
|
||||
}
|
||||
return {
|
||||
authProfileId,
|
||||
@@ -474,6 +476,7 @@ export async function applyCodexAppServerAuthProfile(params: {
|
||||
authProfileId?: string | null;
|
||||
authProfileStore?: AuthProfileStore;
|
||||
preparedAuth?: CodexAppServerResolvedPreparedAuth;
|
||||
authRequirement?: CodexAppServerAuthRequirement;
|
||||
startOptions?: CodexAppServerStartOptions;
|
||||
config?: AuthProfileOrderConfig;
|
||||
}): Promise<void> {
|
||||
@@ -489,16 +492,52 @@ export async function applyCodexAppServerAuthProfile(params: {
|
||||
return;
|
||||
}
|
||||
if (params.authProfileId === null) {
|
||||
if (params.authRequirement === "subscription") {
|
||||
const response = await params.client.request<CodexGetAccountResponse>("account/read", {
|
||||
refreshToken: false,
|
||||
});
|
||||
if (!isJsonObject(response.account) || response.account.type !== "chatgpt") {
|
||||
throw createCodexAppServerAuthError(
|
||||
"Codex subscription auth profile could not produce login credentials.",
|
||||
);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
const loginParams = await resolveCodexAppServerAuthProfileLoginParams({
|
||||
agentDir: params.agentDir,
|
||||
authProfileId: params.authProfileId,
|
||||
authProfileStore: params.authProfileStore,
|
||||
config: params.config,
|
||||
});
|
||||
let loginParams: CodexLoginAccountParams | undefined;
|
||||
try {
|
||||
loginParams = await resolveCodexAppServerAuthProfileLoginParams({
|
||||
agentDir: params.agentDir,
|
||||
authProfileId: params.authProfileId,
|
||||
authProfileStore: params.authProfileStore,
|
||||
config: params.config,
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
params.authRequirement === "subscription" &&
|
||||
error instanceof CodexAppServerAuthProfileUnavailableError
|
||||
) {
|
||||
throw createCodexAppServerAuthError(
|
||||
"Codex subscription auth profile could not produce login credentials.",
|
||||
error,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (params.authRequirement === "subscription" && loginParams?.type !== "chatgptAuthTokens") {
|
||||
throw createCodexAppServerAuthError(
|
||||
"Codex subscription auth profile could not produce login credentials.",
|
||||
);
|
||||
}
|
||||
if (!loginParams) {
|
||||
if (params.startOptions?.transport !== "stdio") {
|
||||
// Observe native state only for explicit API-key routes. A subscription
|
||||
// route must fail here so profile rotation can run before billing changes.
|
||||
if (params.authRequirement === "subscription") {
|
||||
throw createCodexAppServerAuthError(
|
||||
"Codex subscription auth profile could not produce login credentials.",
|
||||
);
|
||||
}
|
||||
if (params.authRequirement !== "api-key" || params.startOptions?.transport !== "stdio") {
|
||||
return;
|
||||
}
|
||||
const env = resolveCodexAppServerSpawnEnv(params.startOptions, process.env);
|
||||
@@ -515,13 +554,40 @@ export async function applyCodexAppServerAuthProfile(params: {
|
||||
await params.client.request("account/login/start", loginParams);
|
||||
}
|
||||
|
||||
function resolveCodexAppServerAuthProfileLoginParams(params: {
|
||||
function createCodexAppServerAuthError(message: string, cause?: unknown): Error & { status: 401 } {
|
||||
const error = cause === undefined ? new Error(message) : new Error(message, { cause });
|
||||
return Object.assign(error, { status: 401 as const });
|
||||
}
|
||||
|
||||
class CodexAppServerAuthProfileUnavailableError extends Error {}
|
||||
|
||||
async function resolveCodexAppServerAuthProfileLoginParams(params: {
|
||||
agentDir: string;
|
||||
authProfileId?: string;
|
||||
authProfileStore?: AuthProfileStore;
|
||||
config?: AuthProfileOrderConfig;
|
||||
}): Promise<CodexLoginAccountParams | undefined> {
|
||||
return resolveCodexAppServerAuthProfileLoginParamsInternal(params);
|
||||
const store = resolveCodexAppServerAuthProfileStore(params);
|
||||
const profileId = resolveCodexAppServerAuthProfileId({
|
||||
authProfileId: params.authProfileId,
|
||||
store,
|
||||
config: params.config,
|
||||
});
|
||||
const profile = profileId ? store.profiles[profileId] : undefined;
|
||||
if (profileId && !profile) {
|
||||
throw new CodexAppServerAuthProfileUnavailableError(
|
||||
`Codex app-server auth profile "${profileId}" was not found.`,
|
||||
);
|
||||
}
|
||||
if (profileId && profile && !isCodexAppServerAuthProfileCredential(profile, params.config)) {
|
||||
throw new CodexAppServerAuthProfileUnavailableError(
|
||||
`Codex app-server auth profile "${profileId}" must be OpenAI Codex auth or an OpenAI API-key backup.`,
|
||||
);
|
||||
}
|
||||
return await resolveCodexAppServerAuthProfileLoginParamsInternal({
|
||||
...params,
|
||||
authProfileStore: store,
|
||||
});
|
||||
}
|
||||
|
||||
export async function refreshCodexAppServerAuthTokens(params: {
|
||||
@@ -582,7 +648,7 @@ async function resolveCodexAppServerAuthProfileLoginParamsInternal(params: {
|
||||
config: params.config,
|
||||
});
|
||||
if (!loginParams) {
|
||||
throw new Error(
|
||||
throw new CodexAppServerAuthProfileUnavailableError(
|
||||
`Codex app-server auth profile "${profileId}" does not contain usable credentials.`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -140,6 +140,13 @@ function buildEmptyToolTelemetry(): CodexAppServerToolTelemetry {
|
||||
};
|
||||
}
|
||||
|
||||
function expectUsageLimitPromptError(value: unknown): Error & { status: 429 } {
|
||||
expect(value).toBeInstanceOf(Error);
|
||||
const error = value as Error & { status?: unknown };
|
||||
expect(error.status).toBe(429);
|
||||
return error as Error & { status: 429 };
|
||||
}
|
||||
|
||||
function requireRecord(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error(`Expected ${label}`);
|
||||
@@ -1851,9 +1858,10 @@ describe("CodexAppServerEventProjector", () => {
|
||||
|
||||
const result = projector.buildResult(buildEmptyToolTelemetry());
|
||||
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
expect(result.promptError).toContain("Wait until the reset time");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Next reset in");
|
||||
expect(promptError.message).toContain("Wait until the reset time");
|
||||
expect(result.promptErrorSource).toBe("prompt");
|
||||
});
|
||||
|
||||
@@ -1880,8 +1888,9 @@ describe("CodexAppServerEventProjector", () => {
|
||||
|
||||
const result = projector.buildResult(buildEmptyToolTelemetry());
|
||||
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Next reset in");
|
||||
expect(result.promptErrorSource).toBe("prompt");
|
||||
});
|
||||
|
||||
@@ -1920,8 +1929,9 @@ describe("CodexAppServerEventProjector", () => {
|
||||
|
||||
const result = projector.buildResult(buildEmptyToolTelemetry());
|
||||
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Next reset in");
|
||||
expect(result.promptErrorSource).toBe("prompt");
|
||||
});
|
||||
|
||||
@@ -1946,9 +1956,10 @@ describe("CodexAppServerEventProjector", () => {
|
||||
|
||||
const result = projector.buildResult(buildEmptyToolTelemetry());
|
||||
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Codex says to try again at May 11th, 2026 9:00 AM.");
|
||||
expect(result.promptError).not.toContain("Codex did not return a reset time");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Codex says to try again at May 11th, 2026 9:00 AM.");
|
||||
expect(promptError.message).not.toContain("Codex did not return a reset time");
|
||||
expect(result.promptErrorSource).toBe("prompt");
|
||||
});
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ import {
|
||||
import { formatCodexUsageLimitErrorMessage } from "./rate-limits.js";
|
||||
import type { CodexTrajectoryRecorder } from "./trajectory.js";
|
||||
import { attachCodexMirrorIdentity } from "./upstream-prompt-provenance.js";
|
||||
import { createCodexUsageLimitPromptError } from "./usage-limit-error.js";
|
||||
import { promptSnapshot } from "./user-prompt-message.js";
|
||||
|
||||
export { CodexNativeToolLifecycleProjector };
|
||||
@@ -575,14 +576,14 @@ export class CodexAppServerEventProjector {
|
||||
}
|
||||
this.completedTurn = turn;
|
||||
if (turn.status === "failed") {
|
||||
this.promptError =
|
||||
formatCodexUsageLimitErrorMessage({
|
||||
message: turn.error?.message,
|
||||
codexErrorInfo: turn.error?.codexErrorInfo as JsonValue | null | undefined,
|
||||
rateLimits: this.options.readRecentRateLimits?.(),
|
||||
}) ??
|
||||
turn.error?.message ??
|
||||
"codex app-server turn failed";
|
||||
const usageLimitMessage = formatCodexUsageLimitErrorMessage({
|
||||
message: turn.error?.message,
|
||||
codexErrorInfo: turn.error?.codexErrorInfo as JsonValue | null | undefined,
|
||||
rateLimits: this.options.readRecentRateLimits?.(),
|
||||
});
|
||||
this.promptError = usageLimitMessage
|
||||
? createCodexUsageLimitPromptError(usageLimitMessage)
|
||||
: (turn.error?.message ?? "codex app-server turn failed");
|
||||
this.promptErrorSource = "prompt";
|
||||
}
|
||||
const turnItems = turn.items ?? [];
|
||||
@@ -674,15 +675,16 @@ export class CodexAppServerEventProjector {
|
||||
});
|
||||
}
|
||||
|
||||
private formatCodexErrorMessage(params: JsonObject): string | undefined {
|
||||
private formatCodexErrorMessage(params: JsonObject): string | Error | undefined {
|
||||
const error = isJsonObject(params.error) ? params.error : undefined;
|
||||
return (
|
||||
formatCodexUsageLimitErrorMessage({
|
||||
message: error ? readString(error, "message") : undefined,
|
||||
codexErrorInfo: error?.codexErrorInfo,
|
||||
rateLimits: this.options.readRecentRateLimits?.(),
|
||||
}) ?? readCodexErrorNotificationMessage(params)
|
||||
);
|
||||
const usageLimitMessage = formatCodexUsageLimitErrorMessage({
|
||||
message: error ? readString(error, "message") : undefined,
|
||||
codexErrorInfo: error?.codexErrorInfo,
|
||||
rateLimits: this.options.readRecentRateLimits?.(),
|
||||
});
|
||||
return usageLimitMessage
|
||||
? createCodexUsageLimitPromptError(usageLimitMessage)
|
||||
: readCodexErrorNotificationMessage(params);
|
||||
}
|
||||
|
||||
private emitAgentEvent(
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
* endpoint, including pagination and shared-client lease handling.
|
||||
*/
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type { resolveCodexAppServerAuthProfileIdForAgent } from "./auth-bridge.js";
|
||||
import type {
|
||||
CodexAppServerAuthRequirement,
|
||||
resolveCodexAppServerAuthProfileIdForAgent,
|
||||
} from "./auth-bridge.js";
|
||||
import type { CodexAppServerClient } from "./client.js";
|
||||
import type { CodexAppServerStartOptions } from "./config.js";
|
||||
import { readCodexModelListResponse } from "./protocol-validators.js";
|
||||
@@ -37,6 +40,7 @@ export type CodexAppServerListModelsOptions = {
|
||||
timeoutMs?: number;
|
||||
startOptions?: CodexAppServerStartOptions;
|
||||
authProfileId?: string;
|
||||
authRequirement?: CodexAppServerAuthRequirement;
|
||||
agentDir?: string;
|
||||
config?: Parameters<typeof resolveCodexAppServerAuthProfileIdForAgent>[0]["config"];
|
||||
sharedClient?: boolean;
|
||||
@@ -93,6 +97,7 @@ async function withCodexAppServerModelClient<T>(
|
||||
startOptions: options.startOptions,
|
||||
timeoutMs,
|
||||
authProfileId: options.authProfileId,
|
||||
authRequirement: options.authRequirement,
|
||||
agentDir: options.agentDir,
|
||||
config: options.config,
|
||||
})
|
||||
@@ -100,6 +105,7 @@ async function withCodexAppServerModelClient<T>(
|
||||
startOptions: options.startOptions,
|
||||
timeoutMs,
|
||||
authProfileId: options.authProfileId,
|
||||
authRequirement: options.authRequirement,
|
||||
agentDir: options.agentDir,
|
||||
config: options.config,
|
||||
});
|
||||
|
||||
@@ -4,11 +4,33 @@ import {
|
||||
buildCodexAppServerUsageSnapshot,
|
||||
formatCodexUsageLimitErrorMessage,
|
||||
resolveCodexUsageLimitResetAtMs,
|
||||
shouldRefreshCodexRateLimitsForUsageLimitMessage,
|
||||
summarizeCodexAccountUsage,
|
||||
summarizeCodexRateLimits,
|
||||
} from "./rate-limits.js";
|
||||
|
||||
describe("formatCodexUsageLimitErrorMessage", () => {
|
||||
it("does not infer a Codex usage limit from unrelated prose", () => {
|
||||
expect(
|
||||
formatCodexUsageLimitErrorMessage({
|
||||
message: "The workspace usage limit setting could not be loaded.",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(shouldRefreshCodexRateLimitsForUsageLimitMessage("temporary usage limit warning")).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts normalized structured Codex usage-limit error info", () => {
|
||||
const message = formatCodexUsageLimitErrorMessage({
|
||||
message: "quota exhausted",
|
||||
codexErrorInfo: "usage_limit-exceeded",
|
||||
});
|
||||
|
||||
expect(message?.startsWith("You've reached your Codex subscription usage limit.")).toBe(true);
|
||||
expect(shouldRefreshCodexRateLimitsForUsageLimitMessage(message)).toBe(true);
|
||||
});
|
||||
|
||||
it("gives actionable guidance when Codex omits reset details", () => {
|
||||
const message = formatCodexUsageLimitErrorMessage({
|
||||
message: "You've reached your usage limit.",
|
||||
|
||||
@@ -25,6 +25,7 @@ const ONE_DAY_MS = 24 * ONE_HOUR_MS;
|
||||
const DAY_WINDOW_MINUTES = 24 * 60;
|
||||
const WEEKLY_WINDOW_MINUTES = 7 * DAY_WINDOW_MINUTES;
|
||||
const WEEKLY_RESET_GAP_MS = 3 * ONE_DAY_MS;
|
||||
const CODEX_USAGE_LIMIT_MESSAGE_PREFIX = "You've reached your Codex subscription usage limit.";
|
||||
|
||||
type LimitWindowKey = (typeof LIMIT_WINDOW_KEYS)[number];
|
||||
|
||||
@@ -58,7 +59,7 @@ export function formatCodexUsageLimitErrorMessage(params: {
|
||||
nowMs?: number;
|
||||
}): string | undefined {
|
||||
const message = normalizeText(params.message);
|
||||
if (!isCodexUsageLimitError(params.codexErrorInfo, message)) {
|
||||
if (!isCodexUsageLimitError(params.codexErrorInfo)) {
|
||||
return undefined;
|
||||
}
|
||||
const nowMs = params.nowMs ?? Date.now();
|
||||
@@ -67,7 +68,7 @@ export function formatCodexUsageLimitErrorMessage(params: {
|
||||
const nextReset =
|
||||
blockingReset ??
|
||||
(usageSummary?.blocked ? undefined : selectNextRateLimitReset(params.rateLimits, nowMs));
|
||||
const parts = ["You've reached your Codex subscription usage limit."];
|
||||
const parts = [CODEX_USAGE_LIMIT_MESSAGE_PREFIX];
|
||||
let recoveryAction = "Wait until Codex becomes available";
|
||||
if (nextReset) {
|
||||
parts.push(`Next reset ${formatResetTime(nextReset.resetsAtMs, nowMs)}.`);
|
||||
@@ -95,9 +96,10 @@ export function shouldRefreshCodexRateLimitsForUsageLimitMessage(
|
||||
message: string | null | undefined,
|
||||
): boolean {
|
||||
const text = normalizeText(message);
|
||||
// Only our formatted prefix is a refresh contract. Provider prose alone is
|
||||
// not structural evidence of a Codex usage-limit failure.
|
||||
return Boolean(
|
||||
text?.includes("You've reached your Codex subscription usage limit.") &&
|
||||
!text.includes("Next reset "),
|
||||
text?.startsWith(CODEX_USAGE_LIMIT_MESSAGE_PREFIX) && !text.includes("Next reset "),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -210,10 +212,7 @@ export function buildCodexAppServerUsageSnapshot(value: unknown): ProviderUsageS
|
||||
};
|
||||
}
|
||||
|
||||
function isCodexUsageLimitError(
|
||||
codexErrorInfo: JsonValue | null | undefined,
|
||||
message: string | undefined,
|
||||
): boolean {
|
||||
function isCodexUsageLimitError(codexErrorInfo: JsonValue | null | undefined): boolean {
|
||||
if (codexErrorInfo === "usageLimitExceeded") {
|
||||
return true;
|
||||
}
|
||||
@@ -223,7 +222,7 @@ function isCodexUsageLimitError(
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return Boolean(message?.toLowerCase().includes("usage limit"));
|
||||
return false;
|
||||
}
|
||||
|
||||
function selectNextRateLimitReset(
|
||||
|
||||
@@ -365,6 +365,7 @@ export async function prepareCodexAttemptConnection({ params, options }: CodexRu
|
||||
isInactiveThreadBootstrapBinding,
|
||||
usesSupervisionConnection,
|
||||
startupAuthProfileId,
|
||||
startupAuthRequirement: preparedAuthRoute?.authRequirement,
|
||||
startupPreparedAuth,
|
||||
startupClientAuthProfileId,
|
||||
effectiveWorkspace,
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
isInvalidCodexImagePayloadError,
|
||||
resolveCodexAppServerReplayBlockedReason,
|
||||
} from "./attempt-results.js";
|
||||
import { readCodexRateLimitsRevision, readRecentCodexRateLimits } from "./rate-limit-cache.js";
|
||||
import type { CodexAttemptActiveTurn } from "./run-attempt-active-turn.js";
|
||||
import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js";
|
||||
import {
|
||||
@@ -36,7 +37,12 @@ import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.
|
||||
import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js";
|
||||
import { normalizeCodexTrajectoryError, recordCodexTrajectoryCompletion } from "./trajectory.js";
|
||||
import { codexTranscriptMirrorRuntime } from "./transcript-mirror.js";
|
||||
import { refreshCodexUsageLimitPromptError } from "./usage-limit-error.js";
|
||||
import {
|
||||
createCodexUsageLimitPromptError,
|
||||
isCodexUsageLimitPromptError,
|
||||
markCodexAuthProfileBlockedFromRateLimits,
|
||||
refreshCodexUsageLimitPromptError,
|
||||
} from "./usage-limit-error.js";
|
||||
|
||||
export async function finalizeCodexAttempt(
|
||||
resources: CodexAttemptResources,
|
||||
@@ -72,6 +78,7 @@ export async function finalizeCodexAttempt(
|
||||
effectiveWorkspace,
|
||||
agentDir,
|
||||
attemptStartedAt,
|
||||
startupAuthProfileId,
|
||||
} = connection;
|
||||
const { toolBridge, toolState } = attemptTools;
|
||||
const {
|
||||
@@ -158,9 +165,11 @@ export async function finalizeCodexAttempt(
|
||||
const finalPromptErrorMessage =
|
||||
typeof finalPromptError === "string"
|
||||
? finalPromptError
|
||||
: finalPromptError
|
||||
? formatErrorMessage(finalPromptError)
|
||||
: undefined;
|
||||
: finalPromptError instanceof Error
|
||||
? finalPromptError.message
|
||||
: finalPromptError
|
||||
? formatErrorMessage(finalPromptError)
|
||||
: undefined;
|
||||
if (isInvalidCodexImagePayloadError(finalPromptErrorMessage)) {
|
||||
await clearCodexBindingAfterInvalidImagePayload(bindingStore, bindingIdentity, {
|
||||
phase: "turn_completed",
|
||||
@@ -197,7 +206,22 @@ export async function finalizeCodexAttempt(
|
||||
signal: runAbortController.signal,
|
||||
});
|
||||
if (refreshedUsageLimitPromptError) {
|
||||
finalPromptError = refreshedUsageLimitPromptError;
|
||||
await markCodexAuthProfileBlockedFromRateLimits({
|
||||
params,
|
||||
authProfileId: startupAuthProfileId,
|
||||
rateLimits: refreshedUsageLimitPromptError.rateLimitsForProfile,
|
||||
});
|
||||
finalPromptError = createCodexUsageLimitPromptError(refreshedUsageLimitPromptError.message);
|
||||
} else if (
|
||||
isCodexUsageLimitPromptError(finalPromptError) &&
|
||||
state.rateLimitsRevisionBeforeLastTurnStart !== undefined &&
|
||||
readCodexRateLimitsRevision(resourceState.client) > state.rateLimitsRevisionBeforeLastTurnStart
|
||||
) {
|
||||
await markCodexAuthProfileBlockedFromRateLimits({
|
||||
params,
|
||||
authProfileId: startupAuthProfileId,
|
||||
rateLimits: readRecentCodexRateLimits(resourceState.client),
|
||||
});
|
||||
}
|
||||
const finalPromptErrorSource =
|
||||
effectiveTimedOut || clientClosedPromptErrorForFinal ? "prompt" : result.promptErrorSource;
|
||||
|
||||
@@ -62,6 +62,7 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources)
|
||||
resolveReviewerPolicyContext,
|
||||
resolveRuntimeOptionsForCurrentBinding,
|
||||
startupAuthProfileId,
|
||||
startupAuthRequirement,
|
||||
abortFromUpstream,
|
||||
} = connection;
|
||||
let pluginAppServer = withCodexAppServerFastModeServiceTier(appServer, runtimeParams);
|
||||
@@ -77,6 +78,7 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources)
|
||||
pluginConfig,
|
||||
computerUseConfig,
|
||||
startupAuthProfileId: startupClientAuthProfileId,
|
||||
startupAuthRequirement,
|
||||
startupAuthBindingFingerprint: preparedAuthBinding?.fingerprint,
|
||||
...(runtimeArtifactRequest ? { runtimeArtifactRequest } : {}),
|
||||
startupPreparedAuth,
|
||||
|
||||
@@ -29,6 +29,7 @@ import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.
|
||||
import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js";
|
||||
import { buildCodexUserPromptMessage } from "./transcript-mirror.js";
|
||||
import {
|
||||
createCodexUsageLimitPromptError,
|
||||
formatCodexTurnStartUsageLimitError,
|
||||
markCodexAuthProfileBlockedFromRateLimits,
|
||||
} from "./usage-limit-error.js";
|
||||
@@ -262,6 +263,7 @@ export async function startCodexAttemptTurn(
|
||||
result: buildCodexTurnStartFailureResult({
|
||||
params,
|
||||
message: usageLimitError.message,
|
||||
promptError: createCodexUsageLimitPromptError(usageLimitError.message),
|
||||
messagesSnapshot,
|
||||
systemPromptReport,
|
||||
}),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Codex tests cover run attempt.usage limits plugin behavior.
|
||||
import path from "node:path";
|
||||
import { saveAuthProfileStore } from "openclaw/plugin-sdk/agent-runtime";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readCodexRateLimitsRevision, rememberCodexRateLimitsRead } from "./rate-limit-cache.js";
|
||||
import {
|
||||
@@ -13,6 +14,13 @@ import {
|
||||
|
||||
setupRunAttemptTestHooks();
|
||||
|
||||
function expectUsageLimitPromptError(value: unknown): Error & { status: 429 } {
|
||||
expect(value).toBeInstanceOf(Error);
|
||||
const error = value as Error & { status?: unknown };
|
||||
expect(error.status).toBe(429);
|
||||
return error as Error & { status: 429 };
|
||||
}
|
||||
|
||||
describe("runCodexAppServerAttempt usage limits", () => {
|
||||
it("preserves Codex usage-limit reset details when turn/start fails", async () => {
|
||||
const sessionFile = path.join(tempDir, "session.jsonl");
|
||||
@@ -39,6 +47,7 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
harnessRef.current = harness;
|
||||
|
||||
const params = createParams(sessionFile, workspaceDir);
|
||||
params.agentDir = path.join(tempDir, "agent");
|
||||
params.authProfileId = authProfileId;
|
||||
params.authProfileStore = {
|
||||
version: 1,
|
||||
@@ -46,8 +55,8 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
[authProfileId]: {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "access",
|
||||
refresh: "refresh",
|
||||
access: "placeholder",
|
||||
refresh: "placeholder",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
@@ -55,8 +64,9 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
|
||||
const result = await runCodexAppServerAttempt(params);
|
||||
expect(result.promptErrorSource).toBe("prompt");
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Next reset in");
|
||||
});
|
||||
|
||||
it("uses a recent Codex rate-limit snapshot when turn/start omits reset details", async () => {
|
||||
@@ -93,8 +103,8 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
[authProfileId]: {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "access",
|
||||
refresh: "refresh",
|
||||
access: "placeholder",
|
||||
refresh: "placeholder",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
@@ -105,8 +115,9 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
|
||||
const result = await run;
|
||||
expect(result.promptErrorSource).toBe("prompt");
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Next reset in");
|
||||
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -153,8 +164,8 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
[authProfileId]: {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "access",
|
||||
refresh: "refresh",
|
||||
access: "placeholder",
|
||||
refresh: "placeholder",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
@@ -162,7 +173,7 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
|
||||
const result = await runCodexAppServerAttempt(params);
|
||||
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in");
|
||||
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -187,15 +198,17 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
|
||||
const result = await run;
|
||||
expect(result.promptErrorSource).toBe("prompt");
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
expect(result.promptError).not.toContain("Codex did not return a reset time");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Next reset in");
|
||||
expect(promptError.message).not.toContain("Codex did not return a reset time");
|
||||
});
|
||||
|
||||
it("refreshes Codex account rate limits when a failed turn omits reset details", async () => {
|
||||
const sessionFile = path.join(tempDir, "session.jsonl");
|
||||
const workspaceDir = path.join(tempDir, "workspace");
|
||||
const resetsAt = Math.ceil(Date.now() / 1000) + 120;
|
||||
const authProfileId = "openai:work";
|
||||
const harness = createStartedThreadHarness(async (method) => {
|
||||
if (method === "account/rateLimits/read") {
|
||||
return rateLimitsUpdated(resetsAt).params;
|
||||
@@ -203,7 +216,23 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const run = runCodexAppServerAttempt(createParams(sessionFile, workspaceDir));
|
||||
const params = createParams(sessionFile, workspaceDir);
|
||||
params.agentDir = path.join(tempDir, "streamed-usage-limit-agent");
|
||||
params.authProfileId = authProfileId;
|
||||
params.authProfileStore = {
|
||||
version: 1,
|
||||
profiles: {
|
||||
[authProfileId]: {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "placeholder",
|
||||
refresh: "placeholder",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
};
|
||||
saveAuthProfileStore(params.authProfileStore, params.agentDir);
|
||||
const run = runCodexAppServerAttempt(params);
|
||||
await harness.waitForMethod("turn/start");
|
||||
await harness.notify({
|
||||
method: "turn/completed",
|
||||
@@ -223,11 +252,114 @@ describe("runCodexAppServerAttempt usage limits", () => {
|
||||
|
||||
const result = await run;
|
||||
|
||||
expect(result.promptError).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(result.promptError).toContain("Next reset in");
|
||||
expect(result.promptError).not.toContain("Codex did not return a reset time");
|
||||
const promptError = expectUsageLimitPromptError(result.promptError);
|
||||
expect(promptError.message).toContain("You've reached your Codex subscription usage limit.");
|
||||
expect(promptError.message).toContain("Next reset in");
|
||||
expect(promptError.message).not.toContain("Codex did not return a reset time");
|
||||
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBe(resetsAt * 1000);
|
||||
expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("blocks after a streamed usage-limit failure with trusted in-turn limits", async () => {
|
||||
const sessionFile = path.join(tempDir, "session.jsonl");
|
||||
const workspaceDir = path.join(tempDir, "workspace");
|
||||
const resetsAt = Math.ceil(Date.now() / 1000) + 120;
|
||||
const authProfileId = "openai:work";
|
||||
const harness = createStartedThreadHarness(async () => undefined);
|
||||
const params = createParams(sessionFile, workspaceDir);
|
||||
params.agentDir = path.join(tempDir, "trusted-streamed-usage-limit-agent");
|
||||
params.authProfileId = authProfileId;
|
||||
params.authProfileStore = {
|
||||
version: 1,
|
||||
profiles: {
|
||||
[authProfileId]: {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "placeholder",
|
||||
refresh: "placeholder",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
};
|
||||
saveAuthProfileStore(params.authProfileStore, params.agentDir);
|
||||
|
||||
const run = runCodexAppServerAttempt(params);
|
||||
await harness.waitForMethod("turn/start");
|
||||
await harness.notify(rateLimitsUpdated(resetsAt));
|
||||
await harness.notify({
|
||||
method: "turn/completed",
|
||||
params: {
|
||||
threadId: "thread-1",
|
||||
turnId: "turn-1",
|
||||
turn: {
|
||||
id: "turn-1",
|
||||
status: "failed",
|
||||
error: {
|
||||
message: "You've reached your usage limit.",
|
||||
codexErrorInfo: "usageLimitExceeded",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await run;
|
||||
|
||||
expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in");
|
||||
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBe(resetsAt * 1000);
|
||||
expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not block after a streamed usage-limit failure with only stale limits", async () => {
|
||||
const sessionFile = path.join(tempDir, "session.jsonl");
|
||||
const workspaceDir = path.join(tempDir, "workspace");
|
||||
const resetsAt = Math.ceil(Date.now() / 1000) + 120;
|
||||
const authProfileId = "openai:work";
|
||||
const harness = createStartedThreadHarness(async () => undefined);
|
||||
rememberCodexRateLimitsRead(harness.client, rateLimitsUpdated(resetsAt).params);
|
||||
const params = createParams(sessionFile, workspaceDir);
|
||||
params.agentDir = path.join(tempDir, "stale-streamed-usage-limit-agent");
|
||||
params.authProfileId = authProfileId;
|
||||
params.authProfileStore = {
|
||||
version: 1,
|
||||
profiles: {
|
||||
[authProfileId]: {
|
||||
type: "oauth",
|
||||
provider: "openai",
|
||||
access: "placeholder",
|
||||
refresh: "placeholder",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const run = runCodexAppServerAttempt(params);
|
||||
await harness.waitForMethod("turn/start");
|
||||
await harness.notify({
|
||||
method: "turn/completed",
|
||||
params: {
|
||||
threadId: "thread-1",
|
||||
turnId: "turn-1",
|
||||
turn: {
|
||||
id: "turn-1",
|
||||
status: "failed",
|
||||
error: {
|
||||
message: "You've reached your usage limit.",
|
||||
codexErrorInfo: "usageLimitExceeded",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await run;
|
||||
|
||||
expect(expectUsageLimitPromptError(result.promptError).message).toContain("Next reset in");
|
||||
expect(params.authProfileStore.usageStats?.[authProfileId]?.blockedUntil).toBeUndefined();
|
||||
expect(harness.requests.some((request) => request.method === "account/rateLimits/read")).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1414,12 +1414,18 @@ describe("shared Codex app-server client", () => {
|
||||
.mockReturnValueOnce("api-key:first")
|
||||
.mockReturnValueOnce("api-key:second");
|
||||
|
||||
const firstList = listCodexAppServerModels({ timeoutMs: 1000 });
|
||||
const firstList = listCodexAppServerModels({
|
||||
timeoutMs: 1000,
|
||||
authRequirement: "api-key",
|
||||
});
|
||||
await sendInitializeResult(first, "openclaw/0.143.0 (macOS; test)");
|
||||
await sendEmptyModelList(first);
|
||||
await expect(firstList).resolves.toEqual({ models: [] });
|
||||
|
||||
const secondList = listCodexAppServerModels({ timeoutMs: 1000 });
|
||||
const secondList = listCodexAppServerModels({
|
||||
timeoutMs: 1000,
|
||||
authRequirement: "api-key",
|
||||
});
|
||||
await sendInitializeResult(second, "openclaw/0.143.0 (macOS; test)");
|
||||
await sendEmptyModelList(second);
|
||||
await expect(secondList).resolves.toEqual({ models: [] });
|
||||
@@ -1429,6 +1435,49 @@ describe("shared Codex app-server client", () => {
|
||||
expect(second.process.stdin.destroyed).toBe(false);
|
||||
});
|
||||
|
||||
it("does not share a client across auth requirements", async () => {
|
||||
const first = createClientHarness();
|
||||
const second = createClientHarness();
|
||||
const startSpy = vi
|
||||
.spyOn(CodexAppServerClient, "start")
|
||||
.mockReturnValueOnce(first.client)
|
||||
.mockReturnValueOnce(second.client);
|
||||
|
||||
const firstList = listCodexAppServerModels({
|
||||
timeoutMs: 1000,
|
||||
authProfileId: "openai:work",
|
||||
authRequirement: "api-key",
|
||||
});
|
||||
await sendInitializeResult(first, "openclaw/0.143.0 (macOS; test)");
|
||||
await sendEmptyModelList(first);
|
||||
await expect(firstList).resolves.toEqual({ models: [] });
|
||||
|
||||
const secondList = listCodexAppServerModels({
|
||||
timeoutMs: 1000,
|
||||
authProfileId: "openai:work",
|
||||
authRequirement: "subscription",
|
||||
});
|
||||
await sendInitializeResult(second, "openclaw/0.143.0 (macOS; test)");
|
||||
await sendEmptyModelList(second);
|
||||
await expect(secondList).resolves.toEqual({ models: [] });
|
||||
|
||||
expect(startSpy).toHaveBeenCalledTimes(2);
|
||||
expect(first.process.stdin.destroyed).toBe(false);
|
||||
expect(second.process.stdin.destroyed).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects prepared auth that conflicts with the auth requirement", async () => {
|
||||
const startSpy = vi.spyOn(CodexAppServerClient, "start");
|
||||
|
||||
await expect(
|
||||
getSharedCodexAppServerClient({
|
||||
authRequirement: "subscription",
|
||||
preparedAuth: { kind: "api-key", apiKey: "placeholder" },
|
||||
}),
|
||||
).rejects.toThrow("Prepared Codex auth does not satisfy the requested auth requirement.");
|
||||
expect(startSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not let one shared-client failure tear down another keyed client", async () => {
|
||||
const first = createClientHarness();
|
||||
const second = createClientHarness();
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
resolveCodexAppServerPreparedAuthProfileSnapshot,
|
||||
resolveCodexAppServerPreparedApiKeyCacheKey,
|
||||
type CodexAppServerPreparedAuth,
|
||||
type CodexAppServerAuthRequirement,
|
||||
type CodexAppServerResolvedPreparedAuth,
|
||||
} from "./auth-bridge.js";
|
||||
import { ensureCodexAppServerClientRuntime } from "./client-runtime.js";
|
||||
@@ -246,6 +247,7 @@ export type CodexAppServerClientOptions = {
|
||||
/** Previously minted exact runtime required before the process may start. */
|
||||
expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding;
|
||||
preparedAuth?: CodexAppServerPreparedAuth;
|
||||
authRequirement?: CodexAppServerAuthRequirement;
|
||||
agentDir?: string;
|
||||
config?: Parameters<typeof resolveCodexAppServerAuthProfileIdForAgent>[0]["config"];
|
||||
onStartedClient?: (client: CodexAppServerClient) => void;
|
||||
@@ -263,10 +265,20 @@ type ResolvedCodexAppServerClientStartContext = {
|
||||
authProfileId: string | undefined;
|
||||
authProfileStore: AuthProfileStore | undefined;
|
||||
preparedAuth: CodexAppServerResolvedPreparedAuth | undefined;
|
||||
authRequirement: CodexAppServerAuthRequirement | undefined;
|
||||
requestedStartOptions: CodexAppServerStartOptions;
|
||||
startOptions: CodexAppServerStartOptions;
|
||||
};
|
||||
|
||||
function inferAuthRequirement(
|
||||
preparedAuth: CodexAppServerPreparedAuth | undefined,
|
||||
): CodexAppServerAuthRequirement | undefined {
|
||||
if (preparedAuth?.kind === "api-key") {
|
||||
return "api-key";
|
||||
}
|
||||
return preparedAuth?.kind === "profile" ? "subscription" : undefined;
|
||||
}
|
||||
|
||||
async function resolveCodexAppServerClientStartContext(
|
||||
options?: CodexAppServerClientOptions,
|
||||
): Promise<ResolvedCodexAppServerClientStartContext> {
|
||||
@@ -287,6 +299,15 @@ async function resolveCodexAppServerClientStartContext(
|
||||
if (preparedAuth && requestedStartOptions.homeScope === "user") {
|
||||
throw new Error("Prepared Codex auth requires an isolated app-server home.");
|
||||
}
|
||||
const preparedAuthRequirement = inferAuthRequirement(preparedAuth);
|
||||
if (
|
||||
options?.authRequirement &&
|
||||
preparedAuthRequirement &&
|
||||
options.authRequirement !== preparedAuthRequirement
|
||||
) {
|
||||
throw new Error("Prepared Codex auth does not satisfy the requested auth requirement.");
|
||||
}
|
||||
const authRequirement = options?.authRequirement ?? preparedAuthRequirement;
|
||||
const usesNativeAuth =
|
||||
!preparedAuth &&
|
||||
(options?.authProfileId === null || requestedStartOptions.homeScope === "user");
|
||||
@@ -361,6 +382,7 @@ async function resolveCodexAppServerClientStartContext(
|
||||
authProfileStore,
|
||||
requestedStartOptions,
|
||||
preparedAuth: resolvedPreparedAuth,
|
||||
authRequirement,
|
||||
startOptions,
|
||||
};
|
||||
}
|
||||
@@ -500,6 +522,7 @@ async function acquireSharedCodexAppServerClient(
|
||||
authProfileId,
|
||||
authProfileStore,
|
||||
preparedAuth,
|
||||
authRequirement,
|
||||
requestedStartOptions,
|
||||
startOptions,
|
||||
} = context;
|
||||
@@ -508,15 +531,15 @@ async function acquireSharedCodexAppServerClient(
|
||||
preparedAuth?.kind === "api-key"
|
||||
? resolveCodexAppServerPreparedApiKeyCacheKey(preparedAuth.apiKey)
|
||||
: (preparedAuth?.snapshot.secretFreeCacheKey ??
|
||||
(authProfileId
|
||||
? undefined
|
||||
: resolveCodexAppServerFallbackApiKeyCacheKey({ startOptions })));
|
||||
const baseKey = codexAppServerStartOptionsKey(startOptions, {
|
||||
(authRequirement === "api-key" && !authProfileId
|
||||
? resolveCodexAppServerFallbackApiKeyCacheKey({ startOptions })
|
||||
: undefined));
|
||||
const baseKey = `${codexAppServerStartOptionsKey(startOptions, {
|
||||
authProfileId,
|
||||
authBindingFingerprint: options?.authBindingFingerprint,
|
||||
agentDir: usesNativeAuth ? undefined : agentDir,
|
||||
fallbackApiKeyCacheKey: authIdentityCacheKey,
|
||||
});
|
||||
})}\0auth-requirement:${authRequirement ?? "native"}`;
|
||||
// Capture turns cannot inherit a normal client whose loaded bytes predate the
|
||||
// filesystem snapshot. Keep their physical process generation separate.
|
||||
const runtimeArtifactMode =
|
||||
@@ -576,6 +599,7 @@ async function acquireSharedCodexAppServerClient(
|
||||
authProfileId: usesNativeAuth || preparedAuth?.kind === "api-key" ? null : authProfileId,
|
||||
authProfileStore,
|
||||
preparedAuth,
|
||||
authRequirement,
|
||||
runtimeArtifactMode,
|
||||
...(options?.expectedRuntimeArtifact
|
||||
? { expectedRuntimeArtifact: options.expectedRuntimeArtifact }
|
||||
@@ -665,6 +689,7 @@ function createSharedCodexAppServerClientStartup(params: {
|
||||
expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding;
|
||||
runtimeArtifactSignal?: AbortSignal;
|
||||
preparedAuth?: CodexAppServerResolvedPreparedAuth;
|
||||
authRequirement?: CodexAppServerAuthRequirement;
|
||||
config?: CodexAppServerClientOptions["config"];
|
||||
}): SharedCodexAppServerClientStartup {
|
||||
const initialized = createDeferred<void>();
|
||||
@@ -675,6 +700,7 @@ function createSharedCodexAppServerClientStartup(params: {
|
||||
authProfileId: params.authProfileId,
|
||||
authProfileStore: params.authProfileStore,
|
||||
preparedAuth: params.preparedAuth,
|
||||
authRequirement: params.authRequirement,
|
||||
runtimeArtifactMode: params.runtimeArtifactMode,
|
||||
...(params.expectedRuntimeArtifact
|
||||
? { expectedRuntimeArtifact: params.expectedRuntimeArtifact }
|
||||
@@ -722,6 +748,7 @@ export async function createIsolatedCodexAppServerClient(
|
||||
authProfileId,
|
||||
authProfileStore,
|
||||
preparedAuth,
|
||||
authRequirement,
|
||||
requestedStartOptions,
|
||||
startOptions,
|
||||
} = await withCodexAppServerAcquireDeadline(
|
||||
@@ -736,6 +763,7 @@ export async function createIsolatedCodexAppServerClient(
|
||||
authProfileId: usesNativeAuth || preparedAuth?.kind === "api-key" ? null : authProfileId,
|
||||
authProfileStore,
|
||||
preparedAuth,
|
||||
authRequirement,
|
||||
runtimeArtifactMode:
|
||||
options?.runtimeArtifactMode ?? (options?.expectedRuntimeArtifact ? "capture" : undefined),
|
||||
...(options?.expectedRuntimeArtifact
|
||||
@@ -759,6 +787,7 @@ async function startInitializedCodexAppServerClient(params: {
|
||||
expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding;
|
||||
runtimeArtifactSignal?: AbortSignal;
|
||||
preparedAuth?: CodexAppServerResolvedPreparedAuth;
|
||||
authRequirement?: CodexAppServerAuthRequirement;
|
||||
config?: CodexAppServerClientOptions["config"];
|
||||
timeoutMs?: number;
|
||||
abandonSignal?: AbortSignal;
|
||||
@@ -859,6 +888,7 @@ async function startInitializedCodexAppServerClient(params: {
|
||||
agentDir: params.agentDir,
|
||||
authProfileId: params.authProfileId,
|
||||
preparedAuth: params.preparedAuth,
|
||||
authRequirement: params.authRequirement,
|
||||
startOptions,
|
||||
config: params.config,
|
||||
...(params.authProfileStore ? { authProfileStore: params.authProfileStore } : {}),
|
||||
|
||||
@@ -288,6 +288,7 @@ export async function runCodexAppServerSideQuestion(
|
||||
const clientOptions = {
|
||||
startOptions: appServer.start,
|
||||
timeoutMs: appServer.requestTimeoutMs,
|
||||
authRequirement: preparedRuntimeAuth.plan.modelRoute?.authRequirement,
|
||||
...(startupPreparedAuth
|
||||
? { preparedAuth: startupPreparedAuth }
|
||||
: { authProfileId: connection.clientAuthProfileId }),
|
||||
|
||||
@@ -41,6 +41,14 @@ type CodexUsageLimitErrorResult = {
|
||||
rateLimitsForProfile?: JsonValue;
|
||||
};
|
||||
|
||||
export function createCodexUsageLimitPromptError(message: string): Error & { status: 429 } {
|
||||
return Object.assign(new Error(message), { status: 429 as const });
|
||||
}
|
||||
|
||||
export function isCodexUsageLimitPromptError(error: unknown): error is Error & { status: 429 } {
|
||||
return error instanceof Error && "status" in error && error.status === 429;
|
||||
}
|
||||
|
||||
/** Marks a Codex auth profile blocked until the reset time advertised by rate limits. */
|
||||
export async function markCodexAuthProfileBlockedFromRateLimits(params: {
|
||||
params: EmbeddedRunAttemptParams;
|
||||
@@ -101,22 +109,20 @@ export async function refreshCodexUsageLimitPromptError(params: {
|
||||
message: string | undefined;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<string | undefined> {
|
||||
}): Promise<CodexUsageLimitErrorResult | undefined> {
|
||||
if (!shouldRefreshCodexRateLimitsForUsageLimitMessage(params.message)) {
|
||||
return undefined;
|
||||
}
|
||||
return (
|
||||
await refreshCodexUsageLimitError({
|
||||
client: params.client,
|
||||
source: {
|
||||
message: params.message,
|
||||
codexErrorInfo: "usageLimitExceeded",
|
||||
rateLimits: readRecentCodexRateLimits(params.client),
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
signal: params.signal,
|
||||
})
|
||||
)?.message;
|
||||
return refreshCodexUsageLimitError({
|
||||
client: params.client,
|
||||
source: {
|
||||
message: params.message,
|
||||
codexErrorInfo: "usageLimitExceeded",
|
||||
rateLimits: readRecentCodexRateLimits(params.client),
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
signal: params.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshCodexUsageLimitError(params: {
|
||||
|
||||
@@ -368,6 +368,56 @@ describe("resolveAuthProfileOrder", () => {
|
||||
).toStrictEqual(["fixture-provider:backup", "fixture-provider:primary"]);
|
||||
});
|
||||
|
||||
it("does not apply a block scoped to another model when ordering profiles", () => {
|
||||
const store: AuthProfileStore = {
|
||||
version: 1,
|
||||
profiles: {
|
||||
"fixture-provider:primary": {
|
||||
type: "api_key",
|
||||
provider: "fixture-provider",
|
||||
key: "placeholder",
|
||||
},
|
||||
"fixture-provider:backup": {
|
||||
type: "api_key",
|
||||
provider: "fixture-provider",
|
||||
key: "placeholder",
|
||||
},
|
||||
},
|
||||
usageStats: {
|
||||
"fixture-provider:primary": {
|
||||
blockedUntil: Date.now() + 60_000,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedModel: "model-a",
|
||||
blockedScope: "model",
|
||||
},
|
||||
},
|
||||
};
|
||||
const cfg = {
|
||||
auth: {
|
||||
order: {
|
||||
"fixture-provider": ["fixture-provider:primary", "fixture-provider:backup"],
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
expect(
|
||||
resolveAuthProfileOrder({
|
||||
cfg,
|
||||
store,
|
||||
provider: "fixture-provider",
|
||||
forModel: "model-b",
|
||||
}),
|
||||
).toStrictEqual(["fixture-provider:primary", "fixture-provider:backup"]);
|
||||
expect(
|
||||
resolveAuthProfileOrder({
|
||||
cfg,
|
||||
store,
|
||||
provider: "fixture-provider",
|
||||
forModel: "model-a",
|
||||
}),
|
||||
).toStrictEqual(["fixture-provider:backup", "fixture-provider:primary"]);
|
||||
});
|
||||
|
||||
it("keeps unresolved OAuth refs only in read-only profile ordering", () => {
|
||||
const store: AuthProfileStore = {
|
||||
version: 1,
|
||||
|
||||
@@ -374,7 +374,7 @@ export function resolveAuthProfileOrderWithMetadata(
|
||||
for (const profileId of deduped) {
|
||||
if (isProfileInCooldown(store, profileId, now, forModel)) {
|
||||
const cooldownUntil =
|
||||
resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}) ?? now;
|
||||
resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}, forModel) ?? now;
|
||||
inCooldown.push({ profileId, cooldownUntil });
|
||||
} else {
|
||||
available.push(profileId);
|
||||
@@ -493,7 +493,8 @@ function orderProfilesByMode(
|
||||
const cooldownSorted = inCooldown
|
||||
.map((profileId) => ({
|
||||
profileId,
|
||||
cooldownUntil: resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}) ?? now,
|
||||
cooldownUntil:
|
||||
resolveProfileUnusableUntil(store.usageStats?.[profileId] ?? {}, forModel) ?? now,
|
||||
}))
|
||||
.toSorted((a, b) => a.cooldownUntil - b.cooldownUntil)
|
||||
.map((entry) => entry.profileId);
|
||||
|
||||
@@ -118,6 +118,7 @@ function normalizeUsageStatsEntry(raw: unknown): ProfileUsageStats | undefined {
|
||||
blockedReason: normalizeEnumValue(raw.blockedReason, AUTH_BLOCKED_REASONS),
|
||||
blockedSource: normalizeEnumValue(raw.blockedSource, AUTH_BLOCKED_SOURCES),
|
||||
blockedModel: normalizeOptionalString(raw.blockedModel),
|
||||
blockedScope: raw.blockedScope === "model" ? "model" : undefined,
|
||||
cooldownUntil: normalizeFiniteNumber(raw.cooldownUntil),
|
||||
cooldownReason: normalizeEnumValue(raw.cooldownReason, AUTH_FAILURE_REASONS),
|
||||
cooldownModel: normalizeOptionalString(raw.cooldownModel),
|
||||
|
||||
@@ -106,6 +106,7 @@ export type ProfileUsageStats = {
|
||||
blockedReason?: AuthProfileBlockedReason;
|
||||
blockedSource?: AuthProfileBlockedSource;
|
||||
blockedModel?: string;
|
||||
blockedScope?: "model";
|
||||
cooldownUntil?: number;
|
||||
cooldownReason?: AuthProfileFailureReason;
|
||||
cooldownModel?: string;
|
||||
|
||||
@@ -23,9 +23,16 @@ export function isModelScopedCooldownReason(reason: AuthProfileFailureReason | u
|
||||
|
||||
/** Resolves the latest active blocked/cooldown/disabled timestamp for a profile. */
|
||||
export function resolveProfileUnusableUntil(
|
||||
stats: Pick<ProfileUsageStats, "blockedUntil" | "cooldownUntil" | "disabledUntil">,
|
||||
stats: Pick<
|
||||
ProfileUsageStats,
|
||||
"blockedUntil" | "blockedModel" | "blockedScope" | "cooldownUntil" | "disabledUntil"
|
||||
>,
|
||||
forModel?: string,
|
||||
): number | null {
|
||||
const values = [stats.blockedUntil, stats.cooldownUntil, stats.disabledUntil]
|
||||
const blockedUntil = isBlockScopedToDifferentModel(stats, forModel)
|
||||
? undefined
|
||||
: stats.blockedUntil;
|
||||
const values = [blockedUntil, stats.cooldownUntil, stats.disabledUntil]
|
||||
.map((value) => asDateTimestampMs(value))
|
||||
.filter((value): value is number => value !== undefined && value > 0);
|
||||
if (values.length === 0) {
|
||||
@@ -40,10 +47,40 @@ export function isActiveUnusableWindow(until: number | undefined, now: number):
|
||||
return timestamp !== undefined && timestamp > 0 && now < timestamp;
|
||||
}
|
||||
|
||||
function isBlockedWindowActiveForModel(
|
||||
stats: Pick<ProfileUsageStats, "blockedUntil" | "blockedModel" | "blockedScope">,
|
||||
now: number,
|
||||
forModel?: string,
|
||||
): boolean {
|
||||
return (
|
||||
!isBlockScopedToDifferentModel(stats, forModel) &&
|
||||
isActiveUnusableWindow(stats.blockedUntil, now)
|
||||
);
|
||||
}
|
||||
|
||||
function isBlockScopedToDifferentModel(
|
||||
stats: Pick<ProfileUsageStats, "blockedModel" | "blockedScope">,
|
||||
forModel?: string,
|
||||
): boolean {
|
||||
// Legacy rows carried blockedModel for profile-wide blocks without a scope marker.
|
||||
// Only explicit model scope narrows them; unmarked rows stay wide until expiry.
|
||||
return Boolean(
|
||||
forModel &&
|
||||
stats.blockedScope === "model" &&
|
||||
stats.blockedModel &&
|
||||
stats.blockedModel !== forModel,
|
||||
);
|
||||
}
|
||||
|
||||
function shouldBypassModelScopedCooldown(
|
||||
stats: Pick<
|
||||
ProfileUsageStats,
|
||||
"blockedUntil" | "cooldownReason" | "cooldownModel" | "disabledUntil"
|
||||
| "blockedUntil"
|
||||
| "blockedModel"
|
||||
| "blockedScope"
|
||||
| "cooldownReason"
|
||||
| "cooldownModel"
|
||||
| "disabledUntil"
|
||||
>,
|
||||
now: number,
|
||||
forModel?: string,
|
||||
@@ -53,7 +90,7 @@ function shouldBypassModelScopedCooldown(
|
||||
isModelScopedCooldownReason(stats.cooldownReason) &&
|
||||
stats.cooldownModel &&
|
||||
stats.cooldownModel !== forModel &&
|
||||
!isActiveUnusableWindow(stats.blockedUntil, now) &&
|
||||
!isBlockedWindowActiveForModel(stats, now, forModel) &&
|
||||
!isActiveUnusableWindow(stats.disabledUntil, now),
|
||||
);
|
||||
}
|
||||
@@ -82,7 +119,7 @@ export function isProfileInCooldown(
|
||||
if (shouldBypassModelScopedCooldown(stats, ts, forModel)) {
|
||||
return false;
|
||||
}
|
||||
const unusableUntil = resolveProfileUnusableUntil(stats);
|
||||
const unusableUntil = resolveProfileUnusableUntil(stats, forModel);
|
||||
return unusableUntil ? ts < unusableUntil : false;
|
||||
}
|
||||
|
||||
@@ -107,7 +144,7 @@ export function getSoonestCooldownExpiry(
|
||||
if (shouldBypassModelScopedCooldown(stats, ts, options?.forModel)) {
|
||||
continue;
|
||||
}
|
||||
const until = resolveProfileUnusableUntil(stats);
|
||||
const until = resolveProfileUnusableUntil(stats, options?.forModel);
|
||||
if (typeof until !== "number" || !Number.isFinite(until) || until <= 0) {
|
||||
continue;
|
||||
}
|
||||
@@ -115,7 +152,7 @@ export function getSoonestCooldownExpiry(
|
||||
options?.forModel &&
|
||||
stats.cooldownReason === "rate_limit" &&
|
||||
stats.cooldownModel === options.forModel &&
|
||||
!isActiveUnusableWindow(stats.blockedUntil, ts) &&
|
||||
!isBlockedWindowActiveForModel(stats, ts, options.forModel) &&
|
||||
!isActiveUnusableWindow(stats.disabledUntil, ts);
|
||||
if (matchingModelScopedCooldown) {
|
||||
latestMatchingModelCooldown =
|
||||
@@ -195,6 +232,7 @@ export function clearExpiredCooldowns(store: AuthProfileStore, now?: number): bo
|
||||
stats.blockedReason = undefined;
|
||||
stats.blockedSource = undefined;
|
||||
stats.blockedModel = undefined;
|
||||
stats.blockedScope = undefined;
|
||||
profileMutated = true;
|
||||
}
|
||||
if (disabledExpired) {
|
||||
|
||||
@@ -83,6 +83,7 @@ function expectProfileErrorStateCleared(
|
||||
) {
|
||||
expect(stats?.blockedUntil).toBeUndefined();
|
||||
expect(stats?.blockedReason).toBeUndefined();
|
||||
expect(stats?.blockedScope).toBeUndefined();
|
||||
expect(stats?.cooldownUntil).toBeUndefined();
|
||||
expect(stats?.disabledUntil).toBeUndefined();
|
||||
expect(stats?.disabledReason).toBeUndefined();
|
||||
@@ -103,6 +104,18 @@ describe("resolveProfileUnusableUntil", () => {
|
||||
).toBe(300);
|
||||
expect(resolveProfileUnusableUntil({ cooldownUntil: 300 })).toBe(300);
|
||||
});
|
||||
|
||||
it("keeps legacy blockedModel rows profile-wide", () => {
|
||||
expect(
|
||||
resolveProfileUnusableUntil({ blockedUntil: 300, blockedModel: "model-a" }, "model-b"),
|
||||
).toBe(300);
|
||||
});
|
||||
|
||||
it("applies explicitly model-scoped blocks only to that model", () => {
|
||||
const stats = { blockedUntil: 300, blockedModel: "model-a", blockedScope: "model" as const };
|
||||
expect(resolveProfileUnusableUntil(stats, "model-a")).toBe(300);
|
||||
expect(resolveProfileUnusableUntil(stats, "model-b")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveProfileUnusableUntilForDisplay", () => {
|
||||
@@ -285,17 +298,32 @@ describe("isProfileInCooldown", () => {
|
||||
expect(isProfileInCooldown(store, "github-copilot:github", undefined, "gpt-4.1")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not bypass model-scoped cooldown when blockedUntil is active", () => {
|
||||
it("bypasses model-scoped blocks and cooldowns for sibling models", () => {
|
||||
const now = Date.now();
|
||||
const store = makeStore({
|
||||
"google:default": {
|
||||
blockedUntil: now + 120_000,
|
||||
blockedReason: "subscription_limit",
|
||||
blockedModel: "gemini-3-flash-preview",
|
||||
blockedScope: "model",
|
||||
cooldownUntil: now + 60_000,
|
||||
cooldownReason: "timeout",
|
||||
cooldownModel: "gemini-3-flash-preview",
|
||||
},
|
||||
});
|
||||
expect(isProfileInCooldown(store, "google:default", now, "gemini-3-flash-preview")).toBe(true);
|
||||
expect(isProfileInCooldown(store, "google:default", now, "gemini-3.1-flash-lite")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps legacy blockedModel rows active for sibling models", () => {
|
||||
const now = Date.now();
|
||||
const store = makeStore({
|
||||
"google:default": {
|
||||
blockedUntil: now + 120_000,
|
||||
blockedModel: "gemini-3-flash-preview",
|
||||
},
|
||||
});
|
||||
|
||||
expect(isProfileInCooldown(store, "google:default", now, "gemini-3.1-flash-lite")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -917,6 +945,89 @@ describe("markAuthProfileFailure — active windows do not extend on retry", ()
|
||||
});
|
||||
|
||||
describe("markAuthProfileBlockedUntil", () => {
|
||||
it("keeps repeated same-model blocks scoped to that model", async () => {
|
||||
const now = Date.parse("2026-05-30T18:00:00.000Z");
|
||||
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: now + 60_000,
|
||||
blockedModel: "gpt-5.4",
|
||||
blockedScope: "model",
|
||||
},
|
||||
});
|
||||
mockLockedUpdateForStore(store);
|
||||
try {
|
||||
await markAuthProfileBlockedUntil({
|
||||
store,
|
||||
profileId: "openai:default",
|
||||
blockedUntil: now + 120_000,
|
||||
source: "codex_rate_limits",
|
||||
modelId: "gpt-5.4",
|
||||
});
|
||||
} finally {
|
||||
nowSpy.mockRestore();
|
||||
}
|
||||
|
||||
expect(store.usageStats?.["openai:default"]?.blockedModel).toBe("gpt-5.4");
|
||||
expect(store.usageStats?.["openai:default"]?.blockedScope).toBe("model");
|
||||
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4")).toBe(true);
|
||||
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(false);
|
||||
});
|
||||
|
||||
it("widens an active block after a different model fails", async () => {
|
||||
const now = Date.parse("2026-05-30T18:00:00.000Z");
|
||||
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: now + 60_000,
|
||||
blockedModel: "gpt-5.4",
|
||||
blockedScope: "model",
|
||||
},
|
||||
});
|
||||
mockLockedUpdateForStore(store);
|
||||
try {
|
||||
await markAuthProfileBlockedUntil({
|
||||
store,
|
||||
profileId: "openai:default",
|
||||
blockedUntil: now + 120_000,
|
||||
source: "codex_rate_limits",
|
||||
modelId: "gpt-5.4-mini",
|
||||
});
|
||||
} finally {
|
||||
nowSpy.mockRestore();
|
||||
}
|
||||
|
||||
expect(store.usageStats?.["openai:default"]?.blockedModel).toBeUndefined();
|
||||
expect(store.usageStats?.["openai:default"]?.blockedScope).toBeUndefined();
|
||||
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(true);
|
||||
});
|
||||
|
||||
it("never narrows an active profile-wide block", async () => {
|
||||
const now = Date.parse("2026-05-30T18:00:00.000Z");
|
||||
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const store = makeStore({
|
||||
"openai:default": {
|
||||
blockedUntil: now + 60_000,
|
||||
},
|
||||
});
|
||||
mockLockedUpdateForStore(store);
|
||||
try {
|
||||
await markAuthProfileBlockedUntil({
|
||||
store,
|
||||
profileId: "openai:default",
|
||||
blockedUntil: now + 120_000,
|
||||
source: "codex_rate_limits",
|
||||
modelId: "gpt-5.4",
|
||||
});
|
||||
} finally {
|
||||
nowSpy.mockRestore();
|
||||
}
|
||||
|
||||
expect(store.usageStats?.["openai:default"]?.blockedModel).toBeUndefined();
|
||||
expect(store.usageStats?.["openai:default"]?.blockedScope).toBeUndefined();
|
||||
expect(isProfileInCooldown(store, "openai:default", now, "gpt-5.4-mini")).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps a later active blocked-until timestamp", async () => {
|
||||
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-05-30T18:00:00.000Z"));
|
||||
const laterBlockedUntil = Date.parse("2031-01-01T00:00:00.000Z");
|
||||
|
||||
@@ -224,6 +224,7 @@ function applyWhamCooldownResult(params: {
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: params.whamResult.blockedSource ?? "wham",
|
||||
blockedModel: undefined,
|
||||
blockedScope: undefined,
|
||||
cooldownUntil: undefined,
|
||||
cooldownReason: undefined,
|
||||
cooldownModel: undefined,
|
||||
@@ -586,6 +587,7 @@ function resetUsageStats(
|
||||
blockedReason: undefined,
|
||||
blockedSource: undefined,
|
||||
blockedModel: undefined,
|
||||
blockedScope: undefined,
|
||||
cooldownUntil: undefined,
|
||||
cooldownReason: undefined,
|
||||
cooldownModel: undefined,
|
||||
@@ -842,12 +844,23 @@ function buildBlockedProfileUsageStats(params: {
|
||||
params.previousStats?.blockedUntil,
|
||||
params.now,
|
||||
);
|
||||
// One active block can stay model-scoped only while every observation names
|
||||
// that same model. Mixed or unknown observations widen the profile.
|
||||
const blockedModel =
|
||||
activeBlockedUntil === 0
|
||||
? params.modelId
|
||||
: params.previousStats?.blockedScope === "model" &&
|
||||
params.previousStats.blockedModel === params.modelId &&
|
||||
params.modelId
|
||||
? params.modelId
|
||||
: undefined;
|
||||
return {
|
||||
...params.previousStats,
|
||||
blockedUntil: Math.max(activeBlockedUntil, params.blockedUntil),
|
||||
blockedReason: "subscription_limit",
|
||||
blockedSource: params.source,
|
||||
blockedModel: params.modelId,
|
||||
blockedModel,
|
||||
blockedScope: blockedModel ? "model" : undefined,
|
||||
cooldownUntil: undefined,
|
||||
cooldownReason: undefined,
|
||||
cooldownModel: undefined,
|
||||
|
||||
@@ -995,6 +995,18 @@ describe("failover-error", () => {
|
||||
expect(coerceToFailoverError(err)?.status).toBe(429);
|
||||
});
|
||||
|
||||
it("classifies a structured prompt error independently of its wording", () => {
|
||||
const promptError = Object.assign(new Error("quota exhausted"), { status: 429 as const });
|
||||
const failoverError = coerceToFailoverError(promptError, {
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
});
|
||||
|
||||
expect(failoverError?.reason).toBe("rate_limit");
|
||||
expect(failoverError?.status).toBe(429);
|
||||
expect(failoverError?.message).toBe("quota exhausted");
|
||||
});
|
||||
|
||||
it("lets wrapped causes override parent context-overflow classifications", () => {
|
||||
const err = new Error("INVALID_ARGUMENT: input exceeds the maximum number of tokens", {
|
||||
cause: { code: "RESOURCE_EXHAUSTED" },
|
||||
|
||||
@@ -210,20 +210,26 @@ function expectFailureCount(
|
||||
expect(failureCounts?.[reason]).toBe(expected);
|
||||
}
|
||||
|
||||
async function writeMultiProfileAuthStore(agentDir: string) {
|
||||
async function writeMultiProfileAuthStore(
|
||||
agentDir: string,
|
||||
options?: { openAiProfileCount?: 2 | 3 },
|
||||
) {
|
||||
const includeThirdOpenAiProfile = options?.openAiProfileCount !== 2;
|
||||
saveAuthProfileStore(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:p1": { type: "api_key", provider: "openai", key: "sk-openai-1" },
|
||||
"openai:p2": { type: "api_key", provider: "openai", key: "sk-openai-2" },
|
||||
"openai:p3": { type: "api_key", provider: "openai", key: "sk-openai-3" },
|
||||
...(includeThirdOpenAiProfile
|
||||
? { "openai:p3": { type: "api_key" as const, provider: "openai", key: "placeholder" } }
|
||||
: {}),
|
||||
"groq:p1": { type: "api_key", provider: "groq", key: "sk-groq" },
|
||||
},
|
||||
usageStats: {
|
||||
"openai:p1": { lastUsed: 1 },
|
||||
"openai:p2": { lastUsed: 2 },
|
||||
"openai:p3": { lastUsed: 3 },
|
||||
...(includeThirdOpenAiProfile ? { "openai:p3": { lastUsed: 3 } } : {}),
|
||||
"groq:p1": { lastUsed: 4 },
|
||||
},
|
||||
},
|
||||
@@ -983,6 +989,41 @@ describe("runWithModelFallback + runEmbeddedAgent failover behavior", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("rotates Codex profiles on structured prompt rate limits before model fallback", async () => {
|
||||
await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
|
||||
await writeMultiProfileAuthStore(agentDir, { openAiProfileCount: 2 });
|
||||
mockPrimaryFailureThenFallbackSuccess(() => {
|
||||
return makeEmbeddedRunnerAttempt({
|
||||
promptError: Object.assign(
|
||||
new Error("You've reached your Codex subscription usage limit."),
|
||||
{ status: 429 as const },
|
||||
),
|
||||
promptErrorSource: "prompt",
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedFallback({
|
||||
agentDir,
|
||||
workspaceDir,
|
||||
sessionKey: "agent:test:codex-structured-prompt-rate-limit",
|
||||
runId: "run:codex-structured-prompt-rate-limit",
|
||||
});
|
||||
|
||||
expect(result.provider).toBe("groq");
|
||||
expect(result.model).toBe("mock-2");
|
||||
expect(result.attempts[0]?.reason).toBe("rate_limit");
|
||||
expectProviderAttemptCounts({ openai: 2, groq: 1 });
|
||||
const primaryCalls = runEmbeddedAttemptMock.mock.calls
|
||||
.map(([params]) => params as EmbeddedAttemptParams)
|
||||
.filter((params) => params.provider === "openai");
|
||||
expect(primaryCalls.map((params) => params.authProfileId)).toStrictEqual([
|
||||
"openai:p1",
|
||||
"openai:p2",
|
||||
]);
|
||||
expect(primaryCalls.map((params) => params.modelId)).toStrictEqual(["mock-1", "mock-1"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("respects prompt-side rateLimitedProfileRotations=0 and falls back immediately", async () => {
|
||||
await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
|
||||
await writeMultiProfileAuthStore(agentDir);
|
||||
|
||||
Reference in New Issue
Block a user