diff --git a/src/gateway/gateway-cli-backend.live-helpers.test.ts b/src/gateway/gateway-cli-backend.live-helpers.test.ts index 4bd7ab5da1ea..693c981ea104 100644 --- a/src/gateway/gateway-cli-backend.live-helpers.test.ts +++ b/src/gateway/gateway-cli-backend.live-helpers.test.ts @@ -190,6 +190,38 @@ describe("gateway cli backend live helpers", () => { }); }); + it("retries Codex CLI timeout payloads only before the final attempt", async () => { + const { isCliBackendLiveTimeoutPayload, shouldRetryCliBackendLiveTimeout } = + await import("./gateway-cli-backend.live-helpers.js"); + const timeoutPayload = { status: "timeout" }; + + expect(isCliBackendLiveTimeoutPayload(timeoutPayload)).toBe(true); + expect( + shouldRetryCliBackendLiveTimeout({ + attempt: 1, + maxAttempts: 2, + payload: timeoutPayload, + providerId: "codex-cli", + }), + ).toBe(true); + expect( + shouldRetryCliBackendLiveTimeout({ + attempt: 2, + maxAttempts: 2, + payload: timeoutPayload, + providerId: "codex-cli", + }), + ).toBe(false); + expect( + shouldRetryCliBackendLiveTimeout({ + attempt: 1, + maxAttempts: 2, + payload: timeoutPayload, + providerId: "claude-cli", + }), + ).toBe(false); + }); + it("allows live env overrides for fresh and resume CLI args", async () => { const { resolveCliBackendLiveArgs } = await import("./gateway-cli-backend.live-helpers.js"); diff --git a/src/gateway/gateway-cli-backend.live-helpers.ts b/src/gateway/gateway-cli-backend.live-helpers.ts index bb326d45070a..ac017af8651a 100644 --- a/src/gateway/gateway-cli-backend.live-helpers.ts +++ b/src/gateway/gateway-cli-backend.live-helpers.ts @@ -232,6 +232,27 @@ export function resolveCliBackendLiveProviderSkipDecision(params: { }; } +export function isCliBackendLiveTimeoutPayload(payload: unknown): boolean { + return ( + typeof payload === "object" && + payload !== null && + (payload as { status?: unknown }).status === "timeout" + ); +} + +export function shouldRetryCliBackendLiveTimeout(params: { + attempt: number; + maxAttempts: number; + payload: unknown; + providerId: string; +}): boolean { + return ( + params.providerId === "codex-cli" && + params.attempt < params.maxAttempts && + isCliBackendLiveTimeoutPayload(params.payload) + ); +} + export function matchesCliBackendReply(text: string, expected: string): boolean { const normalized = text.trim(); const target = expected.trim(); diff --git a/src/gateway/gateway-cli-backend.live.test.ts b/src/gateway/gateway-cli-backend.live.test.ts index f1090ada3b89..636e1454f1a3 100644 --- a/src/gateway/gateway-cli-backend.live.test.ts +++ b/src/gateway/gateway-cli-backend.live.test.ts @@ -18,12 +18,14 @@ import { matchesCliBackendReply, parseImageMode, parseJsonStringArray, + isCliBackendLiveTimeoutPayload, resolveCliBackendLiveArgs, resolveCliBackendLiveModelSelection, resolveCliBackendLiveProviderSkipDecision, resolveCliModelSwitchProbeTarget, restoreCliBackendLiveEnv, shouldAllowCliBackendLiveProviderSkip, + shouldRetryCliBackendLiveTimeout, shouldRunCliImageProbe, shouldRunCliModelSwitchProbe, shouldRunCliMcpProbe, @@ -59,16 +61,22 @@ const MCP_SCHEMA_PROBE_TOOL_NAME = "mcp_schema_probe_no_args"; const DEFAULT_PROVIDER = "claude-cli"; const DEFAULT_MODEL = resolveCliBackendLiveTest(DEFAULT_PROVIDER)?.defaultModelRef ?? "claude-cli/claude-sonnet-4-6"; -// The cron/MCP live probe now tolerates more cancelled tool-call retries in CI, -// so the outer test budget needs enough headroom to finish those retries. -const CLI_BACKEND_LIVE_TIMEOUT_MS = 20 * 60_000; const CLI_BACKEND_REQUEST_TIMEOUT_MS = parsePositiveIntegerEnv( "OPENCLAW_LIVE_CLI_BACKEND_REQUEST_TIMEOUT_MS", 15 * 60_000, ); -const CLI_BACKEND_AGENT_TIMEOUT_SECONDS = Math.max( - 1, - Math.ceil(CLI_BACKEND_REQUEST_TIMEOUT_MS / 1000) - 10, +const CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS = 2; +const CLI_BACKEND_CODEX_TIMEOUT_RETRY_SLEEP_MS = 5_000; +const CLI_BACKEND_RETRY_WRAPPED_AGENT_REQUESTS = 2; +const CLI_BACKEND_CODEX_TIMEOUT_RETRY_SEQUENCE_MS = + CLI_BACKEND_REQUEST_TIMEOUT_MS * CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS + + CLI_BACKEND_CODEX_TIMEOUT_RETRY_SLEEP_MS * (CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS - 1); +// The cron/MCP live probe and Codex timeout retry need enough outer-test headroom +// to finish both the initial agent request and one follow-up probe. +const CLI_BACKEND_LIVE_TIMEOUT_MS = Math.max( + 20 * 60_000, + CLI_BACKEND_CODEX_TIMEOUT_RETRY_SEQUENCE_MS * CLI_BACKEND_RETRY_WRAPPED_AGENT_REQUESTS + + 2 * 60_000, ); function parsePositiveIntegerEnv(name: string, fallback: number): number { @@ -97,6 +105,19 @@ function sleep(ms: number): Promise { }); } +type CliBackendAgentAttemptTimeouts = { + agentTimeoutSeconds: number; + requestTimeoutMs: number; +}; + +function resolveCliBackendAgentAttemptTimeouts(): CliBackendAgentAttemptTimeouts { + const requestTimeoutMs = CLI_BACKEND_REQUEST_TIMEOUT_MS; + return { + requestTimeoutMs, + agentTimeoutSeconds: Math.max(1, Math.ceil(requestTimeoutMs / 1000) - 10), + }; +} + function openAiProviderConfigForCodexCli( modelKey: string, ): NonNullable["providers"]>["openai"] { @@ -180,6 +201,36 @@ async function requestWithProviderCapacityRetry( return undefined; } +async function requestWithCodexTimeoutRetry( + providerId: string, + label: string, + request: (timeouts: CliBackendAgentAttemptTimeouts) => Promise, +): Promise { + const maxAttempts = providerId === "codex-cli" ? CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS : 1; + const retrySleepMs = providerId === "codex-cli" ? CLI_BACKEND_CODEX_TIMEOUT_RETRY_SLEEP_MS : 0; + const attemptTimeouts = resolveCliBackendAgentAttemptTimeouts(); + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const payload = await requestWithProviderCapacityRetry(providerId, label, () => + request(attemptTimeouts), + ); + if (!payload) { + return undefined; + } + if (!isCliBackendLiveTimeoutPayload(payload)) { + return payload; + } + if (shouldRetryCliBackendLiveTimeout({ providerId, payload, attempt, maxAttempts })) { + logCliBackendLiveStep("agent-timeout-retry", { providerId, label, attempt, maxAttempts }); + await sleep(retrySleepMs); + continue; + } + throw new Error( + `${label} for provider "${providerId}" timed out waiting for a model response.`, + ); + } + return undefined; +} + async function createMcpSchemaProbePlugin(tempDir: string): Promise { const pluginDir = path.join(tempDir, MCP_SCHEMA_PROBE_PLUGIN_ID); await fs.mkdir(pluginDir, { recursive: true }); @@ -450,35 +501,32 @@ describeLive("gateway live (cli backend)", () => { const memoryNonce = randomBytes(3).toString("hex").toUpperCase(); const memoryToken = `CLI-MEM-${memoryNonce}`; logCliBackendLiveStep("agent-request:start", { sessionKey, nonce }); - const payload = await requestWithProviderCapacityRetry(providerId, "agent request", () => - client.request( - "agent", - { - sessionKey, - idempotencyKey: `idem-${randomUUID()}`, - message: - providerId === "codex-cli" - ? `Do not inspect files or run tools. Reply with exactly: CLI-BACKEND-${nonce}.` - : enableCliModelSwitchProbe - ? `Please include the token CLI-BACKEND-${nonce} in your reply.` + - ` Also remember this session note for later: ${memoryToken}.` + - " Do not include the note in your reply." - : `Please include the token CLI-BACKEND-${nonce} in your reply.`, - deliver: false, - timeout: CLI_BACKEND_AGENT_TIMEOUT_SECONDS, - }, - { expectFinal: true, timeoutMs: CLI_BACKEND_REQUEST_TIMEOUT_MS }, - ), + const payload = await requestWithCodexTimeoutRetry( + providerId, + "agent request", + (timeouts) => + client.request( + "agent", + { + sessionKey, + idempotencyKey: `idem-${randomUUID()}`, + message: + providerId === "codex-cli" + ? `Do not inspect files or run tools. Reply with exactly: CLI-BACKEND-${nonce}.` + : enableCliModelSwitchProbe + ? `Please include the token CLI-BACKEND-${nonce} in your reply.` + + ` Also remember this session note for later: ${memoryToken}.` + + " Do not include the note in your reply." + : `Please include the token CLI-BACKEND-${nonce} in your reply.`, + deliver: false, + timeout: timeouts.agentTimeoutSeconds, + }, + { expectFinal: true, timeoutMs: timeouts.requestTimeoutMs }, + ), ); if (!payload) { return; } - if (providerId === "codex-cli" && payload?.status === "timeout") { - console.warn( - "SKIP: Codex CLI backend live smoke timed out waiting for a model response.", - ); - return; - } if (payload?.status !== "ok") { throw new Error(`agent status=${String(payload?.status)}`); } @@ -523,10 +571,10 @@ describeLive("gateway live (cli backend)", () => { `sessions.patch failed for model switch: ${JSON.stringify(patchPayload)}`, ); } - const switchPayload = await requestWithProviderCapacityRetry( + const switchPayload = await requestWithCodexTimeoutRetry( providerId, "agent model-switch request", - () => + (timeouts) => client.request( "agent", { @@ -537,9 +585,9 @@ describeLive("gateway live (cli backend)", () => { `What session note did I ask you to remember earlier? ` + `Reply with exactly: CLI backend SWITCH OK ${switchNonce} .`, deliver: false, - timeout: CLI_BACKEND_AGENT_TIMEOUT_SECONDS, + timeout: timeouts.agentTimeoutSeconds, }, - { expectFinal: true, timeoutMs: CLI_BACKEND_REQUEST_TIMEOUT_MS }, + { expectFinal: true, timeoutMs: timeouts.requestTimeoutMs }, ), ); if (!switchPayload) { @@ -559,10 +607,10 @@ describeLive("gateway live (cli backend)", () => { } else if (CLI_RESUME) { const resumeNonce = randomBytes(3).toString("hex").toUpperCase(); logCliBackendLiveStep("agent-resume:start", { sessionKey, resumeNonce }); - const resumePayload = await requestWithProviderCapacityRetry( + const resumePayload = await requestWithCodexTimeoutRetry( providerId, "agent resume request", - () => + (timeouts) => client.request( "agent", { @@ -573,9 +621,9 @@ describeLive("gateway live (cli backend)", () => { ? `Do not inspect files or run tools. Reply with exactly: CLI-RESUME-${resumeNonce}.` : `Reply with exactly: CLI backend RESUME OK ${resumeNonce}.`, deliver: false, - timeout: CLI_BACKEND_AGENT_TIMEOUT_SECONDS, + timeout: timeouts.agentTimeoutSeconds, }, - { expectFinal: true, timeoutMs: CLI_BACKEND_REQUEST_TIMEOUT_MS }, + { expectFinal: true, timeoutMs: timeouts.requestTimeoutMs }, ), ); if (!resumePayload) {