ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-24 11:25:50 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): retry cli backend codex timeouts

Browse Source
This commit is contained in:
Vincent Koc
2026-06-07 02:00:04 +02:00
parent 05c3325b0a
commit 029e8f0153
3 changed files with 140 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gateway/gateway-cli-backend.live-helpers.test.ts
+32
View File
@@ -190,6 +190,38 @@ describe("gateway cli backend live helpers", () => {
});
});
it("retries Codex CLI timeout payloads only before the final attempt", async () => {
const { isCliBackendLiveTimeoutPayload, shouldRetryCliBackendLiveTimeout } =
await import("./gateway-cli-backend.live-helpers.js");
const timeoutPayload = { status: "timeout" };
expect(isCliBackendLiveTimeoutPayload(timeoutPayload)).toBe(true);
expect(
shouldRetryCliBackendLiveTimeout({
attempt: 1,
maxAttempts: 2,
payload: timeoutPayload,
providerId: "codex-cli",
}),
).toBe(true);
expect(
shouldRetryCliBackendLiveTimeout({
attempt: 2,
maxAttempts: 2,
payload: timeoutPayload,
providerId: "codex-cli",
}),
).toBe(false);
expect(
shouldRetryCliBackendLiveTimeout({
attempt: 1,
maxAttempts: 2,
payload: timeoutPayload,
providerId: "claude-cli",
}),
).toBe(false);
});
it("allows live env overrides for fresh and resume CLI args", async () => {
const { resolveCliBackendLiveArgs } = await import("./gateway-cli-backend.live-helpers.js");
src/gateway/gateway-cli-backend.live-helpers.ts
+21
View File
@@ -232,6 +232,27 @@ export function resolveCliBackendLiveProviderSkipDecision(params: {
};
}
export function isCliBackendLiveTimeoutPayload(payload: unknown): boolean {
return (
typeof payload === "object" &&
payload !== null &&
(payload as { status?: unknown }).status === "timeout"
);
}
export function shouldRetryCliBackendLiveTimeout(params: {
attempt: number;
maxAttempts: number;
payload: unknown;
providerId: string;
}): boolean {
return (
params.providerId === "codex-cli" &&
params.attempt < params.maxAttempts &&
isCliBackendLiveTimeoutPayload(params.payload)
);
}
export function matchesCliBackendReply(text: string, expected: string): boolean {
const normalized = text.trim();
const target = expected.trim();
src/gateway/gateway-cli-backend.live.test.ts
+87 -39
View File
@@ -18,12 +18,14 @@ import {
matchesCliBackendReply,
parseImageMode,
parseJsonStringArray,
isCliBackendLiveTimeoutPayload,
resolveCliBackendLiveArgs,
resolveCliBackendLiveModelSelection,
resolveCliBackendLiveProviderSkipDecision,
resolveCliModelSwitchProbeTarget,
restoreCliBackendLiveEnv,
shouldAllowCliBackendLiveProviderSkip,
shouldRetryCliBackendLiveTimeout,
shouldRunCliImageProbe,
shouldRunCliModelSwitchProbe,
shouldRunCliMcpProbe,
@@ -59,16 +61,22 @@ const MCP_SCHEMA_PROBE_TOOL_NAME = "mcp_schema_probe_no_args";
const DEFAULT_PROVIDER = "claude-cli";
const DEFAULT_MODEL =
resolveCliBackendLiveTest(DEFAULT_PROVIDER)?.defaultModelRef ?? "claude-cli/claude-sonnet-4-6";
// The cron/MCP live probe now tolerates more cancelled tool-call retries in CI,
// so the outer test budget needs enough headroom to finish those retries.
const CLI_BACKEND_LIVE_TIMEOUT_MS = 20 * 60_000;
const CLI_BACKEND_REQUEST_TIMEOUT_MS = parsePositiveIntegerEnv(
"OPENCLAW_LIVE_CLI_BACKEND_REQUEST_TIMEOUT_MS",
15 * 60_000,
);
const CLI_BACKEND_AGENT_TIMEOUT_SECONDS = Math.max(
1,
Math.ceil(CLI_BACKEND_REQUEST_TIMEOUT_MS / 1000) - 10,
const CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS = 2;
const CLI_BACKEND_CODEX_TIMEOUT_RETRY_SLEEP_MS = 5_000;
const CLI_BACKEND_RETRY_WRAPPED_AGENT_REQUESTS = 2;
const CLI_BACKEND_CODEX_TIMEOUT_RETRY_SEQUENCE_MS =
CLI_BACKEND_REQUEST_TIMEOUT_MS * CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS +
CLI_BACKEND_CODEX_TIMEOUT_RETRY_SLEEP_MS * (CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS - 1);
// The cron/MCP live probe and Codex timeout retry need enough outer-test headroom
// to finish both the initial agent request and one follow-up probe.
const CLI_BACKEND_LIVE_TIMEOUT_MS = Math.max(
20 * 60_000,
CLI_BACKEND_CODEX_TIMEOUT_RETRY_SEQUENCE_MS * CLI_BACKEND_RETRY_WRAPPED_AGENT_REQUESTS +
2 * 60_000,
);
function parsePositiveIntegerEnv(name: string, fallback: number): number {
@@ -97,6 +105,19 @@ function sleep(ms: number): Promise<void> {
});
}
type CliBackendAgentAttemptTimeouts = {
agentTimeoutSeconds: number;
requestTimeoutMs: number;
};
function resolveCliBackendAgentAttemptTimeouts(): CliBackendAgentAttemptTimeouts {
const requestTimeoutMs = CLI_BACKEND_REQUEST_TIMEOUT_MS;
return {
requestTimeoutMs,
agentTimeoutSeconds: Math.max(1, Math.ceil(requestTimeoutMs / 1000) - 10),
};
}
function openAiProviderConfigForCodexCli(
modelKey: string,
): NonNullable<NonNullable<OpenClawConfig["models"]>["providers"]>["openai"] {
@@ -180,6 +201,36 @@ async function requestWithProviderCapacityRetry<T>(
return undefined;
}
async function requestWithCodexTimeoutRetry<T>(
providerId: string,
label: string,
request: (timeouts: CliBackendAgentAttemptTimeouts) => Promise<T>,
): Promise<T | undefined> {
const maxAttempts = providerId === "codex-cli" ? CLI_BACKEND_CODEX_TIMEOUT_RETRY_ATTEMPTS : 1;
const retrySleepMs = providerId === "codex-cli" ? CLI_BACKEND_CODEX_TIMEOUT_RETRY_SLEEP_MS : 0;
const attemptTimeouts = resolveCliBackendAgentAttemptTimeouts();
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const payload = await requestWithProviderCapacityRetry(providerId, label, () =>
request(attemptTimeouts),
);
if (!payload) {
return undefined;
}
if (!isCliBackendLiveTimeoutPayload(payload)) {
return payload;
}
if (shouldRetryCliBackendLiveTimeout({ providerId, payload, attempt, maxAttempts })) {
logCliBackendLiveStep("agent-timeout-retry", { providerId, label, attempt, maxAttempts });
await sleep(retrySleepMs);
continue;
}
throw new Error(
`${label} for provider "${providerId}" timed out waiting for a model response.`,
);
}
return undefined;
}
async function createMcpSchemaProbePlugin(tempDir: string): Promise<string> {
const pluginDir = path.join(tempDir, MCP_SCHEMA_PROBE_PLUGIN_ID);
await fs.mkdir(pluginDir, { recursive: true });
@@ -450,35 +501,32 @@ describeLive("gateway live (cli backend)", () => {
const memoryNonce = randomBytes(3).toString("hex").toUpperCase();
const memoryToken = `CLI-MEM-${memoryNonce}`;
logCliBackendLiveStep("agent-request:start", { sessionKey, nonce });
const payload = await requestWithProviderCapacityRetry(providerId, "agent request", () =>
client.request(
"agent",
{
sessionKey,
idempotencyKey: `idem-${randomUUID()}`,
message:
providerId === "codex-cli"
? `Do not inspect files or run tools. Reply with exactly: CLI-BACKEND-${nonce}.`
: enableCliModelSwitchProbe
? `Please include the token CLI-BACKEND-${nonce} in your reply.` +
` Also remember this session note for later: ${memoryToken}.` +
" Do not include the note in your reply."
: `Please include the token CLI-BACKEND-${nonce} in your reply.`,
deliver: false,
timeout: CLI_BACKEND_AGENT_TIMEOUT_SECONDS,
},
{ expectFinal: true, timeoutMs: CLI_BACKEND_REQUEST_TIMEOUT_MS },
),
const payload = await requestWithCodexTimeoutRetry(
providerId,
"agent request",
(timeouts) =>
client.request(
"agent",
{
sessionKey,
idempotencyKey: `idem-${randomUUID()}`,
message:
providerId === "codex-cli"
? `Do not inspect files or run tools. Reply with exactly: CLI-BACKEND-${nonce}.`
: enableCliModelSwitchProbe
? `Please include the token CLI-BACKEND-${nonce} in your reply.` +
` Also remember this session note for later: ${memoryToken}.` +
" Do not include the note in your reply."
: `Please include the token CLI-BACKEND-${nonce} in your reply.`,
deliver: false,
timeout: timeouts.agentTimeoutSeconds,
},
{ expectFinal: true, timeoutMs: timeouts.requestTimeoutMs },
),
);
if (!payload) {
return;
}
if (providerId === "codex-cli" && payload?.status === "timeout") {
console.warn(
"SKIP: Codex CLI backend live smoke timed out waiting for a model response.",
);
return;
}
if (payload?.status !== "ok") {
throw new Error(`agent status=${String(payload?.status)}`);
}
@@ -523,10 +571,10 @@ describeLive("gateway live (cli backend)", () => {
`sessions.patch failed for model switch: ${JSON.stringify(patchPayload)}`,
);
}
const switchPayload = await requestWithProviderCapacityRetry(
const switchPayload = await requestWithCodexTimeoutRetry(
providerId,
"agent model-switch request",
() =>
(timeouts) =>
client.request(
"agent",
{
@@ -537,9 +585,9 @@ describeLive("gateway live (cli backend)", () => {
`What session note did I ask you to remember earlier? ` +
`Reply with exactly: CLI backend SWITCH OK ${switchNonce} <remembered-note>.`,
deliver: false,
timeout: CLI_BACKEND_AGENT_TIMEOUT_SECONDS,
timeout: timeouts.agentTimeoutSeconds,
},
{ expectFinal: true, timeoutMs: CLI_BACKEND_REQUEST_TIMEOUT_MS },
{ expectFinal: true, timeoutMs: timeouts.requestTimeoutMs },
),
);
if (!switchPayload) {
@@ -559,10 +607,10 @@ describeLive("gateway live (cli backend)", () => {
} else if (CLI_RESUME) {
const resumeNonce = randomBytes(3).toString("hex").toUpperCase();
logCliBackendLiveStep("agent-resume:start", { sessionKey, resumeNonce });
const resumePayload = await requestWithProviderCapacityRetry(
const resumePayload = await requestWithCodexTimeoutRetry(
providerId,
"agent resume request",
() =>
(timeouts) =>
client.request(
"agent",
{
@@ -573,9 +621,9 @@ describeLive("gateway live (cli backend)", () => {
? `Do not inspect files or run tools. Reply with exactly: CLI-RESUME-${resumeNonce}.`
: `Reply with exactly: CLI backend RESUME OK ${resumeNonce}.`,
deliver: false,
timeout: CLI_BACKEND_AGENT_TIMEOUT_SECONDS,
timeout: timeouts.agentTimeoutSeconds,
},
{ expectFinal: true, timeoutMs: CLI_BACKEND_REQUEST_TIMEOUT_MS },
{ expectFinal: true, timeoutMs: timeouts.requestTimeoutMs },
),
);
if (!resumePayload) {