From ee7cafafeb6885255b2f14adebe1efac5cd865f6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 7 Jun 2026 01:40:44 +0200 Subject: [PATCH] fix(test): retry codex harness live timeouts --- ...gateway-codex-harness.live-helpers.test.ts | 8 +- .../gateway-codex-harness.live-helpers.ts | 10 +- .../gateway-codex-harness.live.test.ts | 229 +++++++++--------- 3 files changed, 123 insertions(+), 124 deletions(-) diff --git a/src/gateway/gateway-codex-harness.live-helpers.test.ts b/src/gateway/gateway-codex-harness.live-helpers.test.ts index 7bc696e98503..0bb7df72cad9 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.test.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.test.ts @@ -5,10 +5,9 @@ import { describe, expect, it } from "vitest"; import { EXPECTED_CODEX_MODELS_COMMAND_TEXT, EXPECTED_CODEX_STATUS_COMMAND_TEXT, - isRetryableCodexHarnessLiveError, isExpectedCodexModelsCommandText, isExpectedCodexStatusCommandText, - shouldSkipRetryableCodexHarnessLiveError, + isRetryableCodexHarnessLiveError, } from "./gateway-codex-harness.live-helpers.js"; const includesExpectedCodexModelsCommandText = (text: string) => @@ -24,19 +23,16 @@ function expectRecognizedCodexModelsCommandText(text: string): void { } describe("gateway codex harness live helpers", () => { - it("does not skip sessions.list timeouts when the Codex subagent probe is enabled", () => { + it("classifies sessions.list timeouts as retryable live Codex errors", () => { const error = new Error("gateway request timeout for sessions.list"); expect(isRetryableCodexHarnessLiveError(error)).toBe(true); - expect(shouldSkipRetryableCodexHarnessLiveError(error, { subagentProbe: false })).toBe(true); - expect(shouldSkipRetryableCodexHarnessLiveError(error, { subagentProbe: true })).toBe(false); }); it("does not classify unrelated live Codex errors as retryable gateway timeouts", () => { const error = new Error("subagent child did not emit lifecycle event"); expect(isRetryableCodexHarnessLiveError(error)).toBe(false); - expect(shouldSkipRetryableCodexHarnessLiveError(error, { subagentProbe: false })).toBe(false); }); it("accepts the current codex status prose from the live harness", () => { diff --git a/src/gateway/gateway-codex-harness.live-helpers.ts b/src/gateway/gateway-codex-harness.live-helpers.ts index c37ea80c81db..c58e441dba2c 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.ts @@ -282,18 +282,10 @@ export function isExpectedCodexModelsCommandText(text: string): boolean { ); } -/** Identifies transient live harness errors that are worth retrying. */ +/** Identifies transient live harness errors that are worth retrying, not skipping. */ export function isRetryableCodexHarnessLiveError(error: unknown): boolean { if (!(error instanceof Error)) { return false; } return error.message.includes("gateway request timeout for sessions.list"); } - -/** Skips retryable live errors only when the subagent probe is not under test. */ -export function shouldSkipRetryableCodexHarnessLiveError( - error: unknown, - params: { subagentProbe: boolean }, -): boolean { - return isRetryableCodexHarnessLiveError(error) && !params.subagentProbe; -} diff --git a/src/gateway/gateway-codex-harness.live.test.ts b/src/gateway/gateway-codex-harness.live.test.ts index a3ebdfaa29b5..c3124a079aeb 100644 --- a/src/gateway/gateway-codex-harness.live.test.ts +++ b/src/gateway/gateway-codex-harness.live.test.ts @@ -27,7 +27,7 @@ import { EXPECTED_CODEX_STATUS_COMMAND_TEXT, isExpectedCodexModelsCommandText, isExpectedCodexStatusCommandText, - shouldSkipRetryableCodexHarnessLiveError, + isRetryableCodexHarnessLiveError, } from "./gateway-codex-harness.live-helpers.js"; import { assertCronJobMatches, @@ -1037,118 +1037,129 @@ describeLive("gateway live (Codex harness)", () => { logCodexLiveStep("client-connected"); try { - try { - const sessionKey = "agent:dev:live-codex-harness"; + const maxAttempts = CODEX_HARNESS_SUBAGENT_PROBE ? 1 : 2; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + const sessionKey = "agent:dev:live-codex-harness"; - if (CODEX_HARNESS_SUBAGENT_PROBE) { - logCodexLiveStep("subagent-probe:start", { sessionKey }); - await verifyCodexSubagentProbe({ client, sessionKey }); - logCodexLiveStep("native-subagent-bridge-probe:start", { sessionKey }); - await verifyCodexNativeSubagentBridgeProbe({ client, sessionKey }); - logCodexLiveStep("subagent-probe:done"); - if (CODEX_HARNESS_SUBAGENT_ONLY) { + if (CODEX_HARNESS_SUBAGENT_PROBE) { + logCodexLiveStep("subagent-probe:start", { sessionKey }); + await verifyCodexSubagentProbe({ client, sessionKey }); + logCodexLiveStep("native-subagent-bridge-probe:start", { sessionKey }); + await verifyCodexNativeSubagentBridgeProbe({ client, sessionKey }); + logCodexLiveStep("subagent-probe:done"); + if (CODEX_HARNESS_SUBAGENT_ONLY) { + return; + } + } + + const unsubscribeDebugEvents = await subscribeCodexLiveDebugEvents(sessionKey); + const firstNonce = randomBytes(3).toString("hex").toUpperCase(); + try { + const firstToken = `CODEX-HARNESS-${firstNonce}`; + const firstText = await requestAgentText({ + client, + sessionKey, + expectedToken: firstToken, + message: `Reply with exactly ${firstToken} and nothing else.`, + }); + expect(firstText).toContain(firstToken); + logCodexLiveStep("first-turn", { firstText }); + + const secondNonce = randomBytes(3).toString("hex").toUpperCase(); + const secondToken = `CODEX-HARNESS-RESUME-${secondNonce}`; + const secondText = await requestAgentText({ + client, + sessionKey, + expectedToken: secondToken, + message: `Reply with exactly ${secondToken} and nothing else. Do not repeat ${firstToken}.`, + }); + expect(secondText).toContain(secondToken); + logCodexLiveStep("second-turn", { secondText }); + + if (CODEX_HARNESS_CODE_MODE_ONLY) { + logCodexLiveStep("code-mode-only-tool-probe:start", { sessionKey }); + await verifyCodexCodeModeOnlyDynamicToolProbe({ client, sessionKey }); + logCodexLiveStep("code-mode-only-tool-probe:done"); + } + } finally { + unsubscribeDebugEvents(); + } + + const statusText = await requestCodexCommandText({ + client, + events: gatewayEvents, + sessionKey, + command: "/codex status", + expectedText: [...EXPECTED_CODEX_STATUS_COMMAND_TEXT], + isExpectedText: isExpectedCodexStatusCommandText, + }); + logCodexLiveStep("codex-status-command", { statusText }); + + const modelsText = await requestCodexCommandText({ + client, + events: gatewayEvents, + sessionKey, + command: "/codex models", + expectedText: [...EXPECTED_CODEX_MODELS_COMMAND_TEXT], + isExpectedText: isExpectedCodexModelsCommandText, + }); + logCodexLiveStep("codex-models-command", { modelsText }); + + if (CODEX_HARNESS_CHAT_IMAGE_PROBE) { + logCodexLiveStep("chat-image-probe:start", { sessionKey }); + await verifyCodexChatImageProbe({ client, sessionKey }); + logCodexLiveStep("chat-image-probe:done"); + } + + if (CODEX_HARNESS_IMAGE_PROBE) { + logCodexLiveStep("image-probe:start", { sessionKey }); + await verifyCodexImageProbe({ client, sessionKey }); + logCodexLiveStep("image-probe:done"); + } + + if (CODEX_HARNESS_MCP_PROBE) { + logCodexLiveStep("cron-mcp-probe:start", { sessionKey }); + await verifyCodexCronMcpProbe({ + client, + sessionKey, + port, + token, + env: process.env, + }); + logCodexLiveStep("cron-mcp-probe:done"); + } + + if (CODEX_HARNESS_GUARDIAN_PROBE) { + const guardianSessionKey = "agent:dev:live-codex-harness-guardian"; + logCodexLiveStep("guardian-probe:start", { sessionKey: guardianSessionKey }); + await verifyCodexGuardianProbe({ client, sessionKey: guardianSessionKey }); + logCodexLiveStep("guardian-probe:done"); + } + break; + } catch (error) { + if (isCodexAccountTokenError(error)) { + console.error( + "SKIP: Codex auth cannot extract accountId from the available token; skipping live Codex harness assertions.", + ); return; } - } - - const unsubscribeDebugEvents = await subscribeCodexLiveDebugEvents(sessionKey); - const firstNonce = randomBytes(3).toString("hex").toUpperCase(); - try { - const firstToken = `CODEX-HARNESS-${firstNonce}`; - const firstText = await requestAgentText({ - client, - sessionKey, - expectedToken: firstToken, - message: `Reply with exactly ${firstToken} and nothing else.`, - }); - expect(firstText).toContain(firstToken); - logCodexLiveStep("first-turn", { firstText }); - - const secondNonce = randomBytes(3).toString("hex").toUpperCase(); - const secondToken = `CODEX-HARNESS-RESUME-${secondNonce}`; - const secondText = await requestAgentText({ - client, - sessionKey, - expectedToken: secondToken, - message: `Reply with exactly ${secondToken} and nothing else. Do not repeat ${firstToken}.`, - }); - expect(secondText).toContain(secondToken); - logCodexLiveStep("second-turn", { secondText }); - - if (CODEX_HARNESS_CODE_MODE_ONLY) { - logCodexLiveStep("code-mode-only-tool-probe:start", { sessionKey }); - await verifyCodexCodeModeOnlyDynamicToolProbe({ client, sessionKey }); - logCodexLiveStep("code-mode-only-tool-probe:done"); + if ( + attempt < maxAttempts && + !CODEX_HARNESS_SUBAGENT_PROBE && + isRetryableCodexHarnessLiveError(error) + ) { + logCodexLiveStep("retryable-timeout:retry", { + attempt, + maxAttempts, + message: error instanceof Error ? error.message : String(error), + }); + gatewayEvents.length = 0; + await delay(2_000); + continue; + } else { + throw error; } - } finally { - unsubscribeDebugEvents(); - } - - const statusText = await requestCodexCommandText({ - client, - events: gatewayEvents, - sessionKey, - command: "/codex status", - expectedText: [...EXPECTED_CODEX_STATUS_COMMAND_TEXT], - isExpectedText: isExpectedCodexStatusCommandText, - }); - logCodexLiveStep("codex-status-command", { statusText }); - - const modelsText = await requestCodexCommandText({ - client, - events: gatewayEvents, - sessionKey, - command: "/codex models", - expectedText: [...EXPECTED_CODEX_MODELS_COMMAND_TEXT], - isExpectedText: isExpectedCodexModelsCommandText, - }); - logCodexLiveStep("codex-models-command", { modelsText }); - - if (CODEX_HARNESS_CHAT_IMAGE_PROBE) { - logCodexLiveStep("chat-image-probe:start", { sessionKey }); - await verifyCodexChatImageProbe({ client, sessionKey }); - logCodexLiveStep("chat-image-probe:done"); - } - - if (CODEX_HARNESS_IMAGE_PROBE) { - logCodexLiveStep("image-probe:start", { sessionKey }); - await verifyCodexImageProbe({ client, sessionKey }); - logCodexLiveStep("image-probe:done"); - } - - if (CODEX_HARNESS_MCP_PROBE) { - logCodexLiveStep("cron-mcp-probe:start", { sessionKey }); - await verifyCodexCronMcpProbe({ - client, - sessionKey, - port, - token, - env: process.env, - }); - logCodexLiveStep("cron-mcp-probe:done"); - } - - if (CODEX_HARNESS_GUARDIAN_PROBE) { - const guardianSessionKey = "agent:dev:live-codex-harness-guardian"; - logCodexLiveStep("guardian-probe:start", { sessionKey: guardianSessionKey }); - await verifyCodexGuardianProbe({ client, sessionKey: guardianSessionKey }); - logCodexLiveStep("guardian-probe:done"); - } - } catch (error) { - if (isCodexAccountTokenError(error)) { - console.error( - "SKIP: Codex auth cannot extract accountId from the available token; skipping live Codex harness assertions.", - ); - } else if ( - shouldSkipRetryableCodexHarnessLiveError(error, { - subagentProbe: CODEX_HARNESS_SUBAGENT_PROBE, - }) - ) { - console.error( - `SKIP: Codex harness live backend hit a retryable gateway timeout; skipping live Codex harness assertions. ${error instanceof Error ? error.message : String(error)}`, - ); - } else { - throw error; } } } finally {