diff --git a/docs/plugins/codex-harness-reference.md b/docs/plugins/codex-harness-reference.md index c614eaf96a6d..eccdb5214b0d 100644 --- a/docs/plugins/codex-harness-reference.md +++ b/docs/plugins/codex-harness-reference.md @@ -85,25 +85,25 @@ For an already-running app-server, use WebSocket transport: Supported `appServer` fields: -| Field | Default | Meaning | -| --------------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `transport` | `"stdio"` | `"stdio"` spawns Codex; `"websocket"` connects to `url`. | -| `command` | managed Codex binary | Executable for stdio transport. Leave unset to use the managed binary. | -| `args` | `["app-server", "--listen", "stdio://"]` | Arguments for stdio transport. | -| `url` | unset | WebSocket app-server URL. | -| `authToken` | unset | Bearer token for WebSocket transport. | -| `headers` | `{}` | Extra WebSocket headers. | -| `clearEnv` | `[]` | Extra environment variable names removed from the spawned stdio app-server process after OpenClaw builds its inherited environment. | -| `requestTimeoutMs` | `60000` | Timeout for app-server control-plane calls. | -| `turnCompletionIdleTimeoutMs` | `60000` | Quiet window after Codex accepts a turn or after a turn-scoped app-server request while OpenClaw waits for `turn/completed`. | -| `postToolRawAssistantCompletionIdleTimeoutMs` | unset | Completion-idle guard used after a tool handoff when Codex emits raw assistant completion or progress but does not send `turn/completed`. Defaults to the assistant completion idle timeout when unset. Use this for trusted or heavy workloads where post-tool synthesis can legitimately stay quiet longer than the final assistant release budget. | -| `mode` | `"yolo"` unless local Codex requirements disallow YOLO | Preset for YOLO or guardian-reviewed execution. | -| `approvalPolicy` | `"never"` or an allowed guardian approval policy | Native Codex approval policy sent to thread start, resume, and turn. | -| `sandbox` | `"danger-full-access"` or an allowed guardian sandbox | Native Codex sandbox mode sent to thread start and resume. Active OpenClaw sandboxes narrow `danger-full-access` turns to Codex `workspace-write`; the turn network flag follows OpenClaw sandbox egress. | -| `approvalsReviewer` | `"user"` or an allowed guardian reviewer | Use `"auto_review"` to let Codex review native approval prompts when allowed. | -| `defaultWorkspaceDir` | current process directory | Workspace used by `/codex bind` when `--cwd` is omitted. | -| `serviceTier` | unset | Optional Codex app-server service tier. `"priority"` enables fast-mode routing, `"flex"` requests flex processing, and `null` clears the override. Legacy `"fast"` is accepted as `"priority"`. | -| `experimental.sandboxExecServer` | `false` | Preview opt-in that registers an OpenClaw sandbox-backed Codex environment with Codex app-server 0.132.0 or newer so native Codex execution can run inside the active OpenClaw sandbox. | +| Field | Default | Meaning | +| --------------------------------------------- | ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `transport` | `"stdio"` | `"stdio"` spawns Codex; `"websocket"` connects to `url`. | +| `command` | managed Codex binary | Executable for stdio transport. Leave unset to use the managed binary. | +| `args` | `["app-server", "--listen", "stdio://"]` | Arguments for stdio transport. | +| `url` | unset | WebSocket app-server URL. | +| `authToken` | unset | Bearer token for WebSocket transport. | +| `headers` | `{}` | Extra WebSocket headers. | +| `clearEnv` | `[]` | Extra environment variable names removed from the spawned stdio app-server process after OpenClaw builds its inherited environment. | +| `requestTimeoutMs` | `60000` | Timeout for app-server control-plane calls. | +| `turnCompletionIdleTimeoutMs` | `60000` | Quiet window after Codex accepts a turn or after a turn-scoped app-server request while OpenClaw waits for `turn/completed`. | +| `postToolRawAssistantCompletionIdleTimeoutMs` | `300000` | Completion-idle guard used after a tool handoff when Codex emits raw assistant completion or progress but does not send `turn/completed`. Use this for trusted or heavy workloads where post-tool synthesis can legitimately stay quiet longer than the final assistant release budget. | +| `mode` | `"yolo"` unless local Codex requirements disallow YOLO | Preset for YOLO or guardian-reviewed execution. | +| `approvalPolicy` | `"never"` or an allowed guardian approval policy | Native Codex approval policy sent to thread start, resume, and turn. | +| `sandbox` | `"danger-full-access"` or an allowed guardian sandbox | Native Codex sandbox mode sent to thread start and resume. Active OpenClaw sandboxes narrow `danger-full-access` turns to Codex `workspace-write`; the turn network flag follows OpenClaw sandbox egress. | +| `approvalsReviewer` | `"user"` or an allowed guardian reviewer | Use `"auto_review"` to let Codex review native approval prompts when allowed. | +| `defaultWorkspaceDir` | current process directory | Workspace used by `/codex bind` when `--cwd` is omitted. | +| `serviceTier` | unset | Optional Codex app-server service tier. `"priority"` enables fast-mode routing, `"flex"` requests flex processing, and `null` clears the override. Legacy `"fast"` is accepted as `"priority"`. | +| `experimental.sandboxExecServer` | `false` | Preview opt-in that registers an OpenClaw sandbox-backed Codex environment with Codex app-server 0.132.0 or newer so native Codex execution can run inside the active OpenClaw sandbox. | The plugin blocks older or unversioned app-server handshakes. Codex app-server must report stable version `0.125.0` or newer. @@ -337,10 +337,15 @@ Codex then goes quiet without `turn/completed`, OpenClaw best-effort interrupts the native turn and releases the session lane. Post-tool raw assistant progress keeps waiting for `turn/completed` while a completion-idle guard stays armed; the guard uses `appServer.postToolRawAssistantCompletionIdleTimeoutMs` when -configured and falls back to the assistant completion idle timeout otherwise. -Timeout diagnostics include the last app-server notification method and, for raw -assistant response items, the item type, role, id, and a bounded assistant text -preview. +configured and defaults to five minutes otherwise. Replay-safe stdio app-server +failures, including turn-completion idle timeouts without assistant, tool, +active-item, or side-effect evidence, are retried once on a fresh app-server +attempt. Unsafe timeouts still retire the stuck app-server client and release +the OpenClaw session lane. They also clear the stale native thread binding and +surface a recoverable timeout message for user or maintainer judgment instead of +being replayed automatically. Timeout diagnostics include the last +app-server notification method and, for raw assistant response items, the item +type, role, id, and a bounded assistant text preview. ## Model discovery diff --git a/docs/plugins/codex-harness.md b/docs/plugins/codex-harness.md index aced51c4bf54..6ddd1dd6b26d 100644 --- a/docs/plugins/codex-harness.md +++ b/docs/plugins/codex-harness.md @@ -525,25 +525,25 @@ Supported top-level Codex plugin fields: Supported `appServer` fields: -| Field | Default | Meaning | -| --------------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `transport` | `"stdio"` | `"stdio"` spawns Codex; `"websocket"` connects to `url`. | -| `command` | managed Codex binary | Executable for stdio transport. Leave unset to use the managed binary; set it only for an explicit override. | -| `args` | `["app-server", "--listen", "stdio://"]` | Arguments for stdio transport. | -| `url` | unset | WebSocket app-server URL. | -| `authToken` | unset | Bearer token for WebSocket transport. | -| `headers` | `{}` | Extra WebSocket headers. | -| `clearEnv` | `[]` | Extra environment variable names removed from the spawned stdio app-server process after OpenClaw builds its inherited environment. OpenClaw keeps per-agent `CODEX_HOME` and inherited `HOME` for local launches. | -| `codeModeOnly` | `false` | Opt into Codex's code-mode-only tool surface. OpenClaw dynamic tools remain registered with Codex so nested `tools.*` calls return through the app-server `item/tool/call` bridge. | -| `requestTimeoutMs` | `60000` | Timeout for app-server control-plane calls. | -| `turnCompletionIdleTimeoutMs` | `60000` | Quiet window after Codex accepts a turn or after a turn-scoped app-server request while OpenClaw waits for `turn/completed`. Raise this for slow post-tool or status-only synthesis phases. | -| `postToolRawAssistantCompletionIdleTimeoutMs` | unset | Completion-idle guard used after a tool handoff when Codex emits raw assistant completion or progress but does not send `turn/completed`. Defaults to the assistant completion idle timeout when unset. Use this for trusted or heavy workloads where post-tool synthesis can legitimately stay quiet longer than the final assistant release budget. | -| `mode` | `"yolo"` unless local Codex requirements disallow YOLO | Preset for YOLO or guardian-reviewed execution. Local stdio requirements that omit `danger-full-access`, `never` approval, or the `user` reviewer make the implicit default guardian. | -| `approvalPolicy` | `"never"` or an allowed guardian approval policy | Native Codex approval policy sent to thread start/resume/turn. Guardian defaults prefer `"on-request"` when allowed. | -| `sandbox` | `"danger-full-access"` or an allowed guardian sandbox | Native Codex sandbox mode sent to thread start/resume. Guardian defaults prefer `"workspace-write"` when allowed, otherwise `"read-only"`. When an OpenClaw sandbox is active, `danger-full-access` turns use Codex `workspace-write` with network access derived from the OpenClaw sandbox egress setting. | -| `approvalsReviewer` | `"user"` or an allowed guardian reviewer | Use `"auto_review"` to let Codex review native approval prompts when allowed, otherwise `guardian_subagent` or `user`. `guardian_subagent` remains a legacy alias. | -| `serviceTier` | unset | Optional Codex app-server service tier. `"priority"` enables fast-mode routing, `"flex"` requests flex processing, `null` clears the override, and legacy `"fast"` is accepted as `"priority"`. | -| `experimental.sandboxExecServer` | `false` | Preview opt-in that registers an OpenClaw sandbox-backed Codex environment with Codex app-server 0.132.0 or newer so native Codex execution can run inside the active OpenClaw sandbox. | +| Field | Default | Meaning | +| --------------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `transport` | `"stdio"` | `"stdio"` spawns Codex; `"websocket"` connects to `url`. | +| `command` | managed Codex binary | Executable for stdio transport. Leave unset to use the managed binary; set it only for an explicit override. | +| `args` | `["app-server", "--listen", "stdio://"]` | Arguments for stdio transport. | +| `url` | unset | WebSocket app-server URL. | +| `authToken` | unset | Bearer token for WebSocket transport. | +| `headers` | `{}` | Extra WebSocket headers. | +| `clearEnv` | `[]` | Extra environment variable names removed from the spawned stdio app-server process after OpenClaw builds its inherited environment. OpenClaw keeps per-agent `CODEX_HOME` and inherited `HOME` for local launches. | +| `codeModeOnly` | `false` | Opt into Codex's code-mode-only tool surface. OpenClaw dynamic tools remain registered with Codex so nested `tools.*` calls return through the app-server `item/tool/call` bridge. | +| `requestTimeoutMs` | `60000` | Timeout for app-server control-plane calls. | +| `turnCompletionIdleTimeoutMs` | `60000` | Quiet window after Codex accepts a turn or after a turn-scoped app-server request while OpenClaw waits for `turn/completed`. Raise this for slow post-tool or status-only synthesis phases. | +| `postToolRawAssistantCompletionIdleTimeoutMs` | `300000` | Completion-idle guard used after a tool handoff when Codex emits raw assistant completion or progress but does not send `turn/completed`. Use this for trusted or heavy workloads where post-tool synthesis can legitimately stay quiet longer than the final assistant release budget. | +| `mode` | `"yolo"` unless local Codex requirements disallow YOLO | Preset for YOLO or guardian-reviewed execution. Local stdio requirements that omit `danger-full-access`, `never` approval, or the `user` reviewer make the implicit default guardian. | +| `approvalPolicy` | `"never"` or an allowed guardian approval policy | Native Codex approval policy sent to thread start/resume/turn. Guardian defaults prefer `"on-request"` when allowed. | +| `sandbox` | `"danger-full-access"` or an allowed guardian sandbox | Native Codex sandbox mode sent to thread start/resume. Guardian defaults prefer `"workspace-write"` when allowed, otherwise `"read-only"`. When an OpenClaw sandbox is active, `danger-full-access` turns use Codex `workspace-write` with network access derived from the OpenClaw sandbox egress setting. | +| `approvalsReviewer` | `"user"` or an allowed guardian reviewer | Use `"auto_review"` to let Codex review native approval prompts when allowed, otherwise `guardian_subagent` or `user`. `guardian_subagent` remains a legacy alias. | +| `serviceTier` | unset | Optional Codex app-server service tier. `"priority"` enables fast-mode routing, `"flex"` requests flex processing, `null` clears the override, and legacy `"fast"` is accepted as `"priority"`. | +| `experimental.sandboxExecServer` | `false` | Preview opt-in that registers an OpenClaw sandbox-backed Codex environment with Codex app-server 0.132.0 or newer so native Codex execution can run inside the active OpenClaw sandbox. | OpenClaw-owned dynamic tool calls are bounded independently from `appServer.requestTimeoutMs`: Codex `item/tool/call` requests use a 90 second @@ -574,10 +574,15 @@ goes quiet without `turn/completed`, OpenClaw best-effort interrupts the native turn and releases the session lane. Post-tool raw assistant progress keeps waiting for `turn/completed` while a completion-idle guard stays armed; the guard uses `appServer.postToolRawAssistantCompletionIdleTimeoutMs` when configured and -falls back to the assistant completion idle timeout otherwise. Timeout -diagnostics include the last app-server notification method and, for raw -assistant response items, the item type, role, id, and a bounded assistant text -preview. +defaults to five minutes otherwise. Replay-safe stdio app-server failures, +including turn-completion idle timeouts without assistant, tool, active-item, or +side-effect evidence, are retried once on a fresh app-server attempt. Unsafe +timeouts still retire the stuck app-server client and release the OpenClaw +session lane. They also clear the stale native thread binding and surface a +recoverable timeout message for user or maintainer judgment instead of being +replayed automatically. Timeout diagnostics include the last app-server +notification method and, for raw assistant response items, the item type, role, +id, and a bounded assistant text preview. Environment overrides remain available for local testing: diff --git a/extensions/codex-supervisor/src/supervisor.test.ts b/extensions/codex-supervisor/src/supervisor.test.ts index d5137792823a..2e736448bde5 100644 --- a/extensions/codex-supervisor/src/supervisor.test.ts +++ b/extensions/codex-supervisor/src/supervisor.test.ts @@ -808,12 +808,15 @@ describe("connectCodexAppServerEndpoint", () => { const sawProbeRequest = new Promise((resolve) => { server.once("connection", (socket) => { socket.on("message", (data) => { - const text = Array.isArray(data) - ? Buffer.concat(data).toString("utf8") - : data instanceof ArrayBuffer - ? Buffer.from(new Uint8Array(data)).toString("utf8") - : Buffer.from(data).toString("utf8"); - const request = JSON.parse(text) as Record; + const messageText = + typeof data === "string" + ? data + : Array.isArray(data) + ? Buffer.concat(data).toString() + : data instanceof ArrayBuffer + ? Buffer.from(new Uint8Array(data)).toString() + : Buffer.from(data).toString(); + const request = JSON.parse(messageText) as Record; if (request.method === "initialize") { socket.send(JSON.stringify({ id: request.id, result: {} })); } diff --git a/extensions/codex/openclaw.plugin.json b/extensions/codex/openclaw.plugin.json index 49ea774364f1..ae9aae07fe7c 100644 --- a/extensions/codex/openclaw.plugin.json +++ b/extensions/codex/openclaw.plugin.json @@ -176,7 +176,8 @@ }, "postToolRawAssistantCompletionIdleTimeoutMs": { "type": "number", - "minimum": 1 + "minimum": 1, + "default": 300000 }, "approvalPolicy": { "type": "string", @@ -360,7 +361,7 @@ }, "appServer.postToolRawAssistantCompletionIdleTimeoutMs": { "label": "Post-Tool Raw Assistant Completion Idle Timeout", - "help": "Completion-idle guard after a tool handoff when Codex emits raw assistant completion or progress without turn/completed. Defaults to the assistant completion idle timeout when unset.", + "help": "Completion-idle guard after a tool handoff when Codex emits raw assistant completion or progress without turn/completed. Defaults to 300000 ms when unset.", "advanced": true }, "appServer.approvalPolicy": { diff --git a/extensions/codex/src/app-server/attempt-notification-state.ts b/extensions/codex/src/app-server/attempt-notification-state.ts index e72a76a9af68..c0b36d05405a 100644 --- a/extensions/codex/src/app-server/attempt-notification-state.ts +++ b/extensions/codex/src/app-server/attempt-notification-state.ts @@ -4,6 +4,7 @@ import { isAssistantCompletionReleaseNotification, isCodexTurnAbortMarkerNotification, isNativeToolProgressNotification, + isNativeResponseStreamDeltaNotification, isPendingOpenClawDynamicToolCompletionNotification, isRawAssistantCompletionNotification, isRawReasoningCompletionNotification, @@ -99,9 +100,10 @@ export function applyCodexTurnNotificationState(params: { params.turnId, ); const isTurnCompletion = notification.method === "turn/completed" && isCurrentTurnNotification; + const isNativeResponseStreamDelta = isNativeResponseStreamDeltaNotification(notification); let turnCrossedToolHandoff = params.turnCrossedToolHandoff; - if (isCurrentTurnNotification) { + if (isCurrentTurnNotification && !isNativeResponseStreamDelta) { turnWatches.touchActivity(`notification:${notification.method}`, { details: describeNotificationActivity(notification), attemptProgress: true, @@ -174,6 +176,9 @@ export function applyCodexTurnNotificationState(params: { } else if (isCurrentTurnNotification && assistantCompletionCanRelease) { turnWatches.armAssistantCompletionIdleWatch(describeNotificationActivity(notification)); } else if (postToolRawAssistantCompletionNeedsTerminalGuard) { + // A post-tool assistant status can be followed by native Codex streaming a + // large custom tool input. Forwarded raw deltas refresh activity at enqueue + // time; keep this guard conservative for versions that do not forward them. turnWatches.armCompletionIdleWatch({ timeoutMs: params.postToolRawAssistantCompletionIdleTimeoutMs, }); @@ -203,6 +208,7 @@ export function applyCodexTurnNotificationState(params: { !turnWatches.isCompletionIdleWatchPinnedByTerminalError() && notification.method !== "turn/completed" && isCurrentTurnNotification && + !isNativeResponseStreamDelta && !trackedDynamicToolCompletion && !rawToolOutputCompletion && !postToolRawAssistantCompletionNeedsTerminalGuard && diff --git a/extensions/codex/src/app-server/attempt-notifications.ts b/extensions/codex/src/app-server/attempt-notifications.ts index 754f1552d5f8..4cd80bf58c66 100644 --- a/extensions/codex/src/app-server/attempt-notifications.ts +++ b/extensions/codex/src/app-server/attempt-notifications.ts @@ -179,6 +179,12 @@ export function isNativeToolProgressNotification(notification: CodexServerNotifi } } +export function isNativeResponseStreamDeltaNotification( + notification: CodexServerNotification, +): boolean { + return notification.method.startsWith("response.") && notification.method.endsWith(".delta"); +} + export function isRawAssistantCompletionNotification( notification: CodexServerNotification, ): boolean { diff --git a/extensions/codex/src/app-server/attempt-results.test.ts b/extensions/codex/src/app-server/attempt-results.test.ts index dc06b7cc1430..c1ed5cc6bf35 100644 --- a/extensions/codex/src/app-server/attempt-results.test.ts +++ b/extensions/codex/src/app-server/attempt-results.test.ts @@ -89,6 +89,28 @@ describe("Codex app-server attempt results", () => { replayInvalid: true, livenessState: "abandoned", }); + expect( + buildCodexAppServerPromptTimeoutOutcome({ + result: createResult({ + assistantTexts: ["I am changing the data model now..."], + }), + turnCompletionIdleTimedOut: true, + }), + ).toEqual({ + message: + "Codex stopped before confirming the turn was complete. The response may be incomplete; retry if needed.", + }); + expect( + buildCodexAppServerPromptTimeoutOutcome({ + result: createResult({ + toolMetas: [{ toolName: "exec" }], + }), + turnCompletionIdleTimedOut: true, + }), + ).toEqual({ + message: + "Codex stopped before confirming the turn was complete. The response may be incomplete; retry if needed.", + }); }); it("classifies replay blocked reasons", () => { diff --git a/extensions/codex/src/app-server/attempt-results.ts b/extensions/codex/src/app-server/attempt-results.ts index e399b6bdef38..61bb08f7782d 100644 --- a/extensions/codex/src/app-server/attempt-results.ts +++ b/extensions/codex/src/app-server/attempt-results.ts @@ -27,10 +27,12 @@ export function buildCodexAppServerPromptTimeoutOutcome(params: { const completionIdleTimeoutHadPotentialSideEffects = hasCodexAppServerPotentialSideEffectEvidence( params.result, ); + const replayBlockedReason = resolveCodexAppServerReplayBlockedReason(params.result); if ( !params.turnCompletionIdleTimedOut || (params.result.itemLifecycle.completedCount === 0 && - !completionIdleTimeoutHadPotentialSideEffects) + !completionIdleTimeoutHadPotentialSideEffects && + replayBlockedReason === undefined) ) { return undefined; } diff --git a/extensions/codex/src/app-server/attempt-timeouts.test.ts b/extensions/codex/src/app-server/attempt-timeouts.test.ts index 6637b6b4c3b8..78443a746b63 100644 --- a/extensions/codex/src/app-server/attempt-timeouts.test.ts +++ b/extensions/codex/src/app-server/attempt-timeouts.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { CODEX_APP_SERVER_STARTUP_TIMEOUT_FLOOR_MS, + CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS, CODEX_TURN_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS, CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS, CODEX_TURN_TERMINAL_IDLE_TIMEOUT_MS, @@ -36,6 +37,11 @@ describe("Codex app-server attempt timeouts", () => { }); it("normalizes turn idle timeout overrides", () => { + expect(CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS).toBe(5 * 60_000); + expect(CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS).toBeGreaterThan( + CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS, + ); + expect(resolveCodexTurnCompletionIdleTimeoutMs(undefined)).toBe( CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS, ); @@ -54,9 +60,21 @@ describe("Codex app-server attempt timeouts", () => { expect(resolveCodexTurnAssistantCompletionIdleTimeoutMs(9.8)).toBe(9); expect(resolveCodexTurnAssistantCompletionIdleTimeoutMs(-10)).toBe(1); - expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 123)).toBe(123); - expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(Number.NaN, 123)).toBe(123); - expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, Number.NaN)).toBe(1); + expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 123)).toBe( + CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS, + ); + expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(Number.NaN, 123)).toBe( + CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS, + ); + expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 120_000)).toBe( + CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS, + ); + expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 6 * 60_000)).toBe( + 6 * 60_000, + ); + expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, Number.NaN)).toBe( + CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS, + ); expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(7.9, 123)).toBe(7); expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(0, 123)).toBe(1); diff --git a/extensions/codex/src/app-server/attempt-timeouts.ts b/extensions/codex/src/app-server/attempt-timeouts.ts index 2c93875e7227..16b5c8494e5a 100644 --- a/extensions/codex/src/app-server/attempt-timeouts.ts +++ b/extensions/codex/src/app-server/attempt-timeouts.ts @@ -3,6 +3,10 @@ import { parseFiniteNumber } from "openclaw/plugin-sdk/number-runtime"; export const CODEX_APP_SERVER_STARTUP_TIMEOUT_FLOOR_MS = 100; export const CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS = 60_000; export const CODEX_TURN_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS = 10_000; +// Native Codex can stream a large custom tool input after a raw assistant +// progress item. Forwarded deltas count as activity, but older native paths may +// not surface them, so keep this terminal guard conservative. +export const CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS = 5 * 60_000; export const CODEX_POST_REASONING_SOURCE_REPLY_IDLE_TIMEOUT_MS = 5 * 60_000; export const CODEX_TURN_TERMINAL_IDLE_TIMEOUT_MS = 30 * 60_000; @@ -91,7 +95,11 @@ export function resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs( value: number | undefined, fallbackMs: number, ): number { - return resolvePositiveIntegerTimeoutMs(value, fallbackMs); + const defaultMs = Math.max( + resolvePositiveIntegerTimeoutMs(undefined, fallbackMs), + CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS, + ); + return resolvePositiveIntegerTimeoutMs(value, defaultMs); } export function resolveCodexTurnTerminalIdleTimeoutMs(value: number | undefined): number { diff --git a/extensions/codex/src/app-server/attempt-turn-watches.ts b/extensions/codex/src/app-server/attempt-turn-watches.ts index 056f5117501a..7d384499d59a 100644 --- a/extensions/codex/src/app-server/attempt-turn-watches.ts +++ b/extensions/codex/src/app-server/attempt-turn-watches.ts @@ -380,9 +380,21 @@ export function createCodexAttemptTurnWatchController(params: { } scheduleProgressWatches(); }, - noteNotificationReceived: (method: string) => { + noteNotificationReceived: ( + method: string, + options?: { details?: Record; attemptProgress?: boolean }, + ) => { completionLastActivityAt = Date.now(); completionLastActivityReason = `notification:${method}`; + if (options?.details !== undefined) { + completionLastActivityDetails = options.details; + } + if (options?.attemptProgress) { + attemptLastProgressAt = completionLastActivityAt; + attemptLastProgressReason = completionLastActivityReason; + attemptLastProgressDetails = options.details; + params.onAttemptProgress(completionLastActivityReason, options.details); + } }, scheduleProgressWatches, clearCompletionIdleTimer, diff --git a/extensions/codex/src/app-server/client.ts b/extensions/codex/src/app-server/client.ts index 8441d54e43df..7e3bd5cd544c 100644 --- a/extensions/codex/src/app-server/client.ts +++ b/extensions/codex/src/app-server/client.ts @@ -102,6 +102,7 @@ export class CodexAppServerClient { private readonly requestHandlers = new Set(); private readonly notificationHandlers = new Set(); private readonly closeHandlers = new Set<(client: CodexAppServerClient) => void>(); + private activeSharedLeaseCountProvider: (() => number | undefined) | undefined; private nextId = 1; private initialized = false; private closed = false; @@ -281,6 +282,16 @@ export class CodexAppServerClient { return () => this.notificationHandlers.delete(handler); } + setActiveSharedLeaseCountProviderForUnscopedNotifications( + provider: (() => number | undefined) | undefined, + ): void { + this.activeSharedLeaseCountProvider = provider; + } + + getActiveSharedLeaseCountForUnscopedNotifications(): number | undefined { + return this.activeSharedLeaseCountProvider?.(); + } + addCloseHandler(handler: (client: CodexAppServerClient) => void): () => void { this.closeHandlers.add(handler); return () => this.closeHandlers.delete(handler); diff --git a/extensions/codex/src/app-server/run-attempt.ts b/extensions/codex/src/app-server/run-attempt.ts index 024220b5e330..3086c5448c20 100644 --- a/extensions/codex/src/app-server/run-attempt.ts +++ b/extensions/codex/src/app-server/run-attempt.ts @@ -80,6 +80,7 @@ import { isCurrentApprovalTurnRequestParams, isCurrentThreadOptionalTurnRequestParams, isCurrentThreadTurnRequestParams, + isNativeResponseStreamDeltaNotification, isTerminalTurnStatus, } from "./attempt-notifications.js"; import { @@ -99,7 +100,10 @@ import { resolveCodexTurnTerminalIdleTimeoutMs, withCodexStartupTimeout, } from "./attempt-timeouts.js"; -import { createCodexAttemptTurnWatchController } from "./attempt-turn-watches.js"; +import { + createCodexAttemptTurnWatchController, + type CodexAttemptTurnWatchTimeoutKind, +} from "./attempt-turn-watches.js"; import { refreshCodexAppServerAuthTokens, resolveCodexAppServerAuthAccountCacheKey, @@ -202,6 +206,7 @@ import { import { releaseCodexSandboxExecServerEnvironment } from "./sandbox-exec-server.js"; import { clearCodexAppServerBinding, + clearCodexAppServerBindingForThread, readCodexAppServerBinding, type CodexAppServerThreadBinding, } from "./session-binding.js"; @@ -941,6 +946,7 @@ export async function runCodexAppServerAttempt( let terminalTurnNotificationQueued = false; let timedOut = false; let turnCompletionIdleTimedOut = false; + let turnWatchTimeoutKind: CodexAttemptTurnWatchTimeoutKind | undefined; let turnCompletionIdleTimeoutMessage: string | undefined; let clientClosedPromptError: string | undefined; let clientClosedAbort = false; @@ -1019,9 +1025,10 @@ export async function runCodexAppServerAttempt( turnTerminalIdleTimeoutMs, interruptTimeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, onInterruptTurn: (input) => interruptCodexTurnBestEffort(client, input), - onTimeout: () => { + onTimeout: (timeout) => { timedOut = true; turnCompletionIdleTimedOut = true; + turnWatchTimeoutKind = timeout.kind; turnCompletionIdleTimeoutMessage = "codex app-server turn idle timed out waiting for turn/completed"; }, @@ -1276,8 +1283,29 @@ export async function runCodexAppServerAttempt( // Touch idle-watch timestamps at receive time, not just after queued // projection. A queued terminal event should suppress short false-idle // guards, while the full attempt watchdog still releases a wedged queue. - if (correlation.matchesActiveTurn !== false) { - turnWatches.noteNotificationReceived(notification.method); + const isNativeResponseStreamDelta = isNativeResponseStreamDeltaNotification(notification); + const nativeResponseStreamDeltaMatchesActiveTurn = + isNativeResponseStreamDelta && + (correlation.matchesActiveTurn === true || + (isUnscopedCodexNotification(correlation) && + canAttributeUnscopedNativeResponseDeltaToThisTurn(client))); + const notificationMatchesActiveTurn = + correlation.matchesActiveTurn === true || + (!isNativeResponseStreamDelta && correlation.matchesActiveTurn !== false) || + nativeResponseStreamDeltaMatchesActiveTurn; + if (notificationMatchesActiveTurn) { + // If a future Codex app-server exposes raw response deltas, treat them as + // activity only when scoped to this turn or attributable to a single lease. + // Today the durable app-server raw-event surface is rawResponseItem/completed. + turnWatches.noteNotificationReceived( + notification.method, + isNativeResponseStreamDelta + ? { + attemptProgress: true, + details: { lastNotificationMethod: notification.method }, + } + : undefined, + ); } notificationQueue = notificationQueue.then( () => handleNotification(notification), @@ -1881,11 +1909,15 @@ export async function runCodexAppServerAttempt( const abortListener = () => { const shouldRetireClient = timedOut; if (shouldRetireClient) { - void retireCodexAppServerClientAfterTimedOutTurn(client, { - threadId: thread.threadId, - turnId: activeTurnId, - reason: String(runAbortController.signal.reason ?? "timeout"), - }).finally(() => { + void (async () => { + // Timed-out native turns cannot be safely resumed on the same thread. + await clearCodexAppServerBindingForThread(activeSessionFile, thread.threadId); + await retireCodexAppServerClientAfterTimedOutTurn(client, { + threadId: thread.threadId, + turnId: activeTurnId, + reason: String(runAbortController.signal.reason ?? "timeout"), + }); + })().finally(() => { resolveCompletion?.(); }); return; @@ -2095,6 +2127,10 @@ export async function runCodexAppServerAttempt( ? { codexAppServerFailure: { kind: codexAppServerFailureKind, + ...(codexAppServerFailureKind === "turn_completion_idle_timeout" && + turnWatchTimeoutKind + ? { turnWatchTimeoutKind } + : {}), transport: appServer.start.transport, threadId: thread.threadId, turnId: activeTurnId, @@ -2202,6 +2238,22 @@ function isNonEmptyString(value: unknown): value is string { return typeof value === "string" && value.length > 0; } +function canAttributeUnscopedNativeResponseDeltaToThisTurn(client: CodexAppServerClient): boolean { + const activeLeases = client.getActiveSharedLeaseCountForUnscopedNotifications?.(); + return activeLeases === undefined || activeLeases <= 1; +} + +function isUnscopedCodexNotification( + correlation: ReturnType, +): boolean { + return ( + !correlation.threadId && + !correlation.turnId && + !correlation.nestedTurnThreadId && + !correlation.nestedTurnId + ); +} + function shouldRetryContextEngineTurnOnFreshCodexThread(params: { error: unknown; contextEngineActive: boolean; diff --git a/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts b/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts index 4c6383ec8f89..1816dace8074 100644 --- a/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts +++ b/extensions/codex/src/app-server/run-attempt.turn-watches.test.ts @@ -1,3 +1,4 @@ +import fs from "node:fs/promises"; import path from "node:path"; import { embeddedAgentLog, @@ -31,6 +32,7 @@ import { turnStartResult, } from "./run-attempt-test-harness.js"; import { testing } from "./run-attempt.js"; +import { resolveCodexAppServerBindingPath } from "./session-binding.js"; setupRunAttemptTestHooks(); @@ -70,6 +72,7 @@ describe("runCodexAppServerAttempt turn watches", () => { path.join(tempDir, "workspace"), ); params.timeoutMs = 200; + const bindingPath = resolveCodexAppServerBindingPath(params.sessionFile); const run = runCodexAppServerAttempt(params, { pluginConfig: { appServer: { turnCompletionIdleTimeoutMs: 5 } }, @@ -115,6 +118,7 @@ describe("runCodexAppServerAttempt turn watches", () => { ), { interval: 1 }, ); + await expect(fs.stat(bindingPath)).rejects.toMatchObject({ code: "ENOENT" }); expect(queueActiveRunMessageForTest("session-1", "after timeout")).toBe(false); }); @@ -1124,7 +1128,7 @@ describe("runCodexAppServerAttempt turn watches", () => { expect(request.mock.calls.some(([method]) => method === "turn/interrupt")).toBe(false); }); - it("times out post-tool raw assistant progress after the assistant idle timeout", async () => { + it("times out post-tool raw assistant progress after the post-tool timeout", async () => { let notify: (notification: CodexServerNotification) => Promise = async () => undefined; let handleRequest: | ((request: { id: string; method: string; params?: unknown }) => Promise) @@ -1167,6 +1171,7 @@ describe("runCodexAppServerAttempt turn watches", () => { const run = runCodexAppServerAttempt(params, { turnCompletionIdleTimeoutMs: 50, turnAssistantCompletionIdleTimeoutMs: 5, + postToolRawAssistantCompletionIdleTimeoutMs: 5, turnTerminalIdleTimeoutMs: 500, }); await vi.waitFor(() => expect(handleRequest).toBeTypeOf("function"), fastWait); @@ -1331,7 +1336,317 @@ describe("runCodexAppServerAttempt turn watches", () => { expect(completionWarnData?.lastActivityReason).toBe("notification:rawResponseItem/completed"); }); - it("times out post-native-tool raw assistant progress after the assistant idle timeout", async () => { + it("counts native response deltas as post-tool raw assistant activity", async () => { + let notify: (notification: CodexServerNotification) => Promise = async () => undefined; + let handleRequest: + | ((request: { id: string; method: string; params?: unknown }) => Promise) + | undefined; + const request = vi.fn(async (method: string) => { + if (method === "thread/start") { + return threadStartResult("thread-1"); + } + if (method === "turn/start") { + return turnStartResult("turn-1", "inProgress"); + } + return {}; + }); + setCodexAppServerClientFactoryForTest( + async () => + ({ + request, + addNotificationHandler: (handler: typeof notify) => { + notify = handler; + return () => undefined; + }, + addRequestHandler: ( + handler: (request: { + id: string; + method: string; + params?: unknown; + }) => Promise, + ) => { + handleRequest = handler; + return () => undefined; + }, + }) as never, + ); + const params = createParams( + path.join(tempDir, "session.jsonl"), + path.join(tempDir, "workspace"), + ); + params.timeoutMs = 60_000; + + let settled = false; + const run = runCodexAppServerAttempt(params, { + turnCompletionIdleTimeoutMs: 500, + turnAssistantCompletionIdleTimeoutMs: 5, + postToolRawAssistantCompletionIdleTimeoutMs: 50, + turnTerminalIdleTimeoutMs: 500, + }).finally(() => { + settled = true; + }); + await vi.waitFor(() => expect(handleRequest).toBeTypeOf("function"), fastWait); + + const toolResult = (await handleRequest?.({ + id: "request-tool-1", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "message", + arguments: { action: "send", text: "already sent" }, + }, + })) as { success?: boolean }; + expect(toolResult.success).toBe(false); + await notify({ + method: "rawResponseItem/completed", + params: { + threadId: "thread-1", + turnId: "turn-1", + item: { + type: "message", + id: "raw-status-1", + role: "assistant", + content: [{ type: "output_text", text: "I'm writing a large patch now." }], + }, + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 30)); + // This covers the future-compatible path for raw response deltas if Codex + // app-server exposes them directly; current Codex primarily emits + // rawResponseItem/completed for the raw-event surface. + await notify({ + method: "response.custom_tool_call_input.delta", + params: { + item_id: "ctc-large-edit-1", + output_index: 0, + delta: '{"cmd":"apply_patch","patch":"large chunk"}', + }, + }); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(settled).toBe(false); + + await notify({ + method: "turn/completed", + params: { + threadId: "thread-1", + turnId: "turn-1", + turn: { id: "turn-1", status: "completed" }, + }, + }); + + const result = await run; + expect(result.aborted).toBe(false); + expect(result.timedOut).toBe(false); + expect(result.promptError).toBeNull(); + }); + + it("keeps the post-tool guard armed for scoped native response deltas", async () => { + let notify: (notification: CodexServerNotification) => Promise = async () => undefined; + let handleRequest: + | ((request: { id: string; method: string; params?: unknown }) => Promise) + | undefined; + const request = vi.fn(async (method: string) => { + if (method === "thread/start") { + return threadStartResult("thread-1"); + } + if (method === "turn/start") { + return turnStartResult("turn-1", "inProgress"); + } + return {}; + }); + setCodexAppServerClientFactoryForTest( + async () => + ({ + request, + addNotificationHandler: (handler: typeof notify) => { + notify = handler; + return () => undefined; + }, + addRequestHandler: ( + handler: (request: { + id: string; + method: string; + params?: unknown; + }) => Promise, + ) => { + handleRequest = handler; + return () => undefined; + }, + }) as never, + ); + const params = createParams( + path.join(tempDir, "session-scoped-delta-timeout.jsonl"), + path.join(tempDir, "workspace-scoped-delta-timeout"), + ); + params.timeoutMs = 2_000; + + const run = runCodexAppServerAttempt(params, { + turnCompletionIdleTimeoutMs: 500, + turnAssistantCompletionIdleTimeoutMs: 5, + postToolRawAssistantCompletionIdleTimeoutMs: 50, + turnTerminalIdleTimeoutMs: 500, + }); + await vi.waitFor(() => expect(handleRequest).toBeTypeOf("function"), fastWait); + + await handleRequest?.({ + id: "request-tool-1", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "message", + arguments: { action: "send", text: "already sent" }, + }, + }); + await notify({ + method: "rawResponseItem/completed", + params: { + threadId: "thread-1", + turnId: "turn-1", + item: { + type: "message", + id: "raw-status-1", + role: "assistant", + content: [{ type: "output_text", text: "I'm writing a large patch now." }], + }, + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 30)); + await notify({ + method: "response.custom_tool_call_input.delta", + params: { + threadId: "thread-1", + turnId: "turn-1", + item_id: "ctc-large-edit-1", + output_index: 0, + delta: '{"cmd":"apply_patch","patch":"large chunk"}', + }, + }); + + const result = await run; + expect(result.timedOut).toBe(true); + expect(result.promptError).toBe( + "codex app-server turn idle timed out waiting for turn/completed", + ); + }); + + it("ignores unscoped native response deltas while another turn leases the client", async () => { + let notify: (notification: CodexServerNotification) => Promise = async () => undefined; + let handleRequest: + | ((request: { id: string; method: string; params?: unknown }) => Promise) + | undefined; + const warn = vi.spyOn(embeddedAgentLog, "warn").mockImplementation(() => undefined); + const request = vi.fn(async (method: string) => { + if (method === "thread/start") { + return threadStartResult("thread-1"); + } + if (method === "turn/start") { + return turnStartResult("turn-1", "inProgress"); + } + return {}; + }); + setCodexAppServerClientFactoryForTest( + async () => + ({ + request, + getActiveSharedLeaseCountForUnscopedNotifications: () => 2, + addNotificationHandler: (handler: typeof notify) => { + notify = handler; + return () => undefined; + }, + addRequestHandler: ( + handler: (request: { + id: string; + method: string; + params?: unknown; + }) => Promise, + ) => { + handleRequest = handler; + return () => undefined; + }, + }) as never, + ); + const params = createParams( + path.join(tempDir, "session.jsonl"), + path.join(tempDir, "workspace"), + ); + params.timeoutMs = 60_000; + + let settled = false; + const run = runCodexAppServerAttempt(params, { + turnCompletionIdleTimeoutMs: 500, + turnAssistantCompletionIdleTimeoutMs: 5, + postToolRawAssistantCompletionIdleTimeoutMs: 80, + turnTerminalIdleTimeoutMs: 500, + }).finally(() => { + settled = true; + }); + await vi.waitFor(() => expect(handleRequest).toBeTypeOf("function"), fastWait); + + await handleRequest?.({ + id: "request-tool-1", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "message", + arguments: { action: "send", text: "already sent" }, + }, + }); + await notify({ + method: "rawResponseItem/completed", + params: { + threadId: "thread-1", + turnId: "turn-1", + item: { + type: "message", + id: "raw-status-1", + role: "assistant", + content: [{ type: "output_text", text: "I'm writing a large patch now." }], + }, + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 40)); + await notify({ + method: "response.custom_tool_call_input.delta", + params: { + item_id: "foreign-large-edit-1", + output_index: 0, + delta: '{"cmd":"apply_patch","patch":"other turn"}', + }, + }); + await vi.waitFor(() => expect(settled).toBe(true), fastWait); + + const result = await run; + expect(result.aborted).toBe(true); + expect(result.timedOut).toBe(true); + expect(result.promptError).toBe( + "codex app-server turn idle timed out waiting for turn/completed", + ); + const completionWarnCall = warn.mock.calls.find( + ([message]) => message === "codex app-server turn idle timed out waiting for completion", + ); + const completionWarnData = completionWarnCall?.[1] as + | { + lastActivityReason?: string; + lastNotificationMethod?: string; + } + | undefined; + expect(completionWarnData?.lastActivityReason).toBe("notification:rawResponseItem/completed"); + expect(completionWarnData?.lastNotificationMethod).toBe("rawResponseItem/completed"); + }); + + it("times out post-native-tool raw assistant progress after the post-tool timeout", async () => { let notify: (notification: CodexServerNotification) => Promise = async () => undefined; const request = vi.fn(async (method: string) => { if (method === "thread/start") { @@ -1362,6 +1677,7 @@ describe("runCodexAppServerAttempt turn watches", () => { const run = runCodexAppServerAttempt(params, { turnCompletionIdleTimeoutMs: 100, turnAssistantCompletionIdleTimeoutMs: 5, + postToolRawAssistantCompletionIdleTimeoutMs: 5, turnTerminalIdleTimeoutMs: 500, }); await vi.waitFor( @@ -1831,6 +2147,7 @@ describe("runCodexAppServerAttempt turn watches", () => { promptError: "codex app-server turn idle timed out waiting for turn/completed", codexAppServerFailure: { kind: "turn_completion_idle_timeout", + turnWatchTimeoutKind: "completion", transport: "stdio", threadId: "thread-1", turnId: "turn-1", diff --git a/extensions/codex/src/app-server/session-binding.test.ts b/extensions/codex/src/app-server/session-binding.test.ts index 036288139dbc..c799772b6e96 100644 --- a/extensions/codex/src/app-server/session-binding.test.ts +++ b/extensions/codex/src/app-server/session-binding.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { clearCodexAppServerBinding, + clearCodexAppServerBindingForThread, readCodexAppServerBinding, resolveCodexAppServerBindingPath, writeCodexAppServerBinding, @@ -302,4 +303,26 @@ describe("codex app-server session binding", () => { await clearCodexAppServerBinding(sessionFile); await expect(readCodexAppServerBinding(sessionFile)).resolves.toBeUndefined(); }); + + it("clears a binding only when the thread matches", async () => { + const sessionFile = path.join(tempDir, "session.json"); + await writeCodexAppServerBinding(sessionFile, { + threadId: "thread-current", + cwd: tempDir, + model: "gpt-5.4-codex", + modelProvider: "openai", + }); + + await expect( + clearCodexAppServerBindingForThread(sessionFile, "thread-transient"), + ).resolves.toBe(false); + await expect(readCodexAppServerBinding(sessionFile)).resolves.toMatchObject({ + threadId: "thread-current", + }); + + await expect(clearCodexAppServerBindingForThread(sessionFile, "thread-current")).resolves.toBe( + true, + ); + await expect(readCodexAppServerBinding(sessionFile)).resolves.toBeUndefined(); + }); }); diff --git a/extensions/codex/src/app-server/session-binding.ts b/extensions/codex/src/app-server/session-binding.ts index 94f6491d6cfd..c41075e595f7 100644 --- a/extensions/codex/src/app-server/session-binding.ts +++ b/extensions/codex/src/app-server/session-binding.ts @@ -300,6 +300,27 @@ export async function clearCodexAppServerBinding( } } +export async function clearCodexAppServerBindingForThread( + sessionFile: string, + threadId: string, + lookup: Omit = {}, +): Promise { + const binding = await readCodexAppServerBinding(sessionFile, lookup); + if (!binding) { + return false; + } + if (binding.threadId !== threadId) { + embeddedAgentLog.debug("codex app-server binding points at a different thread; preserving", { + sessionFile, + threadId, + boundThreadId: binding.threadId, + }); + return false; + } + await clearCodexAppServerBinding(sessionFile); + return true; +} + function isNotFound(error: unknown): boolean { return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT"); } diff --git a/extensions/codex/src/app-server/shared-client.ts b/extensions/codex/src/app-server/shared-client.ts index 83bacfc41998..037a6534bfa4 100644 --- a/extensions/codex/src/app-server/shared-client.ts +++ b/extensions/codex/src/app-server/shared-client.ts @@ -199,6 +199,7 @@ async function acquireSharedCodexAppServerClient( (entry.promise = (async () => { const client = CodexAppServerClient.start(startOptions); entry.client = client; + client.setActiveSharedLeaseCountProviderForUnscopedNotifications(() => entry.activeLeases); client.addCloseHandler((closedClient) => clearSharedClientEntryIfCurrent(key, closedClient)); try { await client.initialize(); @@ -223,6 +224,7 @@ async function acquireSharedCodexAppServerClient( options?.timeoutMs ?? 0, "codex app-server initialize timed out", ); + client.setActiveSharedLeaseCountProviderForUnscopedNotifications(() => entry.activeLeases); const release = leaseOptions?.leased ? retainSharedClientEntry(entry) : undefined; return release ? { client, release } : { client }; } catch (error) { diff --git a/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts b/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts index ee005d941595..d27a07b03b3c 100644 --- a/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts +++ b/src/agents/embedded-agent-runner/run.codex-app-server-recovery.test.ts @@ -3,7 +3,6 @@ import { makeModelFallbackCfg } from "../test-helpers/model-fallback-config-fixt import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; import { loadRunOverflowCompactionHarness, - MockedFailoverError, mockedClassifyFailoverReason, mockedMarkAuthProfileFailure, mockedRunEmbeddedAttempt, @@ -32,6 +31,27 @@ function codexClientClosedAttempt( }); } +function codexTurnCompletionIdleTimeoutAttempt( + overrides: Partial = {}, +): EmbeddedRunAttemptResult { + return makeAttemptResult({ + assistantTexts: [], + aborted: true, + timedOut: true, + promptError: new Error("codex app-server turn idle timed out waiting for turn/completed"), + promptErrorSource: "prompt", + codexAppServerFailure: { + kind: "turn_completion_idle_timeout", + turnWatchTimeoutKind: "completion", + transport: "stdio", + threadId: "thread-1", + turnId: "turn-1", + replaySafe: true, + }, + ...overrides, + }); +} + function successAttempt(): EmbeddedRunAttemptResult { return makeAttemptResult({ promptError: null, @@ -138,14 +158,27 @@ describe("runEmbeddedAgent Codex app-server recovery", () => { expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); }); - it("does not retry turn/completed idle timeouts", async () => { + it("retries a replay-safe stdio turn/completed idle timeout once", async () => { + mockedRunEmbeddedAttempt + .mockResolvedValueOnce(codexTurnCompletionIdleTimeoutAttempt()) + .mockResolvedValueOnce(successAttempt()); + + await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "codex", + model: "gpt-5.5", + runId: "run-codex-turn-completion-idle-timeout", + }); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + }); + + it("does not retry non-completion Codex turn watch timeouts", async () => { mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - promptError: new Error("codex app-server turn idle timed out waiting for turn/completed"), - promptErrorSource: "prompt", + codexTurnCompletionIdleTimeoutAttempt({ codexAppServerFailure: { kind: "turn_completion_idle_timeout", + turnWatchTimeoutKind: "progress", transport: "stdio", threadId: "thread-1", turnId: "turn-1", @@ -154,35 +187,98 @@ describe("runEmbeddedAgent Codex app-server recovery", () => { }), ); - await expect( - runEmbeddedAgent({ - ...overflowBaseRunParams, - provider: "codex", - model: "gpt-5.5", - runId: "run-codex-turn-completion-idle-timeout", - }), - ).rejects.toThrow("codex app-server turn idle timed out waiting for turn/completed"); + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "codex", + model: "gpt-5.5", + runId: "run-codex-progress-idle-timeout", + }); + + expect(result.payloads?.[0]).toMatchObject({ + isError: true, + text: "Request timed out before a response was generated. Please try again, or increase `agents.defaults.timeoutSeconds` in your config.", + }); expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(mockedMarkAuthProfileFailure).not.toHaveBeenCalled(); + }); + + it("returns a timeout payload after a replay-safe turn/completed idle timeout retry is exhausted", async () => { + mockedRunEmbeddedAttempt + .mockResolvedValueOnce(codexTurnCompletionIdleTimeoutAttempt()) + .mockResolvedValueOnce(codexTurnCompletionIdleTimeoutAttempt()); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "codex", + model: "gpt-5.5", + runId: "run-codex-turn-completion-idle-timeout-retry-exhausted", + }); + + expect(result.payloads?.[0]).toMatchObject({ + isError: true, + text: "Request timed out before a response was generated. Please try again, or increase `agents.defaults.timeoutSeconds` in your config.", + }); + expect(result.meta.timeoutPhase).toBe("provider"); + expect(result.meta.providerStarted).toBe(true); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(mockedMarkAuthProfileFailure).not.toHaveBeenCalled(); + }); + + it("surfaces non-stdio turn/completed idle timeouts instead of throwing", async () => { + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + codexTurnCompletionIdleTimeoutAttempt({ + codexAppServerFailure: { + kind: "turn_completion_idle_timeout", + turnWatchTimeoutKind: "completion", + transport: "websocket", + threadId: "thread-1", + turnId: "turn-1", + replaySafe: true, + }, + }), + ); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "codex", + model: "gpt-5.5", + runId: "run-codex-turn-completion-idle-timeout-websocket", + }); + + expect(result.payloads?.[0]).toMatchObject({ + isError: true, + text: "Request timed out before a response was generated. Please try again, or increase `agents.defaults.timeoutSeconds` in your config.", + }); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(mockedMarkAuthProfileFailure).not.toHaveBeenCalled(); }); it("does not hand Codex app-server idle timeouts to model fallback", async () => { mockedClassifyFailoverReason.mockReturnValue("timeout"); mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - promptError: new Error("codex app-server turn idle timed out waiting for turn/completed"), - promptErrorSource: "prompt", + codexTurnCompletionIdleTimeoutAttempt({ + timedOut: true, + didSendViaMessagingTool: true, + replayMetadata: { hadPotentialSideEffects: true, replaySafe: false }, + promptTimeoutOutcome: { + message: + "Codex stopped before confirming the turn was complete. Some work may already have been performed; verify the current state before retrying.", + replayInvalid: true, + livenessState: "abandoned", + }, codexAppServerFailure: { kind: "turn_completion_idle_timeout", + turnWatchTimeoutKind: "completion", transport: "stdio", threadId: "thread-1", turnId: "turn-1", - replaySafe: true, + replaySafe: false, + replayBlockedReason: "potential_side_effect", }, }), ); - const promise = runEmbeddedAgent({ + const result = await runEmbeddedAgent({ ...overflowBaseRunParams, provider: "codex", model: "gpt-5.5", @@ -199,10 +295,12 @@ describe("runEmbeddedAgent Codex app-server recovery", () => { }), }); - await expect(promise).rejects.not.toBeInstanceOf(MockedFailoverError); - await expect(promise).rejects.toThrow( - "codex app-server turn idle timed out waiting for turn/completed", - ); + expect(result.payloads?.[0]).toMatchObject({ + isError: true, + text: "Codex stopped before confirming the turn was complete. Some work may already have been performed; verify the current state before retrying.", + }); + expect(result.meta.replayInvalid).toBe(true); + expect(result.meta.livenessState).toBe("abandoned"); expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); expect(mockedMarkAuthProfileFailure).not.toHaveBeenCalled(); }); diff --git a/src/agents/embedded-agent-runner/run.ts b/src/agents/embedded-agent-runner/run.ts index 158480db7428..4b09aaf1889e 100644 --- a/src/agents/embedded-agent-runner/run.ts +++ b/src/agents/embedded-agent-runner/run.ts @@ -134,7 +134,7 @@ import { forgetPromptBuildDrainCacheForRun } from "./run/attempt.prompt-helpers. import { createEmbeddedRunAuthController } from "./run/auth-controller.js"; import { resolveAuthProfileFailureReason } from "./run/auth-profile-failure-policy.js"; import { runEmbeddedAttemptWithBackend } from "./run/backend.js"; -import { resolveCodexAppServerClientCloseRetry } from "./run/codex-app-server-recovery.js"; +import { resolveCodexAppServerRecoveryRetry } from "./run/codex-app-server-recovery.js"; import { createFailoverDecisionLogger } from "./run/failover-observation.js"; import { mergeRetryFailoverReason, resolveRunFailoverDecision } from "./run/failover-policy.js"; import { hasEmbeddedRunConfiguredModelFallbacks } from "./run/fallbacks.js"; @@ -1209,7 +1209,7 @@ export async function runEmbeddedAgent( }); let rateLimitProfileRotations = 0; let timeoutCompactionAttempts = 0; - let codexAppServerClientCloseRetries = 0; + let codexAppServerRecoveryRetries = 0; // Silent-error retry: non-strict-agentic models (e.g. ollama/glm-5.1) can // end a turn with stopReason="error" + zero output tokens, producing no // user-visible text. This is an orthogonal, model-agnostic resubmission @@ -2378,26 +2378,47 @@ export async function runEmbeddedAgent( }; } - if (promptError && !aborted && promptErrorSource !== "compaction") { - const codexClientCloseRetry = resolveCodexAppServerClientCloseRetry({ + const hasRecoverableCodexAppServerTimeoutOutcome = Boolean( + attempt.codexAppServerFailure && attempt.promptTimeoutOutcome, + ); + let shouldSurfaceCodexCompletionTimeout = false; + if (promptError && promptErrorSource !== "compaction" && attempt.codexAppServerFailure) { + // Retry replay-safe Codex app-server failures. + const codexAppServerRecoveryRetry = resolveCodexAppServerRecoveryRetry({ attempt, - alreadyRetried: codexAppServerClientCloseRetries > 0, + alreadyRetried: codexAppServerRecoveryRetries > 0, }); - if (codexClientCloseRetry.retry) { - codexAppServerClientCloseRetries += 1; + if (codexAppServerRecoveryRetry.retry) { + codexAppServerRecoveryRetries += 1; suppressNextUserMessagePersistence = true; log.warn( - `codex app-server stdio client closed before turn completion; retrying once ` + + `codex app-server replay-safe failure; retrying once ` + + `failureKind=${attempt.codexAppServerFailure?.kind} ` + `runId=${params.runId} sessionId=${params.sessionId}`, ); continue; } - if (attempt.codexAppServerFailure) { + // Completion-idle timeouts are timeout outcomes even when the + // app-server transport is not retryable, or the retry was exhausted. + shouldSurfaceCodexCompletionTimeout = + attempt.codexAppServerFailure?.kind === "turn_completion_idle_timeout" && + attempt.timedOut; + if ( + attempt.codexAppServerFailure && + !hasRecoverableCodexAppServerTimeoutOutcome && + !shouldSurfaceCodexCompletionTimeout + ) { throw promptError; } } - if (promptError && !aborted && promptErrorSource !== "compaction") { + if ( + promptError && + !aborted && + promptErrorSource !== "compaction" && + !hasRecoverableCodexAppServerTimeoutOutcome && + !shouldSurfaceCodexCompletionTimeout + ) { // Normalize wrapped errors (e.g. abort-wrapped RESOURCE_EXHAUSTED) into // FailoverError so rate-limit classification works even for nested shapes. // @@ -2961,7 +2982,7 @@ export async function runEmbeddedAgent( if ( timedOutDuringPrompt && !hasSuccessfulFinalAssistantAfterPromptTimeout && - !hasMessagingToolDeliveryEvidence(attempt) + (shouldSurfaceCodexCompletionTimeout || !hasMessagingToolDeliveryEvidence(attempt)) ) { const defaultTimeoutText = idleTimedOut ? "The model did not produce a response before the model idle timeout. " + diff --git a/src/agents/embedded-agent-runner/run/codex-app-server-recovery.ts b/src/agents/embedded-agent-runner/run/codex-app-server-recovery.ts index 1cf243f74fb8..e92800d7b41d 100644 --- a/src/agents/embedded-agent-runner/run/codex-app-server-recovery.ts +++ b/src/agents/embedded-agent-runner/run/codex-app-server-recovery.ts @@ -1,6 +1,6 @@ import type { EmbeddedRunAttemptResult } from "./types.js"; -export function resolveCodexAppServerClientCloseRetry(params: { +export function resolveCodexAppServerRecoveryRetry(params: { attempt: EmbeddedRunAttemptResult; alreadyRetried: boolean; }): { retry: boolean; reason?: string } { @@ -8,9 +8,18 @@ export function resolveCodexAppServerClientCloseRetry(params: { if (!failure) { return { retry: false, reason: "not_codex_app_server_failure" }; } - if (failure.kind !== "client_closed_before_turn_completed") { + if ( + failure.kind !== "client_closed_before_turn_completed" && + failure.kind !== "turn_completion_idle_timeout" + ) { return { retry: false, reason: failure.kind }; } + if ( + failure.kind === "turn_completion_idle_timeout" && + failure.turnWatchTimeoutKind !== "completion" + ) { + return { retry: false, reason: failure.turnWatchTimeoutKind ?? "unknown_turn_watch_timeout" }; + } if (failure.transport !== "stdio") { return { retry: false, reason: "non_stdio_transport" }; } @@ -39,3 +48,5 @@ export function resolveCodexAppServerClientCloseRetry(params: { } return { retry: true }; } + +export const resolveCodexAppServerClientCloseRetry = resolveCodexAppServerRecoveryRetry; diff --git a/src/agents/embedded-agent-runner/run/types.ts b/src/agents/embedded-agent-runner/run/types.ts index be81f0a21b39..85b3679dd027 100644 --- a/src/agents/embedded-agent-runner/run/types.ts +++ b/src/agents/embedded-agent-runner/run/types.ts @@ -125,6 +125,7 @@ export type EmbeddedRunAttemptResult = { }; codexAppServerFailure?: { kind: "client_closed_before_turn_completed" | "turn_completion_idle_timeout"; + turnWatchTimeoutKind?: "progress" | "completion" | "terminal"; transport: "stdio" | "websocket"; threadId?: string; turnId?: string;