From d204ebe6a4dfd4500bb62a113c2abc10d1cab4d5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 18 Aug 2026 10:22:35 -0700 Subject: [PATCH] fix(codex): recover replies after compaction failure (#125895) Preserve completed tool work when native Codex compaction fails, close failed compaction progress, and bypass unrelated model/auth failover before isolated finalization. Fixes #125789. --- .../event-projector.terminal-errors.test.ts | 105 ++++++++++++++++++ .../codex/src/app-server/event-projector.ts | 55 ++++++--- .../codex/src/app-server/run-attempt.test.ts | 77 +++++++++++++ .../run/attempt-recovery.test.ts | 99 +++++++++++++++++ .../run/settled-turn-finalization.test.ts | 5 + 5 files changed, 324 insertions(+), 17 deletions(-) diff --git a/extensions/codex/src/app-server/event-projector.terminal-errors.test.ts b/extensions/codex/src/app-server/event-projector.terminal-errors.test.ts index ea2c77e2a98d..04f8eca328ad 100644 --- a/extensions/codex/src/app-server/event-projector.terminal-errors.test.ts +++ b/extensions/codex/src/app-server/event-projector.terminal-errors.test.ts @@ -3,9 +3,11 @@ import { registerCodexEventProjectorTestLifecycle, expect, it, + THREAD_ID, TURN_ID, createProjector, buildEmptyToolTelemetry, + createParams, readAttemptTerminal, expectUsageLimitPromptError, forCurrentTurn, @@ -15,6 +17,7 @@ import { turnCompleted, turnWithStatus, pendingCommandStarted, + vi, } from "./event-projector.test-harness.js"; registerCodexEventProjectorTestLifecycle(); @@ -202,6 +205,7 @@ describe("CodexAppServerEventProjector terminal errors", () => { { codexErrorInfo: "serverOverloaded", expected: true }, { codexErrorInfo: "usageLimitExceeded", expected: false }, { codexErrorInfo: "unauthorized", expected: false }, + { codexErrorInfo: "other", expected: false }, ])( "projects $codexErrorInfo terminal error recovery eligibility as $expected", async ({ codexErrorInfo, expected }) => { @@ -212,9 +216,110 @@ describe("CodexAppServerEventProjector terminal errors", () => { ); expect(projector.settledTurnFailureFinalizationAllowed).toBe(expected); + expect( + readAttemptTerminal(projector.buildResult(buildEmptyToolTelemetry())).promptErrorSource, + ).toBe("prompt"); }, ); + it("keeps an active native compaction failure scoped through the failed turn", async () => { + const onAgentEvent = vi.fn(); + const onContextCompacted = vi.fn(); + const projector = await createProjector( + { ...(await createParams()), onAgentEvent }, + { onContextCompacted }, + ); + + await projector.handleNotification( + forCurrentTurn("item/started", { + item: { type: "contextCompaction", id: "compact-failed" }, + }), + ); + await projector.handleNotification( + appServerError({ + message: "remote compaction failed", + willRetry: false, + codexErrorInfo: "other", + }), + ); + + expect(readAttemptTerminal(projector.buildResult(buildEmptyToolTelemetry()))).toMatchObject({ + promptError: "remote compaction failed", + promptErrorSource: "compaction", + }); + expect(projector.settledTurnFailureFinalizationAllowed).toBe(true); + + await projector.handleNotification( + forCurrentTurn("turn/completed", { + turn: { + id: TURN_ID, + status: "failed", + error: { message: "remote compaction failed", codexErrorInfo: "other" }, + items: [], + }, + }), + ); + + const result = projector.buildResult(buildEmptyToolTelemetry()); + expect(readAttemptTerminal(result)).toMatchObject({ + promptError: "remote compaction failed", + promptErrorSource: "compaction", + }); + expect(projector.settledTurnFailureFinalizationAllowed).toBe(true); + expect(projector.isCompacting()).toBe(false); + expect(result.itemLifecycle).toEqual({ startedCount: 0, completedCount: 0, activeCount: 0 }); + expect(result.compactionCount).toBeUndefined(); + expect(onContextCompacted).not.toHaveBeenCalled(); + expect( + onAgentEvent.mock.calls + .map(([event]) => event) + .filter((event) => event.stream === "compaction"), + ).toEqual([ + { + stream: "compaction", + data: { + phase: "start", + backend: "codex-app-server", + threadId: THREAD_ID, + turnId: TURN_ID, + itemId: "compact-failed", + }, + }, + { + stream: "compaction", + data: { + phase: "end", + backend: "codex-app-server", + completed: false, + threadId: THREAD_ID, + turnId: TURN_ID, + itemId: "compact-failed", + }, + }, + ]); + }); + + it("keeps other errors prompt-scoped after native compaction completes", async () => { + const projector = await createProjector(); + const compaction = { item: { type: "contextCompaction", id: "compact-completed" } }; + + await projector.handleNotification(forCurrentTurn("item/started", compaction)); + await projector.handleNotification(forCurrentTurn("item/completed", compaction)); + await projector.handleNotification( + appServerError({ + message: "unrelated provider failure", + willRetry: false, + codexErrorInfo: "other", + }), + ); + + expect(readAttemptTerminal(projector.buildResult(buildEmptyToolTelemetry()))).toMatchObject({ + promptError: "unrelated provider failure", + promptErrorSource: "prompt", + }); + expect(projector.settledTurnFailureFinalizationAllowed).toBe(false); + }); + it("uses Codex rate-limit resets for usage-limit app-server errors", async () => { const resetsAt = Math.ceil(Date.now() / 1000) + 120; const projector = await createProjector(undefined, { diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index 2174cd950123..03dc64024649 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -301,17 +301,19 @@ export class CodexAppServerEventProjector { case "rawResponseItem/completed": await this.handleRawResponseItemCompleted(params); break; - case "error": + case "error": { this.responseCompletions.clear(); if (params.willRetry === true) { break; } + const codexErrorInfo = isJsonObject(params.error) ? params.error.codexErrorInfo : undefined; + const compactionFailure = codexErrorInfo === "other" && this.isCompacting(); this.settledTurnFailureFinalizationAllowed = - (isJsonObject(params.error) ? params.error.codexErrorInfo : undefined) === - "serverOverloaded"; + codexErrorInfo === "serverOverloaded" || compactionFailure; this.promptError = this.formatCodexErrorMessage(params) ?? "codex app-server error"; - this.promptErrorSource = "prompt"; + this.promptErrorSource = compactionFailure ? "compaction" : "prompt"; break; + } case "thread/compacted": case "turn/started": case "turn/diff/updated": @@ -490,17 +492,7 @@ export class CodexAppServerEventProjector { channelId: this.params.messageChannel ?? this.params.messageProvider ?? undefined, }, }); - this.emitAgentEvent({ - stream: "compaction", - data: { - phase: "end", - backend: "codex-app-server", - completed: true, - threadId: this.threadId, - turnId: this.turnId, - itemId, - }, - }); + this.emitCompactionEnd(itemId, true); } this.toolProgressProjection.recordToolMeta(item); this.toolProgressProjection.rememberCommandAggregateOutputEcho(item); @@ -522,8 +514,13 @@ export class CodexAppServerEventProjector { return; } this.completedTurn = turn; + const compactionFailure = + turn.status === "failed" && + (this.promptErrorSource === "compaction" || + (turn.error?.codexErrorInfo === "other" && this.isCompacting())); this.settledTurnFailureFinalizationAllowed = - turn.status === "failed" && turn.error?.codexErrorInfo === "serverOverloaded"; + turn.status === "failed" && + (turn.error?.codexErrorInfo === "serverOverloaded" || compactionFailure); if (turn.status !== "completed") { this.responseCompletions.clear(); } @@ -536,7 +533,17 @@ export class CodexAppServerEventProjector { this.promptError = usageLimitMessage ? createCodexUsageLimitPromptError(usageLimitMessage) : (turn.error?.message ?? "codex app-server turn failed"); - this.promptErrorSource = "prompt"; + this.promptErrorSource = compactionFailure ? "compaction" : "prompt"; + } + if (compactionFailure) { + // Codex omits item/completed on failure, so the terminal turn must close + // every active structural compaction for state and stream consumers. + const failedCompactionItemIds = [...this.activeCompactionItemIds]; + for (const itemId of failedCompactionItemIds) { + this.activeItemIds.delete(itemId); + this.activeCompactionItemIds.delete(itemId); + this.emitCompactionEnd(itemId, false); + } } const turnItems = turn.items ?? []; // The final snapshot is authoritative when item notifications were omitted. @@ -574,6 +581,20 @@ export class CodexAppServerEventProjector { await this.reasoningProjection.maybeEndReasoning(); } + private emitCompactionEnd(itemId: string, completed: boolean): void { + this.emitAgentEvent({ + stream: "compaction", + data: { + phase: "end", + backend: "codex-app-server", + completed, + threadId: this.threadId, + turnId: this.turnId, + itemId, + }, + }); + } + private async emitSnapshotOnlyNativeToolProgress(item: CodexThreadItem): Promise { if ( !shouldSynthesizeToolProgressForItem(item) || diff --git a/extensions/codex/src/app-server/run-attempt.test.ts b/extensions/codex/src/app-server/run-attempt.test.ts index ad0c33d1382a..1d3fd91ead40 100644 --- a/extensions/codex/src/app-server/run-attempt.test.ts +++ b/extensions/codex/src/app-server/run-attempt.test.ts @@ -4586,6 +4586,83 @@ describe("runCodexAppServerAttempt", () => { } }, ); + it("captures settled tool evidence when an active native compaction fails terminally", async () => { + const storePath = path.join(tempDir, "settled-compaction-failure.sqlite"); + const sessionId = "session-settled-compaction-failure"; + const sessionFile = `agent:main:${sessionId}`; + const workspaceDir = path.join(tempDir, "workspace-settled-compaction-failure"); + const harness = createStartedThreadHarness(); + const params = createParams(sessionFile, workspaceDir); + await attachSqliteSessionTarget(params, storePath, sessionId); + params.prompt = "Finish the task and report the result."; + const run = runCodexAppServerAttempt(params); + await harness.waitForMethod("turn/start"); + await harness.notify( + itemNotification("item/started", { + type: "commandExecution", + id: "tool-settled", + command: "echo completed-work", + cwd: workspaceDir, + status: "inProgress", + }), + ); + await harness.notify( + itemNotification("item/completed", { + type: "commandExecution", + id: "tool-settled", + command: "echo completed-work", + cwd: workspaceDir, + status: "completed", + aggregatedOutput: "completed-work\n", + exitCode: 0, + durationMs: 12, + }), + ); + await harness.notify( + itemNotification("item/started", { type: "contextCompaction", id: "compact-failed" }), + ); + await harness.notify({ + method: "error", + params: { + threadId: "thread-1", + turnId: "turn-1", + error: { + message: "remote compaction failed", + codexErrorInfo: "other", + additionalDetails: null, + }, + willRetry: false, + }, + }); + await harness.notify( + turnCompleted({ + id: "turn-1", + status: "failed", + error: { + message: "remote compaction failed", + codexErrorInfo: "other", + additionalDetails: null, + }, + }), + ); + + const result = await run; + + expect(readAttemptTerminal(result)).toMatchObject({ + promptError: "remote compaction failed", + promptErrorSource: "compaction", + }); + expect(result.itemLifecycle).toEqual({ startedCount: 1, completedCount: 1, activeCount: 0 }); + expect(result.settledTurnFinalizationContext).toMatchObject({ + source: "openclaw-transcript", + messages: [ + expect.objectContaining({ role: "user" }), + expect.objectContaining({ role: "assistant" }), + expect.objectContaining({ role: "toolResult", toolCallId: "tool-settled" }), + ], + }); + expect(Object.isFrozen(result.settledTurnFinalizationContext?.messages)).toBe(true); + }); it("preserves every command failure from official app-server events", async () => { const sessionFile = path.join(tempDir, "session-multi-command-failure.jsonl"); const workspaceDir = path.join(tempDir, "workspace-multi-command-failure"); diff --git a/src/agents/embedded-agent-runner/run/attempt-recovery.test.ts b/src/agents/embedded-agent-runner/run/attempt-recovery.test.ts index c7315d63668a..6eb3bc74d714 100644 --- a/src/agents/embedded-agent-runner/run/attempt-recovery.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-recovery.test.ts @@ -7,6 +7,7 @@ import { import { normalizeUsage } from "../../usage.js"; import { createUsageAccumulator } from "../usage-accumulator.js"; import { recoverEmbeddedRunAttempt } from "./attempt-recovery.js"; +import { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; import { resolveEmbeddedRunAttemptTerminalState } from "./terminal-outcome.js"; describe("recoverEmbeddedRunAttempt", () => { @@ -96,4 +97,102 @@ describe("recoverEmbeddedRunAttempt", () => { }, }); }); + + it("bypasses prompt failover for an operation-scoped compaction failure", async () => { + const promptFailover = vi.fn(async () => { + throw new Error("prompt failover must not run"); + }); + const assistant = buildEmbeddedRunnerAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool-read", name: "read", arguments: {} }], + }); + const messagesSnapshot = [ + assistant, + { role: "toolResult", toolCallId: "tool-read", toolName: "read", isError: false }, + ] as never; + const failoverRetryController = { + resolveAuthProfileFailureReason: vi.fn(), + advanceAuthProfile: vi.fn(), + advanceRateLimitAuthProfile: vi.fn(), + maybeMarkAuthProfileFailure: vi.fn(), + maybeBackoffBeforeOverloadFailover: vi.fn(), + }; + const attempt = makeEmbeddedRunnerAttempt({ + terminal: { + kind: "failed", + source: "compaction", + error: new Error("unexpected status 404"), + }, + messagesSnapshot, + lastAssistant: assistant, + currentAttemptAssistant: assistant, + settledTurnFinalizationContext: { + source: "openclaw-transcript", + messages: messagesSnapshot, + }, + replayMetadata: { hadPotentialSideEffects: false, replaySafe: true }, + currentAttemptReplayMetadata: { hadPotentialSideEffects: false, replaySafe: true }, + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + }); + const terminalState = resolveEmbeddedRunAttemptTerminalState({ attempt, assistant }); + + const recovery = await recoverEmbeddedRunAttempt({ + runInput: { + runParams: { + config: {}, + agentId: "main", + sessionId: "session:compaction-failure", + runId: "run:compaction-failure", + }, + resolvedSessionKey: "agent:main:compaction-failure", + startedAtMs: Date.now(), + }, + preparedRuntime: { + provider: "openai", + modelId: "gpt-5.6-luna", + model: { id: "gpt-5.6-luna" }, + genericCompactionRecoveryAllowed: false, + maybeRefreshRuntimeAuthForAuthError: promptFailover, + snapshot: () => ({ + thinkLevel: "off", + agentHarness: { id: "codex" }, + outerContextTokenMeta: {}, + lastProfileId: "profile-1", + pluginHarnessOwnsTransport: false, + }), + }, + normalizedAttempt: { + attempt, + sessionIdUsed: attempt.sessionIdUsed, + attemptAssistant: assistant, + currentAttemptAssistant: assistant, + currentAttemptCompletedAssistant: undefined, + terminalState, + setTerminalLifecycleMeta: vi.fn(), + attemptCompactionCount: 0, + activeErrorContext: { provider: "openai", model: "gpt-5.6-luna" }, + resolveReplayInvalidForAttempt: () => false, + canRestartForLiveSwitch: false, + }, + runtimePlan: { auth: {} }, + sessionPromptState: { sessionFile: "/tmp/session.jsonl" }, + failoverRetryController, + compactionRuntime: {}, + contextRecoveryState: createEmbeddedRunContextRecoveryState(), + usageAccumulator: createUsageAccumulator(), + lastRunPromptUsage: undefined, + runtimeAuthRetry: false, + codexAppServerRecoveryRetryAvailable: false, + codexAppServerRecoveryRetries: 0, + lastRetryFailoverReason: null, + traceAttempts: [], + sessionAgentId: "main", + } as never); + + expect(recovery).toEqual({ action: "proceed", shouldSurfaceCodexCompletionTimeout: false }); + expect(promptFailover).not.toHaveBeenCalled(); + expect(failoverRetryController.advanceAuthProfile).not.toHaveBeenCalled(); + expect(failoverRetryController.advanceRateLimitAuthProfile).not.toHaveBeenCalled(); + expect(failoverRetryController.maybeMarkAuthProfileFailure).not.toHaveBeenCalled(); + }); }); diff --git a/src/agents/embedded-agent-runner/run/settled-turn-finalization.test.ts b/src/agents/embedded-agent-runner/run/settled-turn-finalization.test.ts index 84e870adc60c..33d6d05a7bef 100644 --- a/src/agents/embedded-agent-runner/run/settled-turn-finalization.test.ts +++ b/src/agents/embedded-agent-runner/run/settled-turn-finalization.test.ts @@ -33,6 +33,11 @@ function settledFailedAttempt(): EmbeddedRunAttemptWithReceiptEvidence { { role: "toolResult", toolCallId: "tool-exec", toolName: "exec", isError: true }, ] as never; const attempt = makeEmbeddedRunnerAttempt({ + terminal: { + kind: "failed", + source: "compaction", + error: new Error("native context compaction failed"), + }, sessionIdUsed: "session-settled", sessionFileUsed: "/tmp/session-settled.jsonl", assistantTexts: [],