From f335532deda9504ce82fc05e2e387e9f903e9938 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 18 Aug 2026 18:54:23 -0700 Subject: [PATCH] fix: restart-recovered turns no longer show fatal errors (#126087) * fix: let restart recovery own terminal settlement Once durable restart recovery is armed, propagate the canonical restart abort before terminal payload mapping so the recovered continuation owns the visible outcome. Fixes #126086 * fix(gateway): expose lifecycle context resolver * fix(gateway): keep lifecycle resolver type acyclic Own the live Gateway context resolver option beside lifecycle dispatch so current callers compile without closing the server type cycle. * test(gateway): serialize connection liveness cases The table cases share one hoisted handler mock; explicit sequential execution prevents cross-case 0/2 call interference in the concurrent CI shard. * fix: preserve restart recovery claim through settlement Mark the reply operation restart-aborted before propagating the canonical restart error so terminal cleanup cannot retire the durable recovery claim. --- ...runner-execution-terminal-failures.test.ts | 61 +++++++++++++++++++ .../reply/agent-runner-execution.ts | 3 + .../reply/agent-runner-fallback-settlement.ts | 3 + .../reply/restart-recovery-claim.test.ts | 40 ++++++++++++ src/gateway/server-instance-runtime.types.ts | 4 -- .../server-recovery-runtime-context.ts | 8 ++- ...quest-dispatch.connection-liveness.test.ts | 2 +- 7 files changed, 115 insertions(+), 6 deletions(-) diff --git a/src/auto-reply/reply/agent-runner-execution-terminal-failures.test.ts b/src/auto-reply/reply/agent-runner-execution-terminal-failures.test.ts index dbaeec9b7c90..96a60cf5c36a 100644 --- a/src/auto-reply/reply/agent-runner-execution-terminal-failures.test.ts +++ b/src/auto-reply/reply/agent-runner-execution-terminal-failures.test.ts @@ -441,6 +441,67 @@ describe("executeAgentTurn: terminal failures", () => { ).toBe(true); }); + it.each([ + { + label: "settled result", + result: { + payloads: [{ text: "completed before the restart marker was observed" }], + meta: {}, + }, + }, + { + label: "client-close error result", + result: { + payloads: [ + { + text: "Codex app-server stopped before confirming turn completion.", + isError: true, + }, + ], + meta: { error: { message: "codex app-server client closed before turn completed" } }, + }, + }, + ])("hands an armed restart recovery owner the $label", async ({ label, result }) => { + const runId = `armed-restart-${label.replaceAll(" ", "-")}`; + const { replyOperation, failMock } = createMockReplyOperation(); + let operationResult: typeof replyOperation.result = null; + const abortForRestart = vi.fn(() => { + operationResult = { kind: "aborted", code: "aborted_for_restart" }; + return true; + }); + const complete = vi.fn(() => { + operationResult ??= { kind: "completed" }; + }); + const restartReplyOperation = { + ...replyOperation, + get result() { + return operationResult; + }, + abortForRestart, + complete, + } satisfies typeof replyOperation; + state.runEmbeddedAgentMock.mockResolvedValueOnce(result); + const { executeAgentTurn } = await import("./agent-runner-execution.js"); + + const execution = await executeAgentTurn({ + ...createMinimalRunAgentTurnParams({ replyOperation: restartReplyOperation }), + opts: { runId } as GetReplyOptions, + isRestartRecoveryArmed: () => true, + }); + + expect(execution).toEqual({ + runId, + outcome: { kind: "aborted", reason: "restart" }, + }); + expect(abortForRestart).toHaveBeenCalledOnce(); + expect(complete).not.toHaveBeenCalled(); + expect(restartReplyOperation.result).toEqual({ + kind: "aborted", + code: "aborted_for_restart", + }); + expect(failMock).not.toHaveBeenCalled(); + }); + it("uses compact generic copy for raw external chat errors when verbose is off", async () => { const agentEvents = await import("../../infra/agent-events.js"); const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index 69f2d2e4c67b..ab72949fd279 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -375,6 +375,9 @@ async function executeAgentTurnInternalWithRetryState( terminalRunFailed = cycle.terminalRunFailed; break; } catch (err) { + if (isAgentRunRestartAbortReason(err)) { + throw err; + } if (err instanceof LiveSessionModelSwitchError) { liveModelSwitchRetries += 1; } diff --git a/src/auto-reply/reply/agent-runner-fallback-settlement.ts b/src/auto-reply/reply/agent-runner-fallback-settlement.ts index c2ee243f9734..c3458cfe57ba 100644 --- a/src/auto-reply/reply/agent-runner-fallback-settlement.ts +++ b/src/auto-reply/reply/agent-runner-fallback-settlement.ts @@ -45,6 +45,9 @@ export async function settleAgentFallbackCycle(params: { ? cycle.state.pendingLifecycleTerminal.backstop : undefined; cycle.state.pendingLifecycleTerminal = undefined; + if (turn.isRestartRecoveryArmed?.()) { + turn.replyOperation?.abortForRestart(); + } if (isReplyOperationRestartAbort(turn.replyOperation)) { settledLifecycleTerminal?.emit("end", runResult); throw isAgentRunRestartAbortReason(cycle.runAbortSignal?.reason) diff --git a/src/auto-reply/reply/restart-recovery-claim.test.ts b/src/auto-reply/reply/restart-recovery-claim.test.ts index 163a2a55c683..603c8fc62fc4 100644 --- a/src/auto-reply/reply/restart-recovery-claim.test.ts +++ b/src/auto-reply/reply/restart-recovery-claim.test.ts @@ -93,6 +93,46 @@ describe("createReplyRestartRecoveryClaimController", () => { }, ); + it("preserves lifecycle ownership when cleanup observes a restart abort", async () => { + const root = tempDirs.make("openclaw-reply-claim-restart-abort-"); + const storePath = path.join(root, "sessions.json"); + const sessionKey = "agent:main:main"; + const sessionId = "session"; + let restartAborted = false; + let entry: InternalSessionEntry = { + abortedLastRun: false, + lifecycleRunId: "recovery-run", + restartRecoveryDeliveryRunId: "recovery-run", + sessionId, + startedAt: 1, + status: "running", + updatedAt: 1, + }; + await replaceSessionEntry({ storePath, sessionKey }, entry); + const controller = createReplyRestartRecoveryClaimController({ + admissionRunId: "recovery-run", + getEntry: () => entry, + getSessionId: () => sessionId, + isRestartAbort: () => restartAborted, + resolveDeliveryContext: () => undefined, + sessionKey, + setEntry: (next) => { + entry = next; + }, + storePath, + }); + + await expect(controller.admitUserTurn()).resolves.toBe("admitted"); + restartAborted = true; + await controller.clear(); + + expect(loadSessionEntry({ storePath, sessionKey })).toMatchObject({ + lifecycleRunId: "recovery-run", + restartRecoveryDeliveryRunId: "recovery-run", + status: "running", + }); + }); + it("retargets durable user-turn admission to the prepared reply session", async () => { const root = tempDirs.make("openclaw-reply-admission-"); const storePath = path.join(root, "sessions.json"); diff --git a/src/gateway/server-instance-runtime.types.ts b/src/gateway/server-instance-runtime.types.ts index 7e34d7dc3624..5083717c6821 100644 --- a/src/gateway/server-instance-runtime.types.ts +++ b/src/gateway/server-instance-runtime.types.ts @@ -21,10 +21,6 @@ export type GatewayInstanceAgentDispatchOptions = { syntheticScopes?: string[]; }; -export type GatewayLifecycleAgentDispatchOptions = GatewayInstanceAgentDispatchOptions & { - timeoutMs?: number; -}; - export type GatewayApprovalEventPublisher = { publishRequested: (kind: ChannelApprovalKind, request: unknown) => number; publishResolved: (kind: ChannelApprovalKind, resolved: unknown) => void; diff --git a/src/gateway/server-recovery-runtime-context.ts b/src/gateway/server-recovery-runtime-context.ts index 2e25a79d12bf..87824bfa16e3 100644 --- a/src/gateway/server-recovery-runtime-context.ts +++ b/src/gateway/server-recovery-runtime-context.ts @@ -1,8 +1,14 @@ import type { - GatewayLifecycleAgentDispatchOptions, + GatewayInstanceAgentDispatchOptions, GatewayRecoveryRuntime, } from "./server-instance-runtime.types.js"; import type { AgentRunRequest } from "./server-methods/agent-request-types.js"; +import type { GatewayContextResolver } from "./server-methods/shared-types.js"; + +type GatewayLifecycleAgentDispatchOptions = GatewayInstanceAgentDispatchOptions & { + resolveGatewayContext?: GatewayContextResolver; + timeoutMs?: number; +}; type ActiveGatewayRecoveryRuntime = { owner: symbol; diff --git a/src/gateway/server/ws-connection/authenticated-request-dispatch.connection-liveness.test.ts b/src/gateway/server/ws-connection/authenticated-request-dispatch.connection-liveness.test.ts index 30060f31dca4..f77b53578c52 100644 --- a/src/gateway/server/ws-connection/authenticated-request-dispatch.connection-liveness.test.ts +++ b/src/gateway/server/ws-connection/authenticated-request-dispatch.connection-liveness.test.ts @@ -41,7 +41,7 @@ function createClient(): GatewayWsClient { }; } -describe("authenticated request connection liveness", () => { +describe.sequential("authenticated request connection liveness", () => { beforeEach(() => { runtime.beforeHandler.mockReset(); });