fix: restart-recovered turns no longer show fatal errors (#126087)

* fix: let restart recovery own terminal settlement

Once durable restart recovery is armed, propagate the canonical restart abort before terminal payload mapping so the recovered continuation owns the visible outcome.

Fixes #126086

* fix(gateway): expose lifecycle context resolver

* fix(gateway): keep lifecycle resolver type acyclic

Own the live Gateway context resolver option beside lifecycle dispatch so current callers compile without closing the server type cycle.

* test(gateway): serialize connection liveness cases

The table cases share one hoisted handler mock; explicit sequential execution prevents cross-case 0/2 call interference in the concurrent CI shard.

* fix: preserve restart recovery claim through settlement

Mark the reply operation restart-aborted before propagating the canonical restart error so terminal cleanup cannot retire the durable recovery claim.
This commit is contained in:
Peter Steinberger
2026-08-18 18:54:23 -07:00
committed by GitHub
parent 8674f0c1e9
commit f335532ded
7 changed files with 115 additions and 6 deletions
@@ -441,6 +441,67 @@ describe("executeAgentTurn: terminal failures", () => {
).toBe(true);
});
it.each([
{
label: "settled result",
result: {
payloads: [{ text: "completed before the restart marker was observed" }],
meta: {},
},
},
{
label: "client-close error result",
result: {
payloads: [
{
text: "Codex app-server stopped before confirming turn completion.",
isError: true,
},
],
meta: { error: { message: "codex app-server client closed before turn completed" } },
},
},
])("hands an armed restart recovery owner the $label", async ({ label, result }) => {
const runId = `armed-restart-${label.replaceAll(" ", "-")}`;
const { replyOperation, failMock } = createMockReplyOperation();
let operationResult: typeof replyOperation.result = null;
const abortForRestart = vi.fn(() => {
operationResult = { kind: "aborted", code: "aborted_for_restart" };
return true;
});
const complete = vi.fn(() => {
operationResult ??= { kind: "completed" };
});
const restartReplyOperation = {
...replyOperation,
get result() {
return operationResult;
},
abortForRestart,
complete,
} satisfies typeof replyOperation;
state.runEmbeddedAgentMock.mockResolvedValueOnce(result);
const { executeAgentTurn } = await import("./agent-runner-execution.js");
const execution = await executeAgentTurn({
...createMinimalRunAgentTurnParams({ replyOperation: restartReplyOperation }),
opts: { runId } as GetReplyOptions,
isRestartRecoveryArmed: () => true,
});
expect(execution).toEqual({
runId,
outcome: { kind: "aborted", reason: "restart" },
});
expect(abortForRestart).toHaveBeenCalledOnce();
expect(complete).not.toHaveBeenCalled();
expect(restartReplyOperation.result).toEqual({
kind: "aborted",
code: "aborted_for_restart",
});
expect(failMock).not.toHaveBeenCalled();
});
it("uses compact generic copy for raw external chat errors when verbose is off", async () => {
const agentEvents = await import("../../infra/agent-events.js");
const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent);
@@ -375,6 +375,9 @@ async function executeAgentTurnInternalWithRetryState(
terminalRunFailed = cycle.terminalRunFailed;
break;
} catch (err) {
if (isAgentRunRestartAbortReason(err)) {
throw err;
}
if (err instanceof LiveSessionModelSwitchError) {
liveModelSwitchRetries += 1;
}
@@ -45,6 +45,9 @@ export async function settleAgentFallbackCycle(params: {
? cycle.state.pendingLifecycleTerminal.backstop
: undefined;
cycle.state.pendingLifecycleTerminal = undefined;
if (turn.isRestartRecoveryArmed?.()) {
turn.replyOperation?.abortForRestart();
}
if (isReplyOperationRestartAbort(turn.replyOperation)) {
settledLifecycleTerminal?.emit("end", runResult);
throw isAgentRunRestartAbortReason(cycle.runAbortSignal?.reason)
@@ -93,6 +93,46 @@ describe("createReplyRestartRecoveryClaimController", () => {
},
);
it("preserves lifecycle ownership when cleanup observes a restart abort", async () => {
const root = tempDirs.make("openclaw-reply-claim-restart-abort-");
const storePath = path.join(root, "sessions.json");
const sessionKey = "agent:main:main";
const sessionId = "session";
let restartAborted = false;
let entry: InternalSessionEntry = {
abortedLastRun: false,
lifecycleRunId: "recovery-run",
restartRecoveryDeliveryRunId: "recovery-run",
sessionId,
startedAt: 1,
status: "running",
updatedAt: 1,
};
await replaceSessionEntry({ storePath, sessionKey }, entry);
const controller = createReplyRestartRecoveryClaimController({
admissionRunId: "recovery-run",
getEntry: () => entry,
getSessionId: () => sessionId,
isRestartAbort: () => restartAborted,
resolveDeliveryContext: () => undefined,
sessionKey,
setEntry: (next) => {
entry = next;
},
storePath,
});
await expect(controller.admitUserTurn()).resolves.toBe("admitted");
restartAborted = true;
await controller.clear();
expect(loadSessionEntry({ storePath, sessionKey })).toMatchObject({
lifecycleRunId: "recovery-run",
restartRecoveryDeliveryRunId: "recovery-run",
status: "running",
});
});
it("retargets durable user-turn admission to the prepared reply session", async () => {
const root = tempDirs.make("openclaw-reply-admission-");
const storePath = path.join(root, "sessions.json");
@@ -21,10 +21,6 @@ export type GatewayInstanceAgentDispatchOptions = {
syntheticScopes?: string[];
};
export type GatewayLifecycleAgentDispatchOptions = GatewayInstanceAgentDispatchOptions & {
timeoutMs?: number;
};
export type GatewayApprovalEventPublisher = {
publishRequested: (kind: ChannelApprovalKind, request: unknown) => number;
publishResolved: (kind: ChannelApprovalKind, resolved: unknown) => void;
@@ -1,8 +1,14 @@
import type {
GatewayLifecycleAgentDispatchOptions,
GatewayInstanceAgentDispatchOptions,
GatewayRecoveryRuntime,
} from "./server-instance-runtime.types.js";
import type { AgentRunRequest } from "./server-methods/agent-request-types.js";
import type { GatewayContextResolver } from "./server-methods/shared-types.js";
type GatewayLifecycleAgentDispatchOptions = GatewayInstanceAgentDispatchOptions & {
resolveGatewayContext?: GatewayContextResolver;
timeoutMs?: number;
};
type ActiveGatewayRecoveryRuntime = {
owner: symbol;
@@ -41,7 +41,7 @@ function createClient(): GatewayWsClient {
};
}
describe("authenticated request connection liveness", () => {
describe.sequential("authenticated request connection liveness", () => {
beforeEach(() => {
runtime.beforeHandler.mockReset();
});