mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 11:25:50 -06:00
fix: restart-recovered turns no longer show fatal errors (#126087)
* fix: let restart recovery own terminal settlement Once durable restart recovery is armed, propagate the canonical restart abort before terminal payload mapping so the recovered continuation owns the visible outcome. Fixes #126086 * fix(gateway): expose lifecycle context resolver * fix(gateway): keep lifecycle resolver type acyclic Own the live Gateway context resolver option beside lifecycle dispatch so current callers compile without closing the server type cycle. * test(gateway): serialize connection liveness cases The table cases share one hoisted handler mock; explicit sequential execution prevents cross-case 0/2 call interference in the concurrent CI shard. * fix: preserve restart recovery claim through settlement Mark the reply operation restart-aborted before propagating the canonical restart error so terminal cleanup cannot retire the durable recovery claim.
This commit is contained in:
committed by
GitHub
parent
8674f0c1e9
commit
f335532ded
@@ -441,6 +441,67 @@ describe("executeAgentTurn: terminal failures", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "settled result",
|
||||
result: {
|
||||
payloads: [{ text: "completed before the restart marker was observed" }],
|
||||
meta: {},
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "client-close error result",
|
||||
result: {
|
||||
payloads: [
|
||||
{
|
||||
text: "Codex app-server stopped before confirming turn completion.",
|
||||
isError: true,
|
||||
},
|
||||
],
|
||||
meta: { error: { message: "codex app-server client closed before turn completed" } },
|
||||
},
|
||||
},
|
||||
])("hands an armed restart recovery owner the $label", async ({ label, result }) => {
|
||||
const runId = `armed-restart-${label.replaceAll(" ", "-")}`;
|
||||
const { replyOperation, failMock } = createMockReplyOperation();
|
||||
let operationResult: typeof replyOperation.result = null;
|
||||
const abortForRestart = vi.fn(() => {
|
||||
operationResult = { kind: "aborted", code: "aborted_for_restart" };
|
||||
return true;
|
||||
});
|
||||
const complete = vi.fn(() => {
|
||||
operationResult ??= { kind: "completed" };
|
||||
});
|
||||
const restartReplyOperation = {
|
||||
...replyOperation,
|
||||
get result() {
|
||||
return operationResult;
|
||||
},
|
||||
abortForRestart,
|
||||
complete,
|
||||
} satisfies typeof replyOperation;
|
||||
state.runEmbeddedAgentMock.mockResolvedValueOnce(result);
|
||||
const { executeAgentTurn } = await import("./agent-runner-execution.js");
|
||||
|
||||
const execution = await executeAgentTurn({
|
||||
...createMinimalRunAgentTurnParams({ replyOperation: restartReplyOperation }),
|
||||
opts: { runId } as GetReplyOptions,
|
||||
isRestartRecoveryArmed: () => true,
|
||||
});
|
||||
|
||||
expect(execution).toEqual({
|
||||
runId,
|
||||
outcome: { kind: "aborted", reason: "restart" },
|
||||
});
|
||||
expect(abortForRestart).toHaveBeenCalledOnce();
|
||||
expect(complete).not.toHaveBeenCalled();
|
||||
expect(restartReplyOperation.result).toEqual({
|
||||
kind: "aborted",
|
||||
code: "aborted_for_restart",
|
||||
});
|
||||
expect(failMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses compact generic copy for raw external chat errors when verbose is off", async () => {
|
||||
const agentEvents = await import("../../infra/agent-events.js");
|
||||
const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent);
|
||||
|
||||
@@ -375,6 +375,9 @@ async function executeAgentTurnInternalWithRetryState(
|
||||
terminalRunFailed = cycle.terminalRunFailed;
|
||||
break;
|
||||
} catch (err) {
|
||||
if (isAgentRunRestartAbortReason(err)) {
|
||||
throw err;
|
||||
}
|
||||
if (err instanceof LiveSessionModelSwitchError) {
|
||||
liveModelSwitchRetries += 1;
|
||||
}
|
||||
|
||||
@@ -45,6 +45,9 @@ export async function settleAgentFallbackCycle(params: {
|
||||
? cycle.state.pendingLifecycleTerminal.backstop
|
||||
: undefined;
|
||||
cycle.state.pendingLifecycleTerminal = undefined;
|
||||
if (turn.isRestartRecoveryArmed?.()) {
|
||||
turn.replyOperation?.abortForRestart();
|
||||
}
|
||||
if (isReplyOperationRestartAbort(turn.replyOperation)) {
|
||||
settledLifecycleTerminal?.emit("end", runResult);
|
||||
throw isAgentRunRestartAbortReason(cycle.runAbortSignal?.reason)
|
||||
|
||||
@@ -93,6 +93,46 @@ describe("createReplyRestartRecoveryClaimController", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves lifecycle ownership when cleanup observes a restart abort", async () => {
|
||||
const root = tempDirs.make("openclaw-reply-claim-restart-abort-");
|
||||
const storePath = path.join(root, "sessions.json");
|
||||
const sessionKey = "agent:main:main";
|
||||
const sessionId = "session";
|
||||
let restartAborted = false;
|
||||
let entry: InternalSessionEntry = {
|
||||
abortedLastRun: false,
|
||||
lifecycleRunId: "recovery-run",
|
||||
restartRecoveryDeliveryRunId: "recovery-run",
|
||||
sessionId,
|
||||
startedAt: 1,
|
||||
status: "running",
|
||||
updatedAt: 1,
|
||||
};
|
||||
await replaceSessionEntry({ storePath, sessionKey }, entry);
|
||||
const controller = createReplyRestartRecoveryClaimController({
|
||||
admissionRunId: "recovery-run",
|
||||
getEntry: () => entry,
|
||||
getSessionId: () => sessionId,
|
||||
isRestartAbort: () => restartAborted,
|
||||
resolveDeliveryContext: () => undefined,
|
||||
sessionKey,
|
||||
setEntry: (next) => {
|
||||
entry = next;
|
||||
},
|
||||
storePath,
|
||||
});
|
||||
|
||||
await expect(controller.admitUserTurn()).resolves.toBe("admitted");
|
||||
restartAborted = true;
|
||||
await controller.clear();
|
||||
|
||||
expect(loadSessionEntry({ storePath, sessionKey })).toMatchObject({
|
||||
lifecycleRunId: "recovery-run",
|
||||
restartRecoveryDeliveryRunId: "recovery-run",
|
||||
status: "running",
|
||||
});
|
||||
});
|
||||
|
||||
it("retargets durable user-turn admission to the prepared reply session", async () => {
|
||||
const root = tempDirs.make("openclaw-reply-admission-");
|
||||
const storePath = path.join(root, "sessions.json");
|
||||
|
||||
@@ -21,10 +21,6 @@ export type GatewayInstanceAgentDispatchOptions = {
|
||||
syntheticScopes?: string[];
|
||||
};
|
||||
|
||||
export type GatewayLifecycleAgentDispatchOptions = GatewayInstanceAgentDispatchOptions & {
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
export type GatewayApprovalEventPublisher = {
|
||||
publishRequested: (kind: ChannelApprovalKind, request: unknown) => number;
|
||||
publishResolved: (kind: ChannelApprovalKind, resolved: unknown) => void;
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
import type {
|
||||
GatewayLifecycleAgentDispatchOptions,
|
||||
GatewayInstanceAgentDispatchOptions,
|
||||
GatewayRecoveryRuntime,
|
||||
} from "./server-instance-runtime.types.js";
|
||||
import type { AgentRunRequest } from "./server-methods/agent-request-types.js";
|
||||
import type { GatewayContextResolver } from "./server-methods/shared-types.js";
|
||||
|
||||
type GatewayLifecycleAgentDispatchOptions = GatewayInstanceAgentDispatchOptions & {
|
||||
resolveGatewayContext?: GatewayContextResolver;
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
type ActiveGatewayRecoveryRuntime = {
|
||||
owner: symbol;
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ function createClient(): GatewayWsClient {
|
||||
};
|
||||
}
|
||||
|
||||
describe("authenticated request connection liveness", () => {
|
||||
describe.sequential("authenticated request connection liveness", () => {
|
||||
beforeEach(() => {
|
||||
runtime.beforeHandler.mockReset();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user