fix(gateway): finish successful sessions after timestamp races (#119752)

* fix(gateway): correlate session lifecycle by run

* test(qa): expect settled handoffs to survive restart quietly

* test(agents): split embedded run persistence coverage
This commit is contained in:
Patrick Erichsen
2026-08-05 17:24:12 -07:00
committed by GitHub
parent a20746e3ab
commit 678a3c18a9
39 changed files with 472 additions and 120 deletions
@@ -170,6 +170,8 @@ describe("qa scenario catalog channel contracts", () => {
expect(flow).toContain("task.deliveryStatus === 'delivered'");
expect(flow).toContain("readSettledTerminalTask('restart')");
expect(flow).toContain("readSettledTerminalTask('empty')");
expect(flow).toContain("postRestartUnexpectedPayloads.length === 0");
expect(flow).not.toContain("interrupted by a gateway restart");
expect(flow).toContain("verdicts.length === 5");
expect(flow).not.toContain('"call":"sleep"');
});
@@ -24,7 +24,7 @@ scenario:
- Visible completion output reaches the originating QA DM exactly once.
- Exact NO_REPLY completion output produces no channel delivery.
- Genuinely empty completion output is represented intentionally once.
- Gateway restart does not replay prior terminal payloads, represents the interrupted handoff explicitly, and a post-restart completion is delivered exactly once.
- Gateway restart does not replay prior terminal payloads or synthesize interruption notices for already-settled handoffs, and a post-restart completion is delivered exactly once.
- Direct fallback strips protected internal metadata before one channel delivery.
docsRefs:
- docs/tools/subagents.md
@@ -179,29 +179,6 @@ flow:
args:
- ref: env
- 180000
- call: waitForCondition
args:
- lambda:
expr: "state.getSnapshot().messages.find((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id) && !preRestartOutbound.some((before) => before.id === message.id) && String(message.text ?? '').includes('interrupted by a gateway restart'))"
- 60000
- 250
- set: postRestartOutbound
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id)).map((message) => ({ id: message.id, conversationId: message.conversation.id, text: String(message.text ?? '') }))"
- set: postRestartTerminalPayloads
value:
expr: "postRestartOutbound.filter((message) => verdicts.some((verdict) => verdict.capturedTerminalPayloads.includes(message.text)))"
- set: restartInterruptionPayloads
value:
expr: "postRestartOutbound.filter((message) => !preRestartOutbound.some((before) => before.id === message.id))"
- assert:
expr: "JSON.stringify(postRestartTerminalPayloads) === JSON.stringify(preRestartTerminalPayloads)"
message:
expr: "`Gateway restart replayed or lost a prior terminal payload: before=${JSON.stringify(preRestartTerminalPayloads)} after=${JSON.stringify(postRestartTerminalPayloads)}`"
- assert:
expr: "restartInterruptionPayloads.length === 1 && restartInterruptionPayloads.every((message) => message.text.includes('interrupted by a gateway restart'))"
message:
expr: "`Gateway restart did not represent the interrupted completion handoff exactly once: ${JSON.stringify(restartInterruptionPayloads)}`"
- set: restartStartIndex
value:
expr: state.getSnapshot().messages.length
@@ -241,9 +218,28 @@ flow:
expr: "restartTask.title === 'qa-terminal-restart' && restartTask.status === 'completed' && restartTask.deliveryStatus === 'delivered'"
message:
expr: "`restart completion task lifecycle did not settle authoritatively; task=${JSON.stringify(restartTask)}`"
# Use a new durably delivered task as the observation boundary. By then,
# restart recovery must not have replayed or invented output for settled tasks.
- set: postRestartOutbound
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id)).map((message) => ({ id: message.id, conversationId: message.conversation.id, text: String(message.text ?? '') }))"
- set: postRestartTerminalPayloads
value:
expr: "postRestartOutbound.filter((message) => verdicts.some((verdict) => verdict.capturedTerminalPayloads.includes(message.text)))"
- set: postRestartUnexpectedPayloads
value:
expr: "postRestartOutbound.filter((message) => !preRestartOutbound.some((before) => before.id === message.id))"
- assert:
expr: "JSON.stringify(postRestartTerminalPayloads) === JSON.stringify(preRestartTerminalPayloads)"
message:
expr: "`Gateway restart replayed or lost a prior terminal payload: before=${JSON.stringify(preRestartTerminalPayloads)} after=${JSON.stringify(postRestartTerminalPayloads)}`"
- assert:
expr: "postRestartUnexpectedPayloads.length === 0"
message:
expr: "`Gateway restart synthesized output for already-settled handoffs: ${JSON.stringify(postRestartUnexpectedPayloads)}`"
- set: appendRestartVerdict
value:
expr: "verdicts.push({ case: 'restart', conversationId: restartConversationId, taskId: restartTask.taskId, taskDeliveryStatus: restartTask.deliveryStatus, inputDisposition: 'visible', restart: true, fallback: true, preRestartTerminalMessageCount: preRestartTerminalPayloads.length, postRestartTerminalPayloadCount: postRestartTerminalPayloads.length, priorTerminalPayloadReplayCount: postRestartTerminalPayloads.length - preRestartTerminalPayloads.length, interruptedHandoffRepresentationCount: restartInterruptionPayloads.length, interruptedHandoffPayloads: restartInterruptionPayloads.map((message) => message.text), expectedTerminalSendCount: 1, actualTerminalSendCount: restartMatches.length, capturedTerminalPayloads: restartMatches.map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })"
expr: "verdicts.push({ case: 'restart', conversationId: restartConversationId, taskId: restartTask.taskId, taskDeliveryStatus: restartTask.deliveryStatus, inputDisposition: 'visible', restart: true, fallback: true, preRestartTerminalMessageCount: preRestartTerminalPayloads.length, postRestartTerminalPayloadCount: postRestartTerminalPayloads.length, priorTerminalPayloadReplayCount: postRestartTerminalPayloads.length - preRestartTerminalPayloads.length, unexpectedPostRestartPayloadCount: postRestartUnexpectedPayloads.length, unexpectedPostRestartPayloads: postRestartUnexpectedPayloads.map((message) => message.text), expectedTerminalSendCount: 1, actualTerminalSendCount: restartMatches.length, capturedTerminalPayloads: restartMatches.map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })"
- set: emptyStartIndex
value:
expr: state.getSnapshot().messages.length
+2 -1
View File
@@ -27,7 +27,7 @@ import {
import { resolveChannelResetConfig, resolveSessionResetType } from "../../config/sessions/reset.js";
import { listSessionEntries } from "../../config/sessions/session-accessor.js";
import { resolveSessionKey } from "../../config/sessions/session-key.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import {
classifySessionKeyShape,
@@ -67,6 +67,7 @@ export function clearRotatedSessionMetadata(entry: SessionEntry): SessionEntry {
...entry,
sessionFile: undefined,
status: undefined,
lifecycleRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -0,0 +1,42 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
import type { InternalSessionEntry } from "../../config/sessions/types.js";
import { testing } from "./runs.test-support.js";
describe("embedded-agent runner persistence", () => {
afterEach(() => {
testing.resetActiveEmbeddedRuns();
});
it("clears lifecycle ownership when a forced run clear persists killed state", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-force-clear-"));
const storePath = path.join(root, "sessions.json");
const sessionKey = "agent:main:main";
const entry: InternalSessionEntry = {
lifecycleRunId: "stuck-run",
sessionId: "session-stuck",
startedAt: 10,
status: "running",
updatedAt: 20,
};
try {
await replaceSessionEntry({ storePath, sessionKey }, entry);
await testing.persistForceClearedEmbeddedRunTerminalState({
sessionId: entry.sessionId,
sessionKey,
startedAt: entry.startedAt,
storePath,
updatedAt: entry.updatedAt,
});
const persisted = loadSessionEntry({ storePath, sessionKey }) as InternalSessionEntry;
expect(persisted.status).toBe("killed");
expect(persisted.lifecycleRunId).toBeUndefined();
} finally {
await fs.rm(root, { recursive: true, force: true });
}
});
});
@@ -1,6 +1,13 @@
import "./runs.js";
type EmbeddedRunsTestApi = {
persistForceClearedEmbeddedRunTerminalState(params: {
sessionId: string;
sessionKey: string;
startedAt?: number;
storePath: string;
updatedAt: number;
}): Promise<void>;
resetActiveEmbeddedRuns(): void;
};
+5 -1
View File
@@ -25,6 +25,7 @@ import {
import { getRuntimeConfig } from "../../config/io.js";
import { resolveStorePath } from "../../config/sessions/paths.js";
import { loadSessionEntry, updateSessionEntry } from "../../config/sessions/session-accessor.js";
import type { InternalSessionEntry } from "../../config/sessions/types.js";
import {
getAgentEventLifecycleGeneration,
isAgentEventLifecycleGenerationCurrent,
@@ -958,7 +959,8 @@ async function persistForceClearedEmbeddedRunTerminalState(params: {
try {
await updateSessionEntry(
{ sessionKey: params.sessionKey, storePath: params.storePath },
(entry) => {
(storedEntry) => {
const entry = storedEntry as InternalSessionEntry;
// A replacement can reuse the session id; bind this patch to both owners' exact snapshot.
if (
ACTIVE_EMBEDDED_RUNS.has(params.sessionId) ||
@@ -976,6 +978,7 @@ async function persistForceClearedEmbeddedRunTerminalState(params: {
return {
status: "killed",
abortedLastRun: true,
lifecycleRunId: undefined,
endedAt,
updatedAt: endedAt,
};
@@ -1130,6 +1133,7 @@ function forceClearEmbeddedAgentRun(
}
const testing = {
persistForceClearedEmbeddedRunTerminalState,
resetActiveEmbeddedRuns() {
for (const waiters of EMBEDDED_RUN_WAITERS.values()) {
for (const waiter of waiters) {
@@ -130,6 +130,7 @@ describe("main session recovery state", () => {
it("marks without charging and replaces an older lifecycle owner for the same run", () => {
const entry = interruptedEntry({
lifecycleRunId: "dead-run",
restartRecoveryRuns: [
{ runId: "older-run", lifecycleGeneration: "generation-old" },
{ runId: "shared-run", lifecycleGeneration: "generation-1" },
@@ -161,6 +162,7 @@ describe("main session recovery state", () => {
{ runId: "older-run", lifecycleGeneration: "generation-old" },
{ runId: "shared-run", lifecycleGeneration: "generation-2" },
]);
expect(entry.lifecycleRunId).toBeUndefined();
});
it("rejects foreground work after the automatic recovery budget is exhausted", () => {
@@ -343,6 +345,7 @@ describe("main session recovery state", () => {
},
});
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
expect(entry.lifecycleRunId).toBe("recovery-1");
expect(
transitionMainSessionRecovery(entry, {
@@ -358,6 +361,7 @@ describe("main session recovery state", () => {
expect(entry.abortedLastRun).toBe(true);
expect(entry.restartRecoveryDeliveryRunId).toBeUndefined();
expect(entry.restartRecoveryDeliverySourceRunId).toBe("source-1");
expect(entry.lifecycleRunId).toBeUndefined();
});
it("rejects a reservation created by an older lifecycle generation", () => {
@@ -565,6 +569,7 @@ describe("main session recovery state", () => {
it("tombstones an exhausted cycle and exposes only the Doctor repair action", () => {
const entry = interruptedEntry({
lifecycleRunId: "exhausted-run",
mainRestartRecovery: recoveryState({
chargedAttempts: 3,
}),
@@ -584,6 +589,7 @@ describe("main session recovery state", () => {
}),
).toEqual({ kind: "tombstoned" });
expect(entry.mainRestartRecovery?.tombstone?.reason).toBe(view.reason);
expect(entry.lifecycleRunId).toBeUndefined();
expect(observe(entry, "generation-1")).toEqual({ status: "tombstoned" });
entry.abortedLastRun = true;
@@ -266,6 +266,7 @@ export function transitionMainSessionRecovery(
entry.mainRestartRecovery = createCycle(command.cycleId);
}
entry.status = "running";
entry.lifecycleRunId = undefined;
entry.abortedLastRun = true;
if (command.resetRuntime) {
entry.startedAt = undefined;
@@ -419,6 +420,7 @@ export function transitionMainSessionRecovery(
foregroundClaims: undefined,
});
entry.abortedLastRun = false;
entry.lifecycleRunId = command.runId;
recordLifecycleFence(entry, {
runId: command.runId,
lifecycleGeneration: command.lifecycleGeneration,
@@ -449,6 +451,7 @@ export function transitionMainSessionRecovery(
return { kind: "rejected", reason: "stale_reservation" };
}
entry.status = "running";
entry.lifecycleRunId = undefined;
entry.abortedLastRun = true;
entry.startedAt = undefined;
entry.endedAt = undefined;
@@ -599,6 +602,7 @@ export function transitionMainSessionRecovery(
});
entry.abortedLastRun = false;
entry.status = "failed";
entry.lifecycleRunId = undefined;
entry.endedAt = command.now;
entry.runtimeMs = Math.max(0, command.now - (entry.startedAt ?? command.now));
entry.updatedAt = command.now;
@@ -611,6 +615,7 @@ export function transitionMainSessionRecovery(
}
const noticeEntry = structuredClone(entry);
entry.status = "failed";
entry.lifecycleRunId = undefined;
entry.abortedLastRun = true;
entry.endedAt = command.now;
entry.updatedAt = command.now;
@@ -66,6 +66,7 @@ export async function reconcileInterruptedCompletionReport(params: {
...buildRestartRecoveryClaimCleanupPatch({ entry, recordTerminalSource: false }),
...buildMainSessionRecoveryClearPatch(entry),
status: "killed",
lifecycleRunId: undefined,
abortedLastRun: false,
endedAt,
lastRunError: undefined,
@@ -315,6 +316,7 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
terminalSourceRunId: expectedRecoverySourceRunId,
}),
abortedLastRun: false,
lifecycleRunId: undefined,
endedAt,
pendingFinalDelivery: undefined,
restartRecoveryForceSafeTools: undefined,
@@ -81,6 +81,7 @@ export async function tombstoneMainRestartRecoveryWithNotice(params: {
sessionLifecyclePatch: {
abortedLastRun: false,
endedAt: now,
lifecycleRunId: undefined,
mainRestartRecovery: {
...recoveryState,
revision: recoveryState.revision + 1,
@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { SessionEntry } from "../../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
import { formatSqliteSessionFileMarker } from "../../config/sessions/legacy-sqlite-marker.js";
import {
appendTranscriptMessage,
@@ -78,6 +78,7 @@ describe("resetReplyRunSession", () => {
sessionId: "session",
updatedAt: 1,
sessionFile: path.join(rootDir, "session.jsonl"),
lifecycleRunId: "run-before-reset",
agentHarnessId: "codex",
claudeCliSessionId: "native-before-boundary",
modelProvider: "qwencode",
@@ -150,6 +151,7 @@ describe("resetReplyRunSession", () => {
expect(isNewSession).toBe(true);
expect(activeSessionEntry?.sessionId).toBe("session");
expect(activeSessionEntry?.lifecycleRevision).toBe("00000000-0000-0000-0000-000000000123");
expect(activeSessionEntry?.lifecycleRunId).toBeUndefined();
expect(followupRun.run.sessionId).toBe(activeSessionEntry?.sessionId);
expect(activeSessionEntry?.modelProvider).toBeUndefined();
expect(activeSessionEntry?.agentHarnessId).toBeUndefined();
@@ -3,7 +3,7 @@ import { clearAllCliSessions } from "../../agents/cli-session.js";
import { resetRegisteredAgentHarnessSessions } from "../../agents/harness/registry.js";
// Handles session reset requests produced during agent runner execution.
import { transitionMainSessionRecovery } from "../../agents/main-session-recovery-state.js";
import type { SessionEntry } from "../../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
import { persistSessionResetLifecycle } from "../../config/sessions/session-accessor.js";
import { generateSecureUuid } from "../../infra/secure-random.js";
import { defaultRuntime } from "../../runtime.js";
@@ -79,6 +79,7 @@ export async function resetReplyRunSession(params: {
lastInteractionAt: now,
systemSent: false,
abortedLastRun: false,
lifecycleRunId: undefined,
modelProvider: undefined,
model: undefined,
inputTokens: undefined,
@@ -3,7 +3,7 @@ import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
import type { ReplyPayload } from "../reply-payload.js";
import {
capturePendingFinalDeliveryIdentity,
@@ -29,10 +29,11 @@ describe("pending final delivery restart proof", () => {
async function writePendingFinal(
beforeAgentReplyState: "continue" | "handled-reply",
): Promise<void> {
await replaceSessionEntry({ storePath, sessionKey }, {
const entry: SessionEntry = {
sessionId: "session",
status: "running",
startedAt: 10,
lifecycleRunId: "active-run",
updatedAt: Date.now(),
pendingFinalDelivery: {
kind: "replayable",
@@ -43,7 +44,8 @@ describe("pending final delivery restart proof", () => {
restartRecoveryBeforeAgentReplyState: beforeAgentReplyState,
restartRecoveryForceSafeTools: beforeAgentReplyState === "handled-reply" ? true : undefined,
restartRecoverySourceIngress: "channel",
} satisfies SessionEntry);
};
await replaceSessionEntry({ storePath, sessionKey }, entry);
}
it.each(["continue", "handled-reply"] as const)(
@@ -58,12 +60,15 @@ describe("pending final delivery restart proof", () => {
await clearPendingFinalDeliveryAfterSuccess({ identity, sessionKey, storePath });
const entry = loadSessionEntry({ sessionKey, storePath });
const entry = loadSessionEntry({ sessionKey, storePath }) as SessionEntry | undefined;
expect(entry?.pendingFinalDelivery).toBeUndefined();
expect(entry?.restartRecoveryBeforeAgentReplyState).toBeUndefined();
expect(entry?.restartRecoveryForceSafeTools).toBeUndefined();
expect(entry?.restartRecoverySourceIngress).toBeUndefined();
expect(entry?.status).toBe(beforeAgentReplyState === "handled-reply" ? "done" : "running");
expect(entry?.lifecycleRunId).toBe(
beforeAgentReplyState === "handled-reply" ? undefined : "active-run",
);
if (beforeAgentReplyState === "handled-reply") {
expect(entry?.endedAt).toBeTypeOf("number");
expect(entry?.runtimeMs).toBeGreaterThanOrEqual(0);
@@ -72,22 +77,21 @@ describe("pending final delivery restart proof", () => {
);
it("finalizes a media-only hook turn after its exact transport intent succeeds", async () => {
await replaceSessionEntry(
{ storePath, sessionKey },
{
sessionId: "session",
status: "running",
startedAt: 10,
updatedAt: Date.now(),
pendingFinalDelivery: {
kind: "transport-only",
createdAt: Date.now(),
intentId: "intent-media",
},
restartRecoveryBeforeAgentReplyState: "handled-unrecoverable",
restartRecoverySourceIngress: "channel",
const entry: SessionEntry = {
sessionId: "session",
status: "running",
startedAt: 10,
lifecycleRunId: "media-run",
updatedAt: Date.now(),
pendingFinalDelivery: {
kind: "transport-only",
createdAt: Date.now(),
intentId: "intent-media",
},
);
restartRecoveryBeforeAgentReplyState: "handled-unrecoverable",
restartRecoverySourceIngress: "channel",
};
await replaceSessionEntry({ storePath, sessionKey }, entry);
const identity = capturePendingFinalDeliveryIdentity({
intentId: "intent-media",
sessionKey,
@@ -100,6 +104,9 @@ describe("pending final delivery restart proof", () => {
status: "done",
abortedLastRun: false,
});
expect(
(loadSessionEntry({ sessionKey, storePath }) as SessionEntry | undefined)?.lifecycleRunId,
).toBeUndefined();
});
it("keeps normal-turn provenance when transport fails before delivery", async () => {
@@ -3,7 +3,7 @@ import {
loadSessionEntryReadOnly,
updateSessionEntry,
} from "../../config/sessions/session-accessor.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
import type { ReplyPayload } from "../reply-payload.js";
import { getReplyPayloadMetadata } from "../reply-payload.js";
import {
@@ -47,6 +47,7 @@ function buildPendingFinalDeliveryCleanupPatch(entry: SessionEntry): Partial<Ses
? {
abortedLastRun: false,
endedAt,
lifecycleRunId: undefined,
runtimeMs:
typeof entry.startedAt === "number"
? Math.max(0, endedAt - entry.startedAt)
@@ -38,6 +38,57 @@ function replaceSessionEntryFromIndependentConnection(params: {
}
describe("createReplyRestartRecoveryClaimController", () => {
it.each([
{ receiptState: undefined, expectedStatus: "done" },
{ receiptState: "terminal-pending" as const, expectedStatus: "failed" },
])(
"clears lifecycle ownership when claim cleanup settles $expectedStatus",
async ({ receiptState, expectedStatus }) => {
const root = tempDirs.make(`openclaw-reply-claim-${expectedStatus}-`);
const storePath = path.join(root, "sessions.json");
const sessionKey = "agent:main:main";
const sessionId = "session";
let entry: InternalSessionEntry = {
abortedLastRun: false,
lifecycleRunId: "recovery-run",
restartRecoveryBeforeAgentReplyState: "admitted",
restartRecoveryDeliveryRunId: "recovery-run",
sessionId,
startedAt: 1,
status: "running",
updatedAt: 1,
};
await replaceSessionEntry({ storePath, sessionKey }, entry);
const controller = createReplyRestartRecoveryClaimController({
admissionRunId: "recovery-run",
getEntry: () => entry,
getSessionId: () => sessionId,
isRestartAbort: () => false,
resolveDeliveryContext: () => undefined,
sessionKey,
setEntry: (next) => {
entry = next;
},
storePath,
});
await expect(controller.admitUserTurn()).resolves.toBe("admitted");
if (receiptState) {
entry = (await updateSessionEntry({ storePath, sessionKey }, () => ({
restartRecoveryDeliveryReceiptState: receiptState,
}))) as InternalSessionEntry;
} else {
await expect(controller.beginBeforeAgentReply()).resolves.toBe(true);
await controller.checkpointBeforeAgentReply({ state: "handled-silent" });
}
await controller.clear();
const persisted = loadSessionEntry({ storePath, sessionKey }) as InternalSessionEntry;
expect(persisted.status).toBe(expectedStatus);
expect(persisted.lifecycleRunId).toBeUndefined();
},
);
it("retargets durable user-turn admission to the prepared reply session", async () => {
const root = tempDirs.make("openclaw-reply-admission-");
const storePath = path.join(root, "sessions.json");
@@ -499,6 +499,7 @@ export function createReplyRestartRecoveryClaimController(params: {
}),
abortedLastRun: true,
endedAt,
lifecycleRunId: undefined,
pendingFinalDelivery: undefined,
runtimeMs:
typeof current.startedAt === "number"
@@ -532,6 +533,7 @@ export function createReplyRestartRecoveryClaimController(params: {
? {
abortedLastRun: false,
endedAt,
lifecycleRunId: undefined,
runtimeMs:
typeof current.startedAt === "number"
? Math.max(0, endedAt - current.startedAt)
+8 -1
View File
@@ -9,6 +9,7 @@ import {
replaceSessionEntry,
} from "../../config/sessions/session-accessor.js";
import { replaceSqliteTranscriptEvents } from "../../config/sessions/session-accessor.sqlite.js";
import type { InternalSessionEntry } from "../../config/sessions/types.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import {
forkSessionEntryFromParent,
@@ -165,10 +166,15 @@ describe("forkSessionEntryFromParent", () => {
],
);
const fallbackEntry: InternalSessionEntry = {
lifecycleRunId: "pre-fork-run",
sessionId: "",
updatedAt: 2,
};
const result = await forkSessionEntryFromParent({
agentId: "main",
config: { session: { store: configStorePath } } as OpenClawConfig,
fallbackEntry: { sessionId: "", updatedAt: 2 },
fallbackEntry,
parentSessionKey,
parentStoreKeys: [parentSessionKey],
sessionKey,
@@ -196,6 +202,7 @@ describe("forkSessionEntryFromParent", () => {
sessionId: result.fork.sessionId,
updatedAt: expect.any(Number),
});
expect((stored as InternalSessionEntry | undefined)?.lifecycleRunId).toBeUndefined();
expect(loadSessionEntry({ agentId: "main", sessionKey: staleSessionKey, storePath })).toBe(
undefined,
);
@@ -1,6 +1,6 @@
// Prepares parent-context fork metadata for guarded reply session initialization.
import { buildMainSessionRecoveryClearPatch } from "../../agents/main-session-recovery-clear.js";
import type { SessionEntry } from "../../config/sessions.js";
import type { InternalSessionEntry, SessionEntry } from "../../config/sessions.js";
import { forkSessionFromParent, resolveParentForkDecision } from "./session-fork.js";
export async function prepareReplySessionParentFork(params: {
@@ -54,10 +54,11 @@ export async function prepareReplySessionParentFork(params: {
);
// The fork replaces this thread's transcript identity; recovery state from
// the preseed row must not govern a later interruption of the fork.
return {
const forkedEntry: InternalSessionEntry = {
...params.sessionEntry,
...buildMainSessionRecoveryClearPatch(params.sessionEntry),
sessionId: fork.sessionId,
lifecycleRunId: undefined,
forkSource: {
sessionKey: params.parentSessionKey,
sessionId: parentEntry.sessionId,
@@ -66,4 +67,5 @@ export async function prepareReplySessionParentFork(params: {
totalTokens: undefined,
totalTokensFresh: false,
};
return forkedEntry;
}
@@ -66,7 +66,7 @@ import {
upsertSqliteSessionEntry,
} from "./session-accessor.sqlite.js";
import { setCanonicalSqliteSessionMainKey } from "./session-canonical-key.js";
import type { SessionCompactionCheckpoint, SessionEntry } from "./types.js";
import type { InternalSessionEntry, SessionCompactionCheckpoint, SessionEntry } from "./types.js";
// Keep accessor conformance independent of any real openclaw.json on the machine.
vi.mock("../config.js", async () => ({
@@ -2091,12 +2091,14 @@ describe("sqlite session normalization", () => {
message: { content: "post-two" },
},
]);
await upsertSqliteSessionEntry(sourceEntryScope, {
const sourceEntry: InternalSessionEntry = {
label: "Source",
lifecycleRunId: "source-run",
sessionId: "source-session",
updatedAt: 10,
compactionCheckpoints: [checkpoint],
});
};
await upsertSqliteSessionEntry(sourceEntryScope, sourceEntry);
const notify = vi.fn();
const unsubscribe = onSessionIdentityMutation(notify);
@@ -2134,6 +2136,7 @@ describe("sqlite session normalization", () => {
totalTokensFresh: true,
}),
);
expect((result.entry as InternalSessionEntry).lifecycleRunId).toBeUndefined();
await expect(loadSqliteTranscriptEvents(branchScope)).resolves.toEqual([
expect.objectContaining({ type: "session", id: result.entry.sessionId }),
expect.objectContaining({ id: "pre-msg", type: "message" }),
@@ -27,7 +27,7 @@ import {
readTranscriptIdentityByEventId,
} from "./session-accessor.sqlite-transcript-store.js";
import { createSessionTranscriptHeader } from "./transcript-header.js";
import type { SessionCompactionCheckpoint, SessionEntry } from "./types.js";
import type { InternalSessionEntry as SessionEntry, SessionCompactionCheckpoint } from "./types.js";
// Compaction checkpoint branch/restore owner.
@@ -428,6 +428,7 @@ function cloneSqliteCheckpointSessionEntry(params: {
updatedAt: Date.now(),
systemSent: false,
abortedLastRun: false,
lifecycleRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -24,6 +24,7 @@ import {
upsertSessionEntry,
} from "./session-accessor.js";
import { listSqliteSessionBranches } from "./session-accessor.sqlite.js";
import type { InternalSessionEntry } from "./types.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
const agentId = "main";
@@ -74,7 +75,7 @@ async function createSession(options: { activeLeafTarget?: string } = {}) {
const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir };
const sessionId = "message-cut-source";
const scope = { agentId, env, sessionId, sessionKey };
await upsertSessionEntry(scope, {
const entry: InternalSessionEntry = {
agentHarnessId: "embedded",
claudeCliSessionId: "claude-conversation",
cliSessionBindings: { "claude-cli": { sessionId: "claude-conversation" } },
@@ -89,12 +90,14 @@ async function createSession(options: { activeLeafTarget?: string } = {}) {
}),
forkSource: { sessionKey: "agent:main:root", sessionId: "root-session" },
lifecycleRevision: "source-lifecycle-revision",
lifecycleRunId: "source-run",
modelOverride: "gpt-5",
modelOverrideSource: "user",
providerOverride: "openai",
sessionId,
updatedAt: Date.now(),
});
};
await upsertSessionEntry(scope, entry);
for (const event of [
{ type: "session", id: sessionId, version: 3, timestamp: "2026-07-18T00:00:00.000Z" },
{
@@ -495,6 +498,7 @@ describe("SQLite session message cuts", () => {
).toEqual([result.entry.sessionId, "user-1", "assistant-1"]);
expect(loadSessionEntry(scope)?.sessionId).toBe(scope.sessionId);
expect(result.entry.lifecycleRevision).not.toBe("source-lifecycle-revision");
expect((result.entry as InternalSessionEntry).lifecycleRunId).toBeUndefined();
expect(result.entry.cliSessionBindings).toBeUndefined();
expect(deliveryContextFromSession(result.entry)).toBeUndefined();
expect(result.entry.parentSessionKey).toBe(canonicalSourceKey);
@@ -46,7 +46,7 @@ import {
selectSessionTranscriptTreePathNodes,
type SessionTranscriptTree,
} from "./transcript-tree.js";
import type { SessionEntry } from "./types.js";
import type { InternalSessionEntry as SessionEntry } from "./types.js";
type MessageCut = {
editorText?: string;
@@ -475,6 +475,7 @@ function cloneMessageCutSessionEntry(params: {
updatedAt: Date.now(),
systemSent: false,
abortedLastRun: false,
lifecycleRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -48,7 +48,7 @@ import {
} from "./session-accessor.sqlite-scope.js";
import { appendTranscriptEventsInTransaction } from "./session-accessor.sqlite-transcript-store.js";
import { preserveSqliteSameKeySessionRolloverLineage } from "./session-entry-lineage.js";
import type { SessionEntry } from "./types.js";
import type { InternalSessionEntry, SessionEntry } from "./types.js";
import { mergeSessionEntry, resolveFreshSessionTotalTokens } from "./types.js";
// Parent-session fork owner: decision, transcript copy, and child entry commit.
@@ -218,15 +218,17 @@ export async function forkSqliteSessionEntryFromParentTarget(
fork: fork.transcript,
parentEntry: cloneSessionEntry(freshParent),
});
const next = mergeSessionEntry(freshBase, {
const forkIdentityPatch: Partial<InternalSessionEntry> = {
...patch,
forkSource: {
sessionKey: parentTarget.canonicalKey,
sessionId: freshParent.sessionId,
},
forkedFromParent: true,
lifecycleRunId: undefined,
sessionId: fork.transcript.sessionId,
});
};
const next = mergeSessionEntry(freshBase, forkIdentityPatch);
previousIdentity = readSqliteSessionIdentitySnapshot(writeDatabase, sessionTarget.storeKeys);
writeSessionEntry(writeDatabase, sessionTarget.canonicalKey, next, {
previousEntry: freshBase,
@@ -28,6 +28,7 @@ export type SessionTranscriptTurnExpectedState = {
export type SessionTranscriptTurnLifecyclePatch = {
abortedLastRun?: boolean;
endedAt?: number;
lifecycleRunId?: SessionEntry["lifecycleRunId"];
pendingFinalDelivery?: SessionEntry["pendingFinalDelivery"];
mainRestartRecovery?: SessionEntry["mainRestartRecovery"];
restartRecoveryBeforeAgentReplyState?: SessionRestartRecoveryState["restartRecoveryBeforeAgentReplyState"];
+2 -1
View File
@@ -1,4 +1,4 @@
import type { SessionEntry } from "./types.js";
import type { InternalSessionEntry as SessionEntry } from "./types.js";
/** Returns true for terminal statuses that a later visible turn may recover in place. */
export function isRecoverableTerminalSessionStatus(
@@ -12,6 +12,7 @@ export function recoverTerminalSessionEntryForVisibleTurn(entry: SessionEntry):
return {
...entry,
status: undefined,
lifecycleRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
+2
View File
@@ -597,6 +597,8 @@ export interface SessionEntry extends SessionEntryCore {}
/** Internal durable fields excluded from public/plugin session projections. */
export type InternalSessionEntryCore = SessionEntryCore & {
/** Run that owns the current non-terminal Gateway lifecycle projection. */
lifecycleRunId?: string;
mainRestartRecovery?: MainRestartRecoveryState;
};
+57 -3
View File
@@ -24,6 +24,7 @@ import { subscribePluginSessionsChanged } from "../plugins/gateway-events.js";
const persistGatewaySessionLifecycleEventMock = vi.fn();
const logErrorMock = vi.fn();
const normalizeLiveAssistantBufferedTextMock = vi.hoisted(() => vi.fn());
const loadGatewaySessionRow = vi.hoisted(() => vi.fn());
vi.mock("./server-chat.persist-session-lifecycle.runtime.js", () => ({
persistGatewaySessionLifecycleEvent: (...args: unknown[]) =>
@@ -58,7 +59,7 @@ vi.mock("../infra/heartbeat-visibility.js", () => ({
}));
vi.mock("./server-chat.load-gateway-session-row.runtime.js", () => ({
loadGatewaySessionRow: vi.fn(),
loadGatewaySessionLifecycleSnapshot: vi.fn(),
}));
vi.mock("./session-utils.js", () => {
@@ -96,7 +97,7 @@ import {
resolveChatErrorKindFromError,
type AgentEventHandlerOptions,
} from "./server-chat.js";
import { loadGatewaySessionRow } from "./server-chat.load-gateway-session-row.runtime.js";
import { loadGatewaySessionLifecycleSnapshot } from "./server-chat.load-gateway-session-row.runtime.js";
import { loadSessionEntry } from "./session-utils.js";
function waitForFast<T>(
@@ -127,6 +128,13 @@ describe("agent event handler", () => {
legacyKey: undefined,
});
vi.mocked(loadGatewaySessionRow).mockReset().mockReturnValue(null);
vi.mocked(loadGatewaySessionLifecycleSnapshot)
.mockReset()
.mockImplementation((sessionKey, options) => ({
row: options
? loadGatewaySessionRow(sessionKey, options)
: loadGatewaySessionRow(sessionKey),
}));
persistGatewaySessionLifecycleEventMock.mockReset().mockResolvedValue(undefined);
logErrorMock.mockReset();
normalizeLiveAssistantBufferedTextMock.mockReset();
@@ -174,7 +182,7 @@ describe("agent event handler", () => {
toolEventRecipients,
sessionEventSubscribers,
sessionMessageSubscribers,
loadGatewaySessionRowForSnapshot: loadGatewaySessionRow,
loadGatewaySessionLifecycleSnapshotForEvent: loadGatewaySessionLifecycleSnapshot,
lifecycleErrorRetryGraceMs: params?.lifecycleErrorRetryGraceMs,
isChatSendRunActive: params?.isChatSendRunActive,
clearTrackedActiveRun: params?.clearTrackedActiveRun ?? clearTrackedActiveRun,
@@ -2212,6 +2220,52 @@ describe("agent event handler", () => {
}
});
it.each([
{ eventRunId: "run-current", expectedStartedAt: 1_900 },
{ eventRunId: "run-older", expectedStartedAt: 2_000 },
])(
"projects older lifecycle timestamps only for the owning run ($eventRunId)",
async ({ eventRunId, expectedStartedAt }) => {
vi.mocked(loadGatewaySessionLifecycleSnapshot).mockReturnValue({
lifecycleRunId: "run-current",
row: {
key: "session-owned",
kind: "direct",
sessionId: "session-id",
updatedAt: 2_000,
status: "running",
startedAt: 2_000,
},
});
const { broadcastToConnIds, sessionEventSubscribers, handler } = createHarness({
lifecycleErrorRetryGraceMs: 0,
});
sessionEventSubscribers.subscribe("conn-session");
emitAgentEvent(
handler,
eventRunId,
"lifecycle",
{ phase: "start", startedAt: 1_900 },
{ sessionKey: "session-owned", sessionId: "session-id", ts: 2_200 },
);
await waitForFast(() => {
expect(
broadcastToConnIds.mock.calls.filter(([event]) => event === "sessions.changed"),
).toHaveLength(1);
});
const payload = broadcastToConnIds.mock.calls.find(
([event]) => event === "sessions.changed",
)?.[1];
expectPayloadFields(payload, {
sessionKey: "session-owned",
status: "running",
startedAt: expectedStartedAt,
});
},
);
it("suppresses late interrupted pre-restart lifecycle events from live projections", () => {
mockSessionEntry(
{
@@ -1,3 +1,3 @@
// Runtime barrel for loading Gateway session rows from chat paths without
// pulling the rest of session-utils into static startup imports.
export { loadGatewaySessionRow } from "./session-utils.js";
export { loadGatewaySessionLifecycleSnapshot } from "./session-utils.js";
+12 -5
View File
@@ -52,7 +52,7 @@ import type {
SessionMessageSubscriberRegistry,
ToolEventRecipientRegistry,
} from "./server-chat-state.js";
import { loadGatewaySessionRow } from "./server-chat.load-gateway-session-row.runtime.js";
import { loadGatewaySessionLifecycleSnapshot } from "./server-chat.load-gateway-session-row.runtime.js";
import { persistGatewaySessionLifecycleEvent } from "./server-chat.persist-session-lifecycle.runtime.js";
import { hasSessionChangeReceivers } from "./session-change-receivers.js";
import {
@@ -320,7 +320,7 @@ export type AgentEventHandlerOptions = {
toolEventRecipients: ToolEventRecipientRegistry;
sessionEventSubscribers: SessionEventSubscriberRegistry;
sessionMessageSubscribers: SessionMessageSubscriberRegistry;
loadGatewaySessionRowForSnapshot?: typeof loadGatewaySessionRow;
loadGatewaySessionLifecycleSnapshotForEvent?: typeof loadGatewaySessionLifecycleSnapshot;
lifecycleErrorRetryGraceMs?: number;
isChatSendRunActive?: (runId: string) => boolean;
clearTrackedActiveRun?: (params: {
@@ -374,7 +374,7 @@ export function createAgentEventHandler({
toolEventRecipients,
sessionEventSubscribers,
sessionMessageSubscribers,
loadGatewaySessionRowForSnapshot = loadGatewaySessionRow,
loadGatewaySessionLifecycleSnapshotForEvent = loadGatewaySessionLifecycleSnapshot,
lifecycleErrorRetryGraceMs = AGENT_LIFECYCLE_ERROR_RETRY_GRACE_MS,
isChatSendRunActive = () => false,
clearTrackedActiveRun,
@@ -476,7 +476,7 @@ export function createAgentEventHandler({
}
let result: string | null = null;
try {
result = loadGatewaySessionRow(sessionKey)?.spawnedBy ?? null;
result = loadGatewaySessionLifecycleSnapshotForEvent(sessionKey).row?.spawnedBy ?? null;
} catch {
// result stays null
}
@@ -490,13 +490,20 @@ export function createAgentEventHandler({
agentId?: string,
includeActiveRunState = false,
) => {
const row = loadGatewaySessionRowForSnapshot(sessionKey, agentId ? { agentId } : undefined);
const snapshotOptions = agentId ? { agentId } : undefined;
const lifecycleSnapshot = loadGatewaySessionLifecycleSnapshotForEvent(
sessionKey,
snapshotOptions,
);
const { lifecycleRunId, row } = lifecycleSnapshot;
const omitUnscopedGlobalGoal = sessionKey === "global" && !agentId;
const lifecyclePatch =
evt &&
!isStaleLifecycleEventForSession({
owningSessionId: evt.sessionId,
currentSessionId: row?.sessionId,
eventRunId: evt.runId,
currentRunId: lifecycleRunId,
eventStartedAt: evt.data?.startedAt,
currentStartedAt: row?.startedAt,
})
@@ -1,5 +1,8 @@
import { describe, expect, it } from "vitest";
import { resolveSessionResetPolicy, type SessionEntry } from "../../config/sessions.js";
import {
resolveSessionResetPolicy,
type InternalSessionEntry as SessionEntry,
} from "../../config/sessions.js";
import { buildAgentSessionPatch } from "./agent-session-patch.js";
function buildPatch(touchInteraction: boolean, opts?: { requestLabel?: string; label?: string }) {
@@ -7,6 +10,8 @@ function buildPatch(touchInteraction: boolean, opts?: { requestLabel?: string; l
const entry: SessionEntry = {
sessionId: "session",
updatedAt: now,
lifecycleRunId: "completed-run",
status: "failed",
agentStatus: { note: "Need a password", attention: "key", expiresAt: now + 60_000 },
...(opts?.label ? { label: opts.label } : {}),
};
@@ -37,6 +42,8 @@ describe("agent session patch", () => {
const patch = buildPatch(true);
expect(Object.hasOwn(patch, "agentStatus")).toBe(true);
expect(patch.agentStatus).toBeUndefined();
expect(Object.hasOwn(patch, "lifecycleRunId")).toBe(true);
expect(patch.lifecycleRunId).toBeUndefined();
});
it("does not clear agent status for lifecycle-only patches", () => {
@@ -6,12 +6,12 @@ import {
evaluateSessionFreshness,
hasTerminalMainSessionTranscriptNewerThanRegistrySync,
resolveSessionLifecycleTimestamps,
type SessionEntry,
type SessionFreshness,
} from "../../config/sessions.js";
import { hasProviderOwnedSession } from "../../config/sessions/entry-freshness.js";
import { resolveSessionEntryAccessTarget } from "../../config/sessions/session-accessor.js";
import { isRecoverableTerminalSessionStatus } from "../../config/sessions/terminal-status.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import {
deliveryContextFromSession,
@@ -243,6 +243,7 @@ export function buildAgentSessionPatch(params: {
...(shouldClearRotatedState || shouldClearTerminalState
? {
status: undefined,
lifecycleRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -356,6 +356,7 @@ export function buildRestartSafeChatTranscriptState(params: {
restartRecoveryDeliveryReceiptState: undefined,
restartRecoveryDeliveryToolCallId: undefined,
status: "running",
lifecycleRunId: params.clientRunId,
startedAt: params.startedAt,
endedAt: undefined,
restartRecoveryDeliveryContext: undefined,
@@ -402,6 +403,7 @@ export async function terminalizeRestartSafeChatAdmission(params: {
terminalized = true;
return {
abortedLastRun: params.retryable ? false : params.status === "killed",
lifecycleRunId: undefined,
endedAt,
...(params.retryable
? {}
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import type { SessionEntry } from "../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import { buildForkedGatewaySessionEntry } from "./session-create-fork-entry.js";
describe("buildForkedGatewaySessionEntry", () => {
@@ -7,6 +7,7 @@ describe("buildForkedGatewaySessionEntry", () => {
const previous: SessionEntry = {
sessionId: "adopted-generation",
updatedAt: 1,
lifecycleRunId: "adopted-run",
forkSource: { sessionKey: "agent:main:original", sessionId: "original-generation" },
};
@@ -22,6 +23,7 @@ describe("buildForkedGatewaySessionEntry", () => {
previousSessionId: "adopted-generation",
forkSource: { sessionKey: "agent:main:original", sessionId: "original-generation" },
});
expect(forked.lifecycleRunId).toBeUndefined();
});
it("uses the requested ancestry for a genuinely new node", () => {
+2 -1
View File
@@ -1,5 +1,5 @@
import { buildMainSessionRecoveryClearPatch } from "../agents/main-session-recovery-clear.js";
import type { SessionEntry } from "../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
export function buildForkedGatewaySessionEntry(
entry: SessionEntry,
@@ -12,6 +12,7 @@ export function buildForkedGatewaySessionEntry(
...entry,
...buildMainSessionRecoveryClearPatch(entry),
sessionId: fork.sessionId,
lifecycleRunId: undefined,
forkSource: previousEntry?.forkSource ?? forkSource,
...(previousEntry?.sessionId && previousEntry.sessionId !== fork.sessionId
? { previousSessionId: previousEntry.sessionId }
+77 -8
View File
@@ -105,7 +105,12 @@ async function persistLifecycle(entry: SessionEntry, event: LifecycleEvent): Pro
describe("session lifecycle state", () => {
it("treats a pre-reset run's lifecycle event as stale once the row's sessionId rotated (#88538)", () => {
expect(
isStaleLifecycleEventForSession({ owningSessionId: "old-id", currentSessionId: "new-id" }),
isStaleLifecycleEventForSession({
owningSessionId: "old-id",
currentSessionId: "new-id",
eventRunId: "same-run",
currentRunId: "same-run",
}),
).toBe(true);
});
@@ -122,20 +127,26 @@ describe("session lifecycle state", () => {
});
it.each([
{ eventStartedAt: 100, currentStartedAt: 200, stale: true },
{ eventStartedAt: 200, currentStartedAt: 200, stale: false },
{ eventStartedAt: 300, currentStartedAt: 200, stale: false },
{ eventStartedAt: undefined, currentStartedAt: 200, stale: false },
{ eventStartedAt: Number.NaN, currentStartedAt: 200, stale: false },
{ eventRunId: undefined, currentRunId: undefined, eventStartedAt: 100, stale: true },
{ eventRunId: "run-a", currentRunId: "run-a", eventStartedAt: 100, stale: false },
{ eventRunId: "run-a", currentRunId: "run-b", eventStartedAt: 100, stale: true },
{ eventRunId: "run-a", currentRunId: undefined, eventStartedAt: 100, stale: true },
{ eventRunId: undefined, currentRunId: "run-a", eventStartedAt: 100, stale: true },
{ eventRunId: undefined, currentRunId: undefined, eventStartedAt: 200, stale: false },
{ eventRunId: undefined, currentRunId: undefined, eventStartedAt: 300, stale: false },
{ eventRunId: undefined, currentRunId: undefined, eventStartedAt: undefined, stale: false },
{ eventRunId: undefined, currentRunId: undefined, eventStartedAt: Number.NaN, stale: false },
])(
"correlates explicit same-session run start times",
({ eventStartedAt, currentStartedAt, stale }) => {
({ eventRunId, currentRunId, eventStartedAt, stale }) => {
expect(
isStaleLifecycleEventForSession({
owningSessionId: "session-id",
currentSessionId: "session-id",
eventRunId,
currentRunId,
eventStartedAt,
currentStartedAt,
currentStartedAt: 200,
}),
).toBe(stale);
},
@@ -159,6 +170,7 @@ describe("session lifecycle state", () => {
runId: "run-b",
data: { phase: "start", startedAt: 2_000 },
});
expect(second.lifecycleRunId).toBe("run-b");
const afterOlderTerminal = await persistLifecycle(second, {
ts: 3_000,
sessionId: "session-id",
@@ -172,6 +184,7 @@ describe("session lifecycle state", () => {
});
expect(afterOlderTerminal).toMatchObject({ status: "running", startedAt: 2_000 });
expect(afterOlderTerminal.lifecycleRunId).toBe("run-b");
expect(afterOlderTerminal.endedAt).toBeUndefined();
expect(afterOlderTerminal.lastRunError).toBeUndefined();
@@ -187,9 +200,55 @@ describe("session lifecycle state", () => {
endedAt: 4_000,
runtimeMs: 2_000,
});
expect(completed.lifecycleRunId).toBeUndefined();
},
);
it("settles a same-run terminal event whose outer start time predates its embedded start", async () => {
const started = await persistLifecycle(
{ sessionId: "session-id", updatedAt: 900 },
{
ts: 2_000,
sessionId: "session-id",
runId: "run-a",
data: { phase: "start", startedAt: 2_000 },
},
);
const completed = await persistLifecycle(started, {
ts: 3_000,
sessionId: "session-id",
runId: "run-a",
data: { phase: "end", startedAt: 1_900, endedAt: 3_000 },
});
expect(completed).toMatchObject({
status: "done",
startedAt: 1_900,
endedAt: 3_000,
runtimeMs: 1_100,
});
expect(completed.lifecycleRunId).toBeUndefined();
});
it("clears inherited run ownership when a start event has no run id", async () => {
const started = await persistLifecycle(
{
sessionId: "session-id",
updatedAt: 900,
status: "running",
startedAt: 900,
lifecycleRunId: "old-run",
},
{
ts: 2_000,
sessionId: "session-id",
data: { phase: "start", startedAt: 2_000 },
},
);
expect(started.lifecycleRunId).toBeUndefined();
});
it.each([
{
name: "aborted",
@@ -399,6 +458,7 @@ describe("session lifecycle state", () => {
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
lifecycleRunId: "foreground-run",
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "restart-run", lifecycleGeneration: "pre-restart" }],
mainRestartRecovery: {
@@ -464,6 +524,7 @@ describe("session lifecycle state", () => {
});
expect(persisted.restartRecoveryRuns).toBeUndefined();
expect(persisted.mainRestartRecovery).toBeUndefined();
expect(persisted.lifecycleRunId).toBeUndefined();
});
it("clears every generation of a resumed run when its current owner completes", async () => {
@@ -474,6 +535,7 @@ describe("session lifecycle state", () => {
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
lifecycleRunId: "recovery-run",
abortedLastRun: false,
restartRecoveryRuns: [
{ runId: "recovery-run", lifecycleGeneration: "pre-restart" },
@@ -501,6 +563,7 @@ describe("session lifecycle state", () => {
});
expect(persisted.restartRecoveryRuns).toBeUndefined();
expect(persisted.mainRestartRecovery).toBeUndefined();
expect(persisted.lifecycleRunId).toBeUndefined();
});
it("keeps an active recovery when an older same-run terminal arrives", async () => {
@@ -511,6 +574,7 @@ describe("session lifecycle state", () => {
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
lifecycleRunId: "recovery-run",
abortedLastRun: false,
restartRecoveryDeliveryRunId: "recovery-run",
restartRecoveryRuns: [
@@ -540,6 +604,7 @@ describe("session lifecycle state", () => {
mainRestartRecovery: { cycleId: "cycle-1" },
});
expect(persisted.restartRecoveryTerminalRunIds).toBeUndefined();
expect(persisted.lifecycleRunId).toBe("recovery-run");
});
it("does not settle a foreground owner from a stale lifecycle generation", async () => {
@@ -549,6 +614,7 @@ describe("session lifecycle state", () => {
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
lifecycleRunId: "foreground-run",
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" },
@@ -582,6 +648,7 @@ describe("session lifecycle state", () => {
foregroundClaims: { tokens: ["owner-claim"] },
},
});
expect(persisted.lifecycleRunId).toBe("foreground-run");
});
it("clears only the completed recovery marker", async () => {
@@ -591,6 +658,7 @@ describe("session lifecycle state", () => {
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
lifecycleRunId: "interrupted-run",
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "completed-run", lifecycleGeneration: "pre-restart" },
@@ -610,6 +678,7 @@ describe("session lifecycle state", () => {
{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" },
]);
expect(persisted.status).toBe("running");
expect(persisted.lifecycleRunId).toBe("interrupted-run");
});
it.each([
+48 -12
View File
@@ -55,6 +55,7 @@ type PersistedLifecycleSessionShape = Pick<
| "abortedLastRun"
| "restartRecoveryRuns"
| "mainRestartRecovery"
| "lifecycleRunId"
>;
type GatewaySessionLifecycleSnapshot = Partial<LifecycleSessionShape>;
@@ -214,18 +215,39 @@ function derivePersistedSessionLifecyclePatch(params: {
event: params.event,
snapshotPatch,
});
return projection.action === "suppress" ? {} : projection.patch;
if (projection.action === "suppress") {
return {};
}
const phase = resolveLifecyclePhase(params.event);
const runId = normalizeLifecycleRunId(params.event.runId);
// Run ownership follows the durable running projection. Terminal settlement
// releases it; yielded parents retain it for their continuation lifecycle.
return {
...projection.patch,
...(phase === "start"
? { lifecycleRunId: runId }
: projection.patch.status && projection.patch.status !== "running"
? { lifecycleRunId: undefined }
: {}),
};
}
export function deriveGatewaySessionLifecycleProjectionPatch(params: {
entry?: Partial<PersistedLifecycleSessionShape> | null;
event: LifecycleEventLike;
}): GatewaySessionLifecycleSnapshot {
const { restartRecoveryRuns: _restartRecoveryRuns, ...patch } =
derivePersistedSessionLifecyclePatch(params);
const {
restartRecoveryRuns: _restartRecoveryRuns,
lifecycleRunId: _lifecycleRunId,
...patch
} = derivePersistedSessionLifecyclePatch(params);
return patch;
}
function normalizeLifecycleRunId(value: unknown): string | undefined {
return typeof value === "string" && value.trim() ? value.trim() : undefined;
}
export function isRestartRecoveryLifecycleEvent(params: {
entry?: Pick<SessionEntry, "restartRecoveryRuns"> | null;
event: Pick<LifecycleEventLike, "runId" | "lifecycleGeneration" | "data">;
@@ -240,18 +262,29 @@ export function isRestartRecoveryLifecycleEvent(params: {
export function isStaleLifecycleEventForSession(params: {
owningSessionId?: string;
currentSessionId?: string;
eventRunId?: unknown;
currentRunId?: unknown;
eventStartedAt?: unknown;
currentStartedAt?: number;
}): boolean {
if (
params.owningSessionId &&
params.currentSessionId &&
params.owningSessionId !== params.currentSessionId
) {
return true;
}
const eventRunId = normalizeLifecycleRunId(params.eventRunId);
const currentRunId = normalizeLifecycleRunId(params.currentRunId);
// Matching ownership is stronger than producer timestamps. Missing or
// different identities retain the legacy timestamp fence.
if (eventRunId && currentRunId && eventRunId === currentRunId) {
return false;
}
return (
Boolean(
params.owningSessionId &&
params.currentSessionId &&
params.owningSessionId !== params.currentSessionId,
) ||
(isFiniteTimestamp(params.eventStartedAt) &&
isFiniteTimestamp(params.currentStartedAt) &&
params.eventStartedAt < params.currentStartedAt)
isFiniteTimestamp(params.eventStartedAt) &&
isFiniteTimestamp(params.currentStartedAt) &&
params.eventStartedAt < params.currentStartedAt
);
}
@@ -295,7 +328,8 @@ export async function persistGatewaySessionLifecycleEvent(params: {
storePath: sessionEntry.storePath,
sessionKey: sessionEntry.canonicalKey,
},
async (entry) => {
async (storedEntry) => {
const entry = storedEntry as SessionEntry;
if (
exactCronRun &&
!acceptsCronRunContinuationLifecycleEvent({ entry, event: params.event })
@@ -308,6 +342,8 @@ export async function persistGatewaySessionLifecycleEvent(params: {
isStaleLifecycleEventForSession({
owningSessionId,
currentSessionId: entry.sessionId,
eventRunId: params.event.runId,
currentRunId: entry.lifecycleRunId,
eventStartedAt: params.event.data?.startedAt,
currentStartedAt: entry.startedAt,
})
+41 -24
View File
@@ -7,7 +7,11 @@ import { DEFAULT_MODEL } from "../agents/defaults.js";
import type { ModelCatalogEntry } from "../agents/model-catalog.js";
import { resolveSessionModelIdentityRef } from "../agents/session-model-ref.js";
import { getSessionDisplaySubagentRunByChildSessionKey } from "../agents/subagent-registry-read.js";
import { buildGroupDisplayName, type SessionEntry } from "../config/sessions.js";
import {
buildGroupDisplayName,
type InternalSessionEntry,
type SessionEntry,
} from "../config/sessions.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { normalizeAgentId, parseAgentSessionKey } from "../routing/session-key.js";
import { sessionDeliveryChannel, sessionDeliveryOrigin } from "../utils/delivery-context.shared.js";
@@ -147,16 +151,18 @@ export function resolveSessionListSearchModelFields(params: {
return fields;
}
export function loadGatewaySessionRow(
type LoadGatewaySessionRowOptions = {
agentId?: string;
includeDerivedTitles?: boolean;
includeLastMessage?: boolean;
now?: number;
transcriptUsageMaxBytes?: number;
};
export function loadGatewaySessionLifecycleSnapshot(
sessionKey: string,
options?: {
agentId?: string;
includeDerivedTitles?: boolean;
includeLastMessage?: boolean;
now?: number;
transcriptUsageMaxBytes?: number;
},
): GatewaySessionRow | null {
options?: LoadGatewaySessionRowOptions,
): { lifecycleRunId?: string; row: GatewaySessionRow | null } {
const now = options?.now ?? Date.now();
const { cfg, storePath, store, entry, canonicalKey } = loadSessionEntryReadOnly(sessionKey, {
clone: false,
@@ -164,7 +170,7 @@ export function loadGatewaySessionRow(
...(options?.agentId ? { agentId: options.agentId } : {}),
});
if (!entry) {
return null;
return { row: null };
}
const storeChildSessionsByKey = buildSingleRowStoreChildSessionsByKey({
storePath,
@@ -172,19 +178,30 @@ export function loadGatewaySessionRow(
key: canonicalKey,
now,
});
return buildGatewaySessionRow({
cfg,
storePath,
store,
key: canonicalKey,
entry,
now,
includeDerivedTitles: options?.includeDerivedTitles,
includeLastMessage: options?.includeLastMessage,
transcriptUsageMaxBytes: options?.transcriptUsageMaxBytes,
storeChildSessionsByKey,
...(options?.agentId ? { agentId: options.agentId } : {}),
});
const lifecycleRunId = (entry as InternalSessionEntry).lifecycleRunId;
return {
...(lifecycleRunId === undefined ? {} : { lifecycleRunId }),
row: buildGatewaySessionRow({
cfg,
storePath,
store,
key: canonicalKey,
entry,
now,
includeDerivedTitles: options?.includeDerivedTitles,
includeLastMessage: options?.includeLastMessage,
transcriptUsageMaxBytes: options?.transcriptUsageMaxBytes,
storeChildSessionsByKey,
...(options?.agentId ? { agentId: options.agentId } : {}),
}),
};
}
export function loadGatewaySessionRow(
sessionKey: string,
options?: LoadGatewaySessionRowOptions,
): GatewaySessionRow | null {
return loadGatewaySessionLifecycleSnapshot(sessionKey, options).row;
}
export function buildGatewaySessionInfo(params: {
+1
View File
@@ -28,6 +28,7 @@ export { resolveGatewayModelSupportsImages } from "./session-utils-model.js";
export { resolveSessionDisplayModelIdentityRef } from "./session-utils-model.js";
export { buildGatewaySessionRow } from "./session-utils-row.js";
export { loadGatewaySessionRow } from "./session-utils-search.js";
export { loadGatewaySessionLifecycleSnapshot } from "./session-utils-search.js";
export { buildGatewaySessionInfo } from "./session-utils-search.js";
export { filterAndSortSessionEntries } from "./session-utils-list.js";
export { listSessionsFromStore } from "./session-utils-list.js";
+1
View File
@@ -50,6 +50,7 @@ const SESSION_ENTRY_RESERVED_SLOT_KEY_LIST = [
"inheritedToolPolicyVersion",
"inheritedToolDeny",
"inheritedToolAllow",
"lifecycleRunId",
"mainRestartRecovery",
"subagentRecovery",
"pluginOwnerId",