fix(ui): recover queued follow-ups from settled runs (#126180)

* fix(ui): recover queued follow-ups from settled runs

Co-authored-by: jesse-merhi <79823012+jesse-merhi@users.noreply.github.com>

* fix(ui): preserve terminal client run ownership

Co-authored-by: jesse-merhi <79823012+jesse-merhi@users.noreply.github.com>

* fix(sessions): persist recovered terminal client runs

Co-authored-by: jesse-merhi <79823012+jesse-merhi@users.noreply.github.com>

* test(gateway): publish configured reply runtime

Co-authored-by: jesse-merhi <79823012+jesse-merhi@users.noreply.github.com>

* fix(protocol): refresh Swift session row

Co-authored-by: jesse-merhi <79823012+jesse-merhi@users.noreply.github.com>

---------

Co-authored-by: Ian Moog <ianmoog42@gmail.com>
Co-authored-by: jesse-merhi <79823012+jesse-merhi@users.noreply.github.com>
Co-authored-by: Roboclaw <roboclaw-bot@users.noreply.github.com>
This commit is contained in:
ClawSweeper
2026-08-23 06:55:21 -07:00
committed by GitHub
parent b26ee4fb35
commit a77fbdef33
47 changed files with 405 additions and 23 deletions
@@ -6434,6 +6434,7 @@ public struct SessionRow: Codable, Sendable {
public let lastinteractionat: Double?
public let status: AnyCodable?
public let lastrunerror: String?
public let lastrunid: String?
public let restartrecoverystatus: String?
public let activeleafentryid: AnyCodable?
public let spawnedby: String?
@@ -6504,6 +6505,7 @@ public struct SessionRow: Codable, Sendable {
lastinteractionat: Double? = nil,
status: AnyCodable? = nil,
lastrunerror: String? = nil,
lastrunid: String? = nil,
restartrecoverystatus: String? = nil,
activeleafentryid: AnyCodable? = nil,
spawnedby: String? = nil,
@@ -6573,6 +6575,7 @@ public struct SessionRow: Codable, Sendable {
self.lastinteractionat = lastinteractionat
self.status = status
self.lastrunerror = lastrunerror
self.lastrunid = lastrunid
self.restartrecoverystatus = restartrecoverystatus
self.activeleafentryid = activeleafentryid
self.spawnedby = spawnedby
@@ -6644,6 +6647,7 @@ public struct SessionRow: Codable, Sendable {
case lastinteractionat = "lastInteractionAt"
case status
case lastrunerror = "lastRunError"
case lastrunid = "lastRunId"
case restartrecoverystatus = "restartRecoveryStatus"
case activeleafentryid = "activeLeafEntryId"
case spawnedby = "spawnedBy"
@@ -8,6 +8,7 @@ describe("SessionRowSchema", () => {
const row = {
key: "agent:main:main",
kind: "global",
lastRunId: "run-settled",
activeLeafEntryId: "leaf-rendered",
createdActor: {
type: "human",
@@ -37,6 +38,7 @@ describe("SessionRowSchema", () => {
const roundTripped = structuredClone(row);
expect(SessionRowSchema.properties.activeLeafEntryId).toBeDefined();
expect(SessionRowSchema.properties.lastRunId).toBeDefined();
expect(Value.Check(SessionRowSchema, roundTripped)).toBe(true);
expect(Value.Check(SessionRowSchema, { ...roundTripped, activeLeafEntryId: null })).toBe(true);
expect(
@@ -50,6 +52,7 @@ describe("SessionRowSchema", () => {
).toBe(false);
expect(roundTripped).toMatchObject({
activeLeafEntryId: "leaf-rendered",
lastRunId: "run-settled",
createdActor: { avatarUrl: "/api/users/profile-ada/avatar?v=7" },
participantCount: 2,
archivedBy: { type: "human", id: "profile-bob", label: "Bob" },
@@ -63,6 +66,7 @@ describe("SessionRowSchema", () => {
expect(Value.Check(SessionRowSchema, { ...roundTripped, permissionMode: "unrestricted" })).toBe(
false,
);
expect(Value.Check(SessionRowSchema, { ...roundTripped, lastRunId: "" })).toBe(false);
});
it("keeps sessions.assignOwner target actors closed and non-empty", () => {
@@ -88,6 +88,8 @@ export const SessionRowSchema = Type.Object(
]),
),
lastRunError: Type.Optional(Type.String()),
/** Exact run that produced the latest terminal lifecycle projection. */
lastRunId: Type.Optional(NonEmptyString),
restartRecoveryStatus: Type.Optional(Type.Literal("tombstoned")),
activeLeafEntryId: Type.Optional(Type.Union([NonEmptyString, Type.Null()])),
spawnedBy: Type.Optional(Type.String()),
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
const hoisted = vi.hoisted(() => ({
store: {} as Record<string, SessionEntry>,
@@ -79,6 +79,7 @@ describe("command resolveSession provider-owned daily reset", () => {
pendingTranscriptRepair: [
{ id: "predecessor-repair", text: "old reply", createdAt: startedAt },
],
lastRunId: "settled-old-run",
},
};
@@ -91,6 +92,7 @@ describe("command resolveSession provider-owned daily reset", () => {
expect(result.isNewSession).toBe(true);
expect(result.sessionId).not.toBe("old-session-id");
expect(result.sessionEntry?.pendingTranscriptRepair).toBeUndefined();
expect(result.sessionEntry?.lastRunId).toBeUndefined();
});
it("keeps a model-locked session across the daily boundary", () => {
+1
View File
@@ -80,6 +80,7 @@ export function clearRotatedSessionMetadata(entry: InternalSessionEntry): Intern
sessionFile: undefined,
status: undefined,
lifecycleRunId: undefined,
lastRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -176,6 +176,7 @@ describe("main session recovery state", () => {
it("marks without charging and replaces an older lifecycle owner for the same run", () => {
const entry = interruptedEntry({
lifecycleRunId: "dead-run",
lastRunId: "settled-run",
restartRecoveryRuns: [
{ runId: "older-run", lifecycleGeneration: "generation-old" },
{ runId: "shared-run", lifecycleGeneration: "generation-1" },
@@ -208,6 +209,7 @@ describe("main session recovery state", () => {
{ runId: "shared-run", lifecycleGeneration: "generation-2" },
]);
expect(entry.lifecycleRunId).toBeUndefined();
expect(entry.lastRunId).toBeUndefined();
});
it("rejects foreground work after the automatic recovery budget is exhausted", () => {
@@ -346,6 +348,7 @@ describe("main session recovery state", () => {
it("moves a reservation into the lifecycle fence during Gateway admission", () => {
const entry = interruptedEntry({
lastRunId: "settled-run",
pendingFinalDelivery: { kind: "replayable", text: " captured reply ", createdAt: 1 },
restartRecoveryDeliveryRunId: "recovery-1",
restartRecoveryDeliverySourceRunId: "source-1",
@@ -391,6 +394,7 @@ describe("main session recovery state", () => {
});
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
expect(entry.lifecycleRunId).toBe("recovery-1");
expect(entry.lastRunId).toBeUndefined();
expect(
transitionMainSessionRecovery(entry, {
@@ -407,6 +411,7 @@ describe("main session recovery state", () => {
expect(entry.restartRecoveryDeliveryRunId).toBeUndefined();
expect(entry.restartRecoveryDeliverySourceRunId).toBe("source-1");
expect(entry.lifecycleRunId).toBeUndefined();
expect(entry.lastRunId).toBeUndefined();
});
it("rejects a reservation created by an older lifecycle generation", () => {
@@ -21,7 +21,10 @@ import type {
MainSessionRecoveryTransitionResult,
MainSessionRecoveryView,
} from "./main-session-recovery-types.js";
import { MAX_RECOVERY_RETRIES } from "./main-session-restart-recovery-shared.js";
import {
MAX_RECOVERY_RETRIES,
resolveRestartRecoveryTerminalClientRunId,
} from "./main-session-restart-recovery-shared.js";
export type {
MainSessionRecoveryCommand,
@@ -334,6 +337,7 @@ export function transitionMainSessionRecovery(
}
entry.status = "running";
entry.lifecycleRunId = undefined;
entry.lastRunId = undefined;
entry.abortedLastRun = true;
if (command.resetRuntime) {
entry.startedAt = undefined;
@@ -517,6 +521,7 @@ export function transitionMainSessionRecovery(
});
entry.abortedLastRun = false;
entry.lifecycleRunId = command.runId;
entry.lastRunId = undefined;
recordLifecycleFence(entry, {
runId: command.runId,
lifecycleGeneration: command.lifecycleGeneration,
@@ -548,6 +553,7 @@ export function transitionMainSessionRecovery(
}
entry.status = "running";
entry.lifecycleRunId = undefined;
entry.lastRunId = undefined;
entry.abortedLastRun = true;
entry.startedAt = undefined;
entry.endedAt = undefined;
@@ -699,6 +705,7 @@ export function transitionMainSessionRecovery(
entry.abortedLastRun = false;
entry.status = "failed";
entry.lifecycleRunId = undefined;
entry.lastRunId = resolveRestartRecoveryTerminalClientRunId(entry);
entry.endedAt = command.now;
entry.runtimeMs = Math.max(0, command.now - (entry.startedAt ?? command.now));
entry.updatedAt = command.now;
@@ -23,6 +23,7 @@ import { isRestartAbortTailArtifact } from "./main-session-restart-recovery-resu
import {
buildRestartRecoveryExpectedState,
mainSessionRecoveryLog,
resolveRestartRecoveryTerminalClientRunId,
} from "./main-session-restart-recovery-shared.js";
export function hasOnlyAnnounceRecoveryRuns(entry: SessionEntry): boolean {
@@ -70,6 +71,7 @@ export async function reconcileInterruptedCompletionReport(params: {
...buildMainSessionRecoveryClearPatch(entry),
status: "killed",
lifecycleRunId: undefined,
lastRunId: resolveRestartRecoveryTerminalClientRunId(entry),
abortedLastRun: false,
endedAt,
lastRunError: undefined,
@@ -323,6 +325,7 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
}),
abortedLastRun: false,
lifecycleRunId: undefined,
lastRunId: resolveRestartRecoveryTerminalClientRunId(params.entry),
endedAt,
pendingFinalDelivery: undefined,
restartRecoveryForceSafeTools: undefined,
@@ -15,6 +15,7 @@ import { resolveRestartRecoveryDeliveryContext } from "./main-session-restart-di
import {
buildRestartRecoveryExpectedState,
mainSessionRecoveryLog,
resolveRestartRecoveryTerminalClientRunId,
} from "./main-session-restart-recovery-shared.js";
const TOMBSTONED_SESSION_NOTICE =
@@ -151,6 +152,7 @@ export async function tombstoneMainRestartRecoveryWithNotice(params: {
abortedLastRun: false,
endedAt: now,
lifecycleRunId: undefined,
lastRunId: resolveRestartRecoveryTerminalClientRunId(entry),
mainRestartRecovery: {
...recoveryState,
revision: recoveryState.revision + 1,
@@ -1,5 +1,6 @@
import path from "node:path";
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { resolveStateDir } from "../../config/paths.js";
import {
listConfiguredSessionStoreAgentIds,
@@ -54,6 +55,14 @@ export function buildRestartRecoveryExpectedState(
};
}
export function resolveRestartRecoveryTerminalClientRunId(
entry: Pick<SessionEntry, "restartRecoveryDeliverySourceRunId" | "restartRecoverySourceIngress">,
): string | undefined {
return entry.restartRecoverySourceIngress === "control-ui"
? normalizeOptionalString(entry.restartRecoveryDeliverySourceRunId)
: undefined;
}
export function normalizeStringSet(values: Iterable<string> | undefined): Set<string> {
const normalized = new Set<string>();
for (const value of values ?? []) {
@@ -53,6 +53,7 @@ import {
mainSessionRecoveryLog,
MAX_RECOVERY_RETRIES,
normalizeStringSet,
resolveRestartRecoveryTerminalClientRunId,
} from "./main-session-restart-recovery-shared.js";
function pendingFinalRecoveryAction(
@@ -112,6 +113,7 @@ async function completePendingFinalRecoveryWithNotice(
abortedLastRun: false,
endedAt,
lifecycleRunId: undefined,
lastRunId: resolveRestartRecoveryTerminalClientRunId(current),
pendingFinalDelivery: undefined,
...(pending?.context &&
pending.intentId &&
@@ -3752,7 +3752,7 @@ describe("main-session-restart-recovery", () => {
});
it("tombstones when the final owner-release retry consumes the last charge", async () => {
const { sessionsDir, storePath } = await makeMainSessionFixture({
const { sessionsDir, storePath } = await makeControlUiRecoveryFixture({
mainRestartRecovery: {
cycleId: "cycle-final-attempt",
revision: 1,
@@ -3777,6 +3777,7 @@ describe("main-session-restart-recovery", () => {
await waitForFast(() => {
expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })).toMatchObject({
status: "failed",
lastRunId: "control-ui-run",
mainRestartRecovery: { tombstone: expect.any(Object) },
});
});
@@ -4124,6 +4125,7 @@ describe("main-session-restart-recovery", () => {
restartRecoveryTerminalRunIds: ["discord-message-1"],
});
const completed = loadSessionEntry({ sessionKey, storePath });
expect(completed?.lastRunId).toBeUndefined();
expect(completed?.restartRecoveryDeliveryRunId).toBeUndefined();
expect(completed?.restartRecoveryDeliverySourceRunId).toBeUndefined();
expect(completed?.restartRecoveryDeliveryContext).toBeUndefined();
@@ -4334,10 +4336,8 @@ describe("main-session-restart-recovery", () => {
});
it("matches a checkpointed Control UI hook to its run-keyed user turn", async () => {
const { sessionsDir, storePath, sessionKey } = await makeMainSessionFixture({
const { sessionsDir, storePath, sessionKey } = await makeControlUiRecoveryFixture({
restartRecoveryBeforeAgentReplyState: "handled-silent",
restartRecoveryDeliveryRunId: "control-ui-run",
restartRecoveryDeliverySourceRunId: "control-ui-run",
});
await writeTranscript(sessionsDir, "main-session", [
{ role: "user", content: "quiet", idempotencyKey: "control-ui-run:user" },
@@ -4349,6 +4349,7 @@ describe("main-session-restart-recovery", () => {
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
status: "done",
abortedLastRun: false,
lastRunId: "control-ui-run",
restartRecoveryTerminalRunIds: ["control-ui-run"],
});
});
@@ -128,6 +128,7 @@ describe("resetReplyRunSession", () => {
updatedAt: 1,
sessionFile: path.join(rootDir, "session.jsonl"),
lifecycleRunId: "run-before-reset",
lastRunId: "run-before-reset",
agentHarnessId: "codex",
claudeCliSessionId: "native-before-boundary",
modelProvider: "qwencode",
@@ -202,6 +203,7 @@ describe("resetReplyRunSession", () => {
expect(activeSessionEntry?.sessionId).toBe("session");
expect(activeSessionEntry?.lifecycleRevision).toBe("00000000-0000-0000-0000-000000000123");
expect(activeSessionEntry?.lifecycleRunId).toBeUndefined();
expect(activeSessionEntry?.lastRunId).toBeUndefined();
expect(followupRun.run.sessionId).toBe(activeSessionEntry?.sessionId);
expect(activeSessionEntry?.modelProvider).toBeUndefined();
expect(activeSessionEntry?.agentHarnessId).toBeUndefined();
@@ -85,6 +85,7 @@ export async function resetReplyRunSession(params: {
systemSent: false,
abortedLastRun: false,
lifecycleRunId: undefined,
lastRunId: undefined,
modelProvider: undefined,
model: undefined,
inputTokens: undefined,
@@ -0,0 +1,53 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { InternalSessionEntry, SessionEntry } from "../../config/sessions.js";
const forkMocks = vi.hoisted(() => ({
forkSessionFromParent: vi.fn(),
resolveParentForkDecision: vi.fn(),
}));
vi.mock("./session-fork.js", () => forkMocks);
import { prepareReplySessionParentFork } from "./session-parent-fork-prepare.js";
describe("prepareReplySessionParentFork", () => {
beforeEach(() => {
forkMocks.forkSessionFromParent.mockReset().mockResolvedValue({
sessionId: "forked-session",
sessionFile: "/tmp/forked-session.jsonl",
});
forkMocks.resolveParentForkDecision.mockReset().mockResolvedValue({
status: "fork",
maxTokens: 100_000,
parentTokens: 10_000,
});
});
it("clears run identities when the parent fork replaces the transcript generation", async () => {
const parentEntry: SessionEntry = {
sessionId: "parent-session",
updatedAt: 1,
};
const sessionEntry: InternalSessionEntry = {
sessionId: "provisional-session",
updatedAt: 2,
lifecycleRunId: "active-provisional-run",
lastRunId: "settled-provisional-run",
};
const result = (await prepareReplySessionParentFork({
agentId: "main",
alreadyForked: false,
parentSessionKey: "agent:main:parent",
readEntry: () => parentEntry,
sessionEntry,
sessionKey: "agent:main:child",
storePath: "/tmp/sessions.json",
warn: vi.fn(),
})) as InternalSessionEntry;
expect(result.sessionId).toBe("forked-session");
expect(result.lifecycleRunId).toBeUndefined();
expect(result.lastRunId).toBeUndefined();
});
});
@@ -59,6 +59,7 @@ export async function prepareReplySessionParentFork(params: {
...buildMainSessionRecoveryClearPatch(params.sessionEntry),
sessionId: fork.sessionId,
lifecycleRunId: undefined,
lastRunId: undefined,
forkSource: {
sessionKey: params.parentSessionKey,
sessionId: parentEntry.sessionId,
@@ -2366,6 +2366,7 @@ describe("sqlite session normalization", () => {
const sourceEntry: InternalSessionEntry = {
label: "Source",
lifecycleRunId: "source-run",
lastRunId: "settled-source-run",
sessionId: "source-session",
updatedAt: 10,
compactionCheckpoints: [checkpoint],
@@ -2409,6 +2410,7 @@ describe("sqlite session normalization", () => {
}),
);
expect((result.entry as InternalSessionEntry).lifecycleRunId).toBeUndefined();
expect((result.entry as InternalSessionEntry).lastRunId).toBeUndefined();
await expect(loadTranscriptEvents(branchScope)).resolves.toEqual([
expect.objectContaining({ type: "session", id: result.entry.sessionId }),
expect.objectContaining({ id: "pre-msg", type: "message" }),
@@ -386,6 +386,7 @@ function cloneSqliteCheckpointSessionEntry(params: {
systemSent: false,
abortedLastRun: false,
lifecycleRunId: undefined,
lastRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -96,6 +96,7 @@ async function createSession(options: { activeLeafTarget?: string } = {}) {
forkSource: { sessionKey: "agent:main:root", sessionId: "root-session" },
lifecycleRevision: "source-lifecycle-revision",
lifecycleRunId: "source-run",
lastRunId: "settled-source-run",
modelOverride: "gpt-5",
modelOverrideSource: "user",
providerOverride: "openai",
@@ -578,6 +579,7 @@ describe("SQLite session message cuts", () => {
expect(loadSessionEntry(scope)?.sessionId).toBe(scope.sessionId);
expect(result.entry.lifecycleRevision).not.toBe("source-lifecycle-revision");
expect((result.entry as InternalSessionEntry).lifecycleRunId).toBeUndefined();
expect((result.entry as InternalSessionEntry).lastRunId).toBeUndefined();
expect(result.entry.cliSessionBindings).toBeUndefined();
expect(deliveryContextFromSession(result.entry)).toBeUndefined();
expect(result.entry.parentSessionKey).toBe(canonicalSourceKey);
@@ -509,6 +509,7 @@ function cloneMessageCutSessionEntry(params: {
systemSent: false,
abortedLastRun: false,
lifecycleRunId: undefined,
lastRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -235,6 +235,7 @@ export async function forkSessionEntryFromParentTarget(
},
forkedFromParent: true,
lifecycleRunId: undefined,
lastRunId: undefined,
sessionId: fork.transcript.sessionId,
totalTokens: undefined,
totalTokensFresh: false,
@@ -8,6 +8,7 @@ type RetiredThinkingSelectionQuarantine = {
export const SESSION_ENTRY_PRIVATE_CLEAR_PATCH = {
activeWriterRunId: undefined,
lastRunId: undefined,
lifecycleRunId: undefined,
mainRestartRecovery: undefined,
sessionDiffBaselineCapture: undefined,
@@ -15,6 +16,7 @@ export const SESSION_ENTRY_PRIVATE_CLEAR_PATCH = {
const PRIVATE_SESSION_ENTRY_KEYS = [
"activeWriterRunId",
"lastRunId",
"lifecycleRunId",
"mainRestartRecovery",
"sessionDiffBaselineCapture",
@@ -30,6 +30,7 @@ export type SessionTranscriptTurnLifecyclePatch = {
abortedLastRun?: boolean;
endedAt?: number;
lifecycleRunId?: SessionEntry["lifecycleRunId"];
lastRunId?: SessionEntry["lastRunId"];
pendingFinalDelivery?: SessionEntry["pendingFinalDelivery"];
mainRestartRecovery?: SessionEntry["mainRestartRecovery"];
restartRecoveryBeforeAgentReplyState?: SessionRestartRecoveryState["restartRecoveryBeforeAgentReplyState"];
+1
View File
@@ -13,6 +13,7 @@ export function recoverTerminalSessionEntryForVisibleTurn(entry: SessionEntry):
...entry,
status: undefined,
lifecycleRunId: undefined,
lastRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
+2
View File
@@ -625,6 +625,8 @@ export interface SessionEntry extends SessionEntryCore {}
export type InternalSessionEntryCore = SessionEntryCore & {
/** Run that owns the current non-terminal Gateway lifecycle projection. */
lifecycleRunId?: string;
/** Exact run that produced the latest terminal Gateway lifecycle projection. */
lastRunId?: string;
/** Run admitted by the session lane; overwritten at admission and checked by transcript writes. */
activeWriterRunId?: string;
/** Private per-generation ownership for the pre-runtime checkout baseline capture. */
@@ -2316,6 +2316,34 @@ describe("agent event handler", () => {
});
});
it("persists the linked client run without replacing provider lifecycle ownership", async () => {
const { chatRunState, handler, sessionEventSubscribers } = createHarness();
sessionEventSubscribers.subscribe("conn-session");
registerChatRun(chatRunState, "provider-run", "session-linked", "client-run");
emitAgentEvent(
handler,
"provider-run",
"lifecycle",
{ phase: "end", startedAt: 1_000, endedAt: 2_000 },
{ ts: 2_000 },
);
await waitForFast(() => {
expect(persistGatewaySessionLifecycleEventMock).toHaveBeenCalledTimes(1);
});
const params = requireRecord(
requireMockArg(persistGatewaySessionLifecycleEventMock, 0, 0, "persist lifecycle params"),
"persist lifecycle params",
);
expect(params.sessionKey).toBe("session-linked");
expect(params.event).toMatchObject({
runId: "provider-run",
clientRunId: "client-run",
data: { phase: "end" },
});
});
it("publishes run lifecycle changes to plugins without websocket subscribers", async () => {
const sessionKey = "agent:main:headless-run";
const received = vi.fn();
+14 -4
View File
@@ -602,6 +602,7 @@ export function createAgentEventHandler({
updatedAt: row.updatedAt ?? undefined,
status: row.status,
lastRunError: row.lastRunError,
lastRunId: row.lastRunId,
startedAt: row.startedAt,
endedAt: row.endedAt,
runtimeMs: row.runtimeMs,
@@ -629,6 +630,8 @@ export function createAgentEventHandler({
: {};
const clearsLastRunError =
Object.hasOwn(lifecyclePatch, "lastRunError") && lifecyclePatch.lastRunError === undefined;
const clearsLastRunId =
Object.hasOwn(lifecyclePatch, "lastRunId") && lifecyclePatch.lastRunId === undefined;
const projectedRow = row
? lifecycleProjection
? buildGatewaySessionEventRow(row, { lifecycle: true })
@@ -639,9 +642,10 @@ export function createAgentEventHandler({
...projectedRow,
...lifecyclePatch,
...activeRunFields,
// JSON drops undefined values, so a start/success must send null to
// evict a prior failure reason from the subscribed client row.
// JSON drops undefined values, so starts/successes need tombstones
// for terminal fields retained in the subscribed client row.
...(clearsLastRunError ? { lastRunError: null } : {}),
...(clearsLastRunId ? { lastRunId: null } : {}),
}
: undefined;
if (session && omitUnscopedGlobalGoal) {
@@ -897,7 +901,10 @@ export function createAgentEventHandler({
const persistence = persistGatewaySessionLifecycleEventForEvent({
sessionKey,
agentId: sessionAgentId,
event: evt,
event: {
...evt,
...(eventRunId !== evt.runId ? { clientRunId: eventRunId } : {}),
},
});
trackTrackedRunTerminalPersistence?.({
runId: evt.runId,
@@ -1770,7 +1777,10 @@ export function createAgentEventHandler({
void persistGatewaySessionLifecycleEventForEvent({
sessionKey,
agentId: sessionAgentId,
event: evt,
event: {
...evt,
...(eventRunId !== evt.runId ? { clientRunId: eventRunId } : {}),
},
}).catch((err: unknown) => {
// Surface the swallowed start-phase persistence failure: a silent write
// failure drops the run's start marker from restart-recovery accounting
@@ -12,6 +12,7 @@ function buildPatch(touchInteraction: boolean, opts?: { requestLabel?: string; l
sessionId: "session",
updatedAt: now,
lifecycleRunId: "completed-run",
lastRunId: "completed-run",
status: "failed",
agentStatus: { note: "Need a password", attention: "key", expiresAt: now + 60_000 },
...(opts?.label ? { label: opts.label } : {}),
@@ -75,6 +76,7 @@ describe("agent session patch", () => {
expect(patch.agentStatus).toBeUndefined();
expect(Object.hasOwn(patch, "lifecycleRunId")).toBe(true);
expect(patch.lifecycleRunId).toBeUndefined();
expect(patch.lastRunId).toBeUndefined();
});
it("does not clear agent status for lifecycle-only patches", () => {
@@ -264,6 +264,7 @@ export function buildAgentSessionPatch(params: {
? {
status: undefined,
lifecycleRunId: undefined,
lastRunId: undefined,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
@@ -379,6 +379,7 @@ export function buildRestartSafeChatTranscriptState(params: {
restartRecoveryDeliveryToolCallId: undefined,
status: "running",
lifecycleRunId: params.clientRunId,
lastRunId: undefined,
startedAt: params.startedAt,
endedAt: undefined,
restartRecoveryDeliveryContext: undefined,
@@ -424,6 +425,7 @@ export async function terminalizeRestartSafeChatAdmission(params: {
return {
abortedLastRun: params.retryable ? false : params.status === "killed",
lifecycleRunId: undefined,
lastRunId: params.clientRunId,
endedAt,
...(params.retryable
? {}
@@ -3156,6 +3156,9 @@ describe("gateway server chat", () => {
const dispatchRelease = createDeferred();
try {
await writeStoredMainSession(makeDoneSessionEntry());
await patchSessionEntryCore({ sessionKey: "agent:main:main", storePath }, () => ({
lastRunId: "previous-run",
}));
const context = createDirectChatContext();
dispatchInboundMessageMock.mockImplementationOnce(async () => dispatchRelease.promise);
let snapshotAtAck:
@@ -3188,6 +3191,7 @@ describe("gateway server chat", () => {
sessionId: "sess-main",
status: "running",
});
expect(snapshotAtAck?.entry?.lastRunId).toBeUndefined();
expect(snapshotAtAck?.entry?.restartRecoveryDeliveryContext).toBeUndefined();
expect(snapshotAtAck?.events).toEqual(
expect.arrayContaining([
@@ -3545,6 +3549,7 @@ describe("gateway server chat", () => {
const stored = loadSessionEntry(scope);
expect(stored).toMatchObject({
abortedLastRun: !retryable,
lastRunId: runId,
sessionId: "sess-main",
status: "killed",
});
@@ -1108,6 +1108,7 @@ describe("gateway server chat", () => {
expectRecordFields(sessionChanged.payload, {
sessionId: "sess-main",
status: "failed",
lastRunId: "idem-dispatch-error-1",
hasActiveRun: false,
});
@@ -1121,6 +1122,7 @@ describe("gateway server chat", () => {
);
const actualSession = expectRecordFields(session, {
status: "failed",
lastRunId: "idem-dispatch-error-1",
hasActiveRun: false,
});
expect(typeof actualSession.startedAt).toBe("number");
@@ -8,6 +8,7 @@ describe("buildForkedGatewaySessionEntry", () => {
sessionId: "adopted-generation",
updatedAt: 1,
lifecycleRunId: "adopted-run",
lastRunId: "settled-adopted-run",
forkSource: { sessionKey: "agent:main:original", sessionId: "original-generation" },
};
@@ -24,6 +25,7 @@ describe("buildForkedGatewaySessionEntry", () => {
forkSource: { sessionKey: "agent:main:original", sessionId: "original-generation" },
});
expect(forked.lifecycleRunId).toBeUndefined();
expect(forked.lastRunId).toBeUndefined();
});
it("uses the requested ancestry for a genuinely new node", () => {
+1
View File
@@ -13,6 +13,7 @@ export function buildForkedGatewaySessionEntry(
...buildMainSessionRecoveryClearPatch(entry),
sessionId: fork.sessionId,
lifecycleRunId: undefined,
lastRunId: undefined,
forkSource: previousEntry?.forkSource ?? forkSource,
...(previousEntry?.sessionId && previousEntry.sessionId !== fork.sessionId
? { previousSessionId: previousEntry.sessionId }
+2
View File
@@ -128,6 +128,8 @@ export function buildGatewaySessionEventFields(params: {
status: params.status ?? sessionRow.status,
// Explicit null lets subscribed clients clear the previous run's failure reason.
lastRunError: sessionRow.lastRunError ?? null,
// Explicit null lets a newer start evict the previous terminal run identity.
lastRunId: sessionRow.lastRunId ?? null,
// Explicit false lets subscribed clients drop the flag during merge-reconcile.
hasAutomation: sessionRow.hasAutomation ?? false,
...(params.hasActiveRun === undefined ? {} : { hasActiveRun: params.hasActiveRun }),
@@ -201,6 +201,7 @@ describe("session lifecycle state", () => {
runtimeMs: 2_000,
});
expect(completed.lifecycleRunId).toBeUndefined();
expect(completed.lastRunId).toBe("run-b");
},
);
@@ -230,6 +231,31 @@ describe("session lifecycle state", () => {
expect(completed.lifecycleRunId).toBeUndefined();
});
it("keeps provider lifecycle ownership while recording the client terminal run", async () => {
const started = await persistLifecycle(
{ sessionId: "session-id", updatedAt: 900 },
{
ts: 1_000,
sessionId: "session-id",
runId: "provider-run",
clientRunId: "client-run",
data: { phase: "start", startedAt: 1_000 },
},
);
expect(started.lifecycleRunId).toBe("provider-run");
const completed = await persistLifecycle(started, {
ts: 2_000,
sessionId: "session-id",
runId: "provider-run",
clientRunId: "client-run",
data: { phase: "end", startedAt: 1_000, endedAt: 2_000 },
});
expect(completed.lifecycleRunId).toBeUndefined();
expect(completed.lastRunId).toBe("client-run");
});
it("clears inherited run ownership when a start event has no run id", async () => {
const started = await persistLifecycle(
{
@@ -238,6 +264,7 @@ describe("session lifecycle state", () => {
status: "running",
startedAt: 900,
lifecycleRunId: "old-run",
lastRunId: "old-run",
},
{
ts: 2_000,
@@ -247,6 +274,7 @@ describe("session lifecycle state", () => {
);
expect(started.lifecycleRunId).toBeUndefined();
expect(started.lastRunId).toBeUndefined();
});
it.each([
+13 -3
View File
@@ -25,6 +25,7 @@ type LifecyclePhase = "start" | "end" | "error";
type LifecycleEventLike = Pick<AgentEventPayload, "ts" | "sessionId"> & {
runId?: string;
clientRunId?: string;
lifecycleGeneration?: string;
data?: {
phase?: unknown;
@@ -43,7 +44,14 @@ type LifecycleEventLike = Pick<AgentEventPayload, "ts" | "sessionId"> & {
type LifecycleSessionShape = Pick<
GatewaySessionRow,
"updatedAt" | "status" | "lastRunError" | "startedAt" | "endedAt" | "runtimeMs" | "abortedLastRun"
| "updatedAt"
| "status"
| "lastRunError"
| "lastRunId"
| "startedAt"
| "endedAt"
| "runtimeMs"
| "abortedLastRun"
>;
type PersistedLifecycleSessionShape = Pick<
@@ -51,6 +59,7 @@ type PersistedLifecycleSessionShape = Pick<
| "updatedAt"
| "status"
| "lastRunError"
| "lastRunId"
| "startedAt"
| "endedAt"
| "runtimeMs"
@@ -222,14 +231,15 @@ function derivePersistedSessionLifecyclePatch(params: {
}
const phase = resolveLifecyclePhase(params.event);
const runId = normalizeLifecycleRunId(params.event.runId);
const clientRunId = normalizeLifecycleRunId(params.event.clientRunId) ?? runId;
// Run ownership follows the durable running projection. Terminal settlement
// releases it; yielded parents retain it for their continuation lifecycle.
return {
...projection.patch,
...(phase === "start"
? { lifecycleRunId: runId }
? { lifecycleRunId: runId, lastRunId: undefined }
: projection.patch.status && projection.patch.status !== "running"
? { lifecycleRunId: undefined }
? { lifecycleRunId: undefined, lastRunId: clientRunId }
: {}),
};
}
+2 -1
View File
@@ -209,7 +209,7 @@ export function buildGatewaySessionRow(params: {
storePath: string;
store: Record<string, SessionEntry>;
key: string;
entry?: SessionEntry;
entry?: InternalSessionEntry;
modelCatalog?: SessionListModelCatalog | ModelCatalogEntry[];
now?: number;
includeDerivedTitles?: boolean;
@@ -643,6 +643,7 @@ export function buildGatewaySessionRow(params: {
estimatedCostUsd,
status: subagentRun ? subagentStatus : entry?.status,
lastRunError: entry?.lastRunError,
lastRunId: entry?.lastRunId,
hasAutomation: sessionHasAutomation(key, cfg, sessionAgentId) ? true : undefined,
subagentRunState,
hasActiveSubagentRun: subagentRun || hasActiveSubagentRun ? hasActiveSubagentRun : undefined,
+23
View File
@@ -1009,6 +1009,29 @@ describe("gateway session utils", () => {
expect(buildGatewaySessionEventFields({ sessionRow: cleared }).lastRunError).toBeNull();
});
test("session rows and update events project the exact settled run identity", () => {
const settled = buildGatewaySessionRow({
cfg: createModelDefaultsConfig({ primary: "openai/gpt-5.4" }),
storePath: "",
store: {},
key: "agent:main:settled",
lightweightListRow: true,
skipTranscriptUsageFallback: true,
entry: {
sessionId: "session-settled",
updatedAt: 1,
status: "done",
lastRunId: "run-settled",
},
});
expect(settled.lastRunId).toBe("run-settled");
expect(buildGatewaySessionEventFields({ sessionRow: settled }).lastRunId).toBe("run-settled");
const running = { ...settled, status: "running" as const, lastRunId: undefined };
expect(buildGatewaySessionEventFields({ sessionRow: running }).lastRunId).toBeNull();
});
test("session rows ignore malformed compaction checkpoints", () => {
const row = buildGatewaySessionRow({
cfg: createModelDefaultsConfig({ primary: "openai/gpt-5.4" }),
+2
View File
@@ -164,6 +164,8 @@ export type GatewaySessionRow = {
status?: SessionRunStatus;
/** Compact user-facing reason for the latest failed or timed-out run. */
lastRunError?: string;
/** Exact run that produced the latest terminal lifecycle projection. */
lastRunId?: string;
hasActiveRun?: boolean;
/** Complete exact active set when present; omitted for active owners without exact identities. */
activeRunIds?: string[];
+1
View File
@@ -58,6 +58,7 @@ const SESSION_ENTRY_RESERVED_SLOT_KEY_LIST = [
"inheritedToolDeny",
"inheritedToolAllow",
"lifecycleRunId",
"lastRunId",
"activeWriterRunId",
"mainRestartRecovery",
"subagentRecovery",
@@ -435,6 +435,87 @@ suite.define(() => {
}
});
it("sends a queued follow-up after an exact terminal session publication", async () => {
const context = await suite.newBrowserContext({
locale: "en-US",
serviceWorkers: "block",
viewport: { height: 900, width: 1280 },
});
const page = await context.newPage();
const gateway = await installMockGateway(page, {
sessionInfo: { hasActiveRun: false, status: "done" },
});
try {
await page.goto(`${suite.server.baseUrl}settings/appearance`);
await page.locator("[data-settings-follow-up-mode]").selectOption("queue");
await page.goto(`${suite.server.baseUrl}chat`);
const composer = page.locator(".agent-chat__composer-combobox textarea");
const initialText = "keep this run active until session state settles it";
await composer.fill(initialText);
await page.getByRole("button", { name: "Send message" }).click();
const initialSend = await gateway.waitForRequest("chat.send");
const initialSendParams = requireRecord(initialSend.params);
const activeRunId = requireString(initialSendParams.idempotencyKey, "active chat run id");
const activeSessionKey = requireString(initialSendParams.sessionKey, "active session key");
await page.getByRole("button", { name: "Stop generating" }).waitFor({ timeout: 10_000 });
const followUp = "send after the missed terminal event";
await composer.fill(followUp);
await page.getByRole("button", { name: "Queue message" }).click();
const queuedRow = page.locator(".chat-queue__item", { hasText: followUp });
await queuedRow.getByText("Waiting for current run").waitFor({ timeout: 10_000 });
await expectRequestCountStable(gateway, "chat.send", 1);
await gateway.setHistoryMessages([
{
__openclaw: {
idempotencyKey: `${activeRunId}:user`,
},
content: [{ text: initialText, type: "text" }],
role: "user",
timestamp: Date.now(),
},
]);
const sessionListsBeforeTerminal = (await gateway.getRequests("sessions.list")).length;
await gateway.deferNext("sessions.list");
await gateway.emitGatewayEvent("sessions.changed", {
activeRunIds: [activeRunId],
hasActiveRun: true,
key: activeSessionKey,
kind: "direct",
reason: "lifecycle",
status: "running",
updatedAt: Date.now(),
});
await expect
.poll(async () => (await gateway.getRequests("sessions.list")).length)
.toBeGreaterThan(sessionListsBeforeTerminal);
await gateway.resolveDeferred(
"sessions.list",
chatSessionListResponse([
{
activeRunIds: [],
hasActiveRun: false,
key: activeSessionKey,
kind: "direct",
label: "Main",
lastRunId: activeRunId,
status: "done",
updatedAt: Date.now(),
},
]),
);
const sends = await waitForRequests(gateway, "chat.send", 2);
expect(requireRecord(sends[1]?.params)).toMatchObject({ message: followUp });
await queuedRow.waitFor({ state: "detached", timeout: 10_000 });
} finally {
await suite.closeBrowserContext(context);
}
});
it("honors a session interrupt override ahead of the webchat config default", async () => {
const context = await suite.newBrowserContext({
locale: "en-US",
+5 -3
View File
@@ -40,7 +40,7 @@ import { resolveChatAgentId, selectedChatSessionRow } from "./chat-state-route.t
import { releaseChatMediaResourceSubscriber } from "./components/chat-message-media.ts";
import { retireSessionWorkspaceCheckout } from "./components/chat-session-workspace.ts";
import {
reconcileStaleChatRunAfterSessionStatePublication,
reconcileChatRunAfterSessionStatePublication,
replayPendingChatAbort,
} from "./run-lifecycle.ts";
import { cancelChatScroll } from "./scroll.ts";
@@ -167,9 +167,11 @@ export abstract class ChatPaneContext extends ChatPaneLifecycle {
);
return;
}
const reconciledLocalCompletion = reconcileStaleChatRunAfterSessionStatePublication(state);
const reconciledLocalCompletion = reconcileChatRunAfterSessionStatePublication(state);
this.reconcileWaitingApprovalSnapshot();
if (!reconciledLocalCompletion) {
if (reconciledLocalCompletion) {
void retryReconnectableQueuedChatSends(state);
} else {
state.requestUpdate?.();
}
}
@@ -14,6 +14,37 @@ function advertiseSessionRecovery(pane: TestChatPane) {
}
describe("chat pane session recovery", () => {
it("unlocks the composer when shared session state settles the exact local run", () => {
const { pane, state } = createTestChatPane({
client: {} as GatewayBrowserClient,
sessions: {} as SessionCapability,
});
state.chatRunId = "run-missed-terminal";
state.chatStream = "answer already rendered";
pane.applySessionsState({
result: {
sessions: [
{
key: state.sessionKey,
kind: "direct",
updatedAt: 20,
status: "done",
hasActiveRun: false,
lastRunId: "run-missed-terminal",
},
],
},
agentId: "main",
loading: false,
error: null,
deletedSessions: [],
} as unknown as Parameters<typeof pane.applySessionsState>[0]);
expect(state.chatRunId).toBeNull();
expect(state.chatStream).toBeNull();
});
it("recovers a tombstoned session into a fresh continuing session", async () => {
const created = createDeferred<Awaited<ReturnType<SessionCapability["recover"]>>>();
const sessions = {
+2 -2
View File
@@ -48,7 +48,7 @@ import { readChatSessionProjectionScope, reduceChatSessionProjection } from "./h
import {
reconcileChatRunFromCurrentSessionRow,
reconcileChatRunFromSessionRow,
reconcileStaleChatRunAfterSessionStatePublication,
reconcileChatRunAfterSessionStatePublication,
} from "./run-lifecycle.ts";
import { applySessionMessagePayload } from "./session-message-apply.ts";
import { rememberAuthoritativeTerminal } from "./terminal-message-identity.ts";
@@ -94,7 +94,7 @@ function reconcileSessionEvent(state: ChatPageHost, payload: unknown): SessionCh
state.sessionsResult = state.sessions.state.result;
state.sessionsResultAgentId = state.sessions.state.agentId;
state.sessionsError = state.sessions.state.error;
reconcileStaleChatRunAfterSessionStatePublication(state);
reconcileChatRunAfterSessionStatePublication(state);
}
return reconciled;
}
+34 -2
View File
@@ -13,7 +13,7 @@ import {
reconcileChatRunFromCurrentSessionRow,
reconcileChatRunFromSessionRow,
reconcileChatRunLifecycle,
reconcileStaleChatRunAfterSessionStatePublication,
reconcileChatRunAfterSessionStatePublication,
replayPendingChatAbort,
} from "./run-lifecycle.ts";
import { buildToolStreamIdentity } from "./tool-stream-identity.ts";
@@ -25,6 +25,7 @@ type TestRow = {
hasActiveSubagentRun?: boolean;
activeRunIds?: string[];
status?: string;
lastRunId?: string;
startedAt?: number;
};
@@ -850,10 +851,41 @@ describe("reconcileChatRunFromCurrentSessionRow stale-active suppression (#87875
lastLocalTerminalReconcile: makeLocalTerminalReconcile(),
});
expect(reconcileStaleChatRunAfterSessionStatePublication(host)).toBe(true);
expect(reconcileChatRunAfterSessionStatePublication(host)).toBe(true);
expect(rowActive(host)).toBe(false);
});
it("recovers a missed terminal event from the exact settled session row", () => {
const host = makeHost({
chatRunId: "r1",
chatStream: "complete reply",
sessionsResult: makeSessionsResult([
{ key: "s1", hasActiveRun: false, lastRunId: "r1", status: "done" },
]),
});
expect(reconcileChatRunAfterSessionStatePublication(host)).toBe(true);
expect(host.chatRunId).toBeNull();
expect(host.chatStream).toBeNull();
});
it.each([undefined, "older-run"])(
"does not settle a live run from a %s terminal row identity",
(lastRunId) => {
const host = makeHost({
chatRunId: "r1",
chatStream: "still running",
sessionsResult: makeSessionsResult([
{ key: "s1", hasActiveRun: false, lastRunId, status: "done" },
]),
});
expect(reconcileChatRunAfterSessionStatePublication(host)).toBe(false);
expect(host.chatRunId).toBe("r1");
expect(host.chatStream).toBe("still running");
},
);
it("keeps suppressing repeated stale active refreshes for the completed run", () => {
const host = makeHost({
lastLocalTerminalReconcile: makeLocalTerminalReconcile(),
+6 -2
View File
@@ -338,7 +338,7 @@ function scheduleRunStatusClear(host: RunLifecycleHost, status: ChatRunUiStatus)
host.chatRunStatusClearTimer = null;
// Terminal status temporarily masks stale active rows from session polling.
// Reconcile again as the mask expires so the composer cannot revert to Stop.
if (!reconcileStaleChatRunAfterSessionStatePublication(host)) {
if (!reconcileChatRunAfterSessionStatePublication(host)) {
host.requestUpdate?.();
}
}, CHAT_RUN_STATUS_TOAST_DURATION_MS);
@@ -561,7 +561,11 @@ export function reconcileChatRunFromCurrentSessionRow(
return reconcileChatRunFromSessionRow(host, row, options);
}
export function reconcileStaleChatRunAfterSessionStatePublication(host: RunLifecycleHost): boolean {
export function reconcileChatRunAfterSessionStatePublication(host: RunLifecycleHost): boolean {
const row = currentSessionRow(host);
if (host.chatRunId && row?.lastRunId === host.chatRunId) {
return reconcileChatRunFromSessionRow(host, row, { publishRunStatus: false });
}
// Both session subscriptions and direct event reconciliation can republish
// canonical rows after the local terminal projection; guard both paths.
const canReconcile =