diff --git a/scripts/bench-agent-concurrency-worker.ts b/scripts/bench-agent-concurrency-worker.ts index df90c42dac40..264408667b79 100644 --- a/scripts/bench-agent-concurrency-worker.ts +++ b/scripts/bench-agent-concurrency-worker.ts @@ -636,6 +636,7 @@ async function runSweepSample(childCount: number): Promise { resumeRequesterSettleWake: () => {}, startSubagentAnnounceCleanupFlow: () => true, completeCleanupBookkeeping: () => {}, + discardTerminalDelivery: () => {}, shouldEmitEndedHookForRun: () => false, emitSubagentEndedHookForRun: async () => {}, callGateway: (async () => { diff --git a/src/agents/subagents/completion/subagent-completion-admission.store.test.ts b/src/agents/subagents/completion/subagent-completion-admission.store.test.ts index 47650cdb5a4c..c9e78f125881 100644 --- a/src/agents/subagents/completion/subagent-completion-admission.store.test.ts +++ b/src/agents/subagents/completion/subagent-completion-admission.store.test.ts @@ -1,3 +1,4 @@ +import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../../../test/helpers/temp-dir.js"; @@ -18,6 +19,7 @@ import type { TaskRecord } from "../../../tasks/task-registry.types.js"; import { resetTaskRegistryForTests } from "../../../tasks/task-runtime.test-helpers.js"; import { withEnvAsync } from "../../../test-utils/env.js"; import { createSubagentRunRecord } from "../../subagent-test-fixtures.test-helpers.js"; +import { SubagentLifecycleController } from "../registry/subagent-registry-lifecycle.js"; import { subagentRuns } from "../registry/subagent-registry-memory.js"; import { loadSubagentRegistryFromSqlite } from "../registry/subagent-registry.store.sqlite.js"; import type { SubagentRunRecord } from "../registry/subagent-registry.types.js"; @@ -33,6 +35,8 @@ import { const resumeSubagentRun = vi.hoisted(() => vi.fn()); const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const discardTerminalDelivery = (entry: SubagentRunRecord, completedAt: number) => + SubagentLifecycleController.discardTerminalDelivery(entry, completedAt); vi.mock("../registry/subagent-registry.js", () => ({ resumeSubagentRun })); @@ -373,6 +377,12 @@ describe("atomic subagent completion admission store", () => { }); const cappedSubagent = structuredClone(subagentRuns.get(input.subagent.runId)!); + const attachmentsRootDir = path.join(tempDir, "attachments"); + const attachmentsDir = path.join(attachmentsRootDir, "completion-run"); + await fs.mkdir(attachmentsDir, { recursive: true }); + await fs.writeFile(path.join(attachmentsDir, "result.txt"), "retained result"); + cappedSubagent.attachmentsRootDir = attachmentsRootDir; + cappedSubagent.attachmentsDir = attachmentsDir; Object.assign(cappedSubagent.delivery!, { status: "suspended", generation: 10, @@ -400,8 +410,43 @@ describe("atomic subagent completion admission store", () => { reason: "completion delivery redrive limit reached", }); expect(resumeSubagentRun).not.toHaveBeenCalled(); + const discardInsideTransaction = vi.fn((entry: SubagentRunRecord, completedAt: number) => { + expect(database.db.isTransaction).toBe(true); + discardTerminalDelivery(entry, completedAt); + }); + database.db.exec(` + CREATE TRIGGER fail_dismissed_task_persist + BEFORE UPDATE ON task_runs + BEGIN + SELECT RAISE(ABORT, 'injected dismissal persistence failure'); + END; + `); - const dismissed = dismissSubagentCompletionDelivery(input.task.taskId); + await expect( + dismissSubagentCompletionDelivery(input.task.taskId, { + discardTerminalDelivery: discardInsideTransaction, + databaseOptions: { database }, + }), + ).rejects.toThrow("injected dismissal persistence failure"); + expect(subagentRuns.get(input.subagent.runId)?.delivery?.status).toBe("suspended"); + expect(getTaskById(input.task.taskId)?.deliveryStatus).toBe("failed"); + const rolledBackSubagent = database.db + .prepare("SELECT payload_json FROM subagent_runs WHERE run_id = ?") + .get(input.subagent.runId) as { payload_json: string }; + expect(JSON.parse(rolledBackSubagent.payload_json).delivery.status).toBe("suspended"); + const rolledBackTask = database.db + .prepare("SELECT delivery_status FROM task_runs WHERE task_id = ?") + .get(input.task.taskId) as { delivery_status: string }; + expect(rolledBackTask.delivery_status).toBe("failed"); + await expect(fs.stat(attachmentsDir)).resolves.toBeDefined(); + database.db.exec("DROP TRIGGER fail_dismissed_task_persist"); + + const dismissed = await dismissSubagentCompletionDelivery(input.task.taskId, { + discardTerminalDelivery: discardInsideTransaction, + databaseOptions: { database }, + }); + expect(discardInsideTransaction).toHaveBeenCalledTimes(2); + await expect(fs.stat(attachmentsDir)).rejects.toMatchObject({ code: "ENOENT" }); expect(dismissed).toMatchObject({ ok: true, task: { @@ -413,7 +458,11 @@ describe("atomic subagent completion admission store", () => { expect(subagentRuns.get(input.subagent.runId)?.delivery).toMatchObject({ status: "discarded", disposition: "intentional_non_delivery", + payload: undefined, + suspendedAt: undefined, + suspendedReason: undefined, }); + expect(subagentRuns.get(input.subagent.runId)?.cleanupCompletedAt).toBeTypeOf("number"); resetTaskRegistryForTests({ persist: false }); subagentRuns.clear(); @@ -426,6 +475,15 @@ describe("atomic subagent completion admission store", () => { terminalOutcome: "blocked", progressSummary: "canonical result", }); + expect(subagentRuns.get(input.subagent.runId)).toMatchObject({ + cleanupHandled: true, + cleanupCompletedAt: expect.any(Number), + delivery: { + status: "discarded", + disposition: "intentional_non_delivery", + }, + }); + expect(subagentRuns.get(input.subagent.runId)?.delivery).not.toHaveProperty("payload"); }); }); }); diff --git a/src/agents/subagents/completion/subagent-completion-admission.store.ts b/src/agents/subagents/completion/subagent-completion-admission.store.ts index 7781b54cd5cb..0e4e800c046b 100644 --- a/src/agents/subagents/completion/subagent-completion-admission.store.ts +++ b/src/agents/subagents/completion/subagent-completion-admission.store.ts @@ -122,12 +122,13 @@ export function settleSubagentCompletionDelivery(params: { subagent: SubagentRunRecord; task: TaskRecord; databaseOptions?: OpenClawStateDatabaseOptions; + mutateSubagent?: (entry: SubagentRunRecord) => unknown; }): void { - const boundSubagent = bindSubagentRunRecord(params.subagent); const boundTask = bindTaskRecord(params.task); runOpenClawStateWriteTransaction( (database) => { - upsertSubagentRunRowInDatabase(database, boundSubagent); + invokeSynchronousHook(() => params.mutateSubagent?.(params.subagent)); + upsertSubagentRunRowInDatabase(database, bindSubagentRunRecord(params.subagent)); upsertTaskRunRowInDatabase(database, boundTask); }, params.databaseOptions, diff --git a/src/agents/subagents/completion/subagent-completion-delivery.ts b/src/agents/subagents/completion/subagent-completion-delivery.ts index 901e7019eff8..816adbe7c6ec 100644 --- a/src/agents/subagents/completion/subagent-completion-delivery.ts +++ b/src/agents/subagents/completion/subagent-completion-delivery.ts @@ -21,7 +21,11 @@ import { } from "../../../tasks/runtime-internal.js"; import type { TaskRecord } from "../../../tasks/task-registry.types.js"; import { ensureDeliveryState } from "../registry/subagent-delivery-state.js"; -import { ANNOUNCE_COMPLETION_HARD_EXPIRY_MS } from "../registry/subagent-registry-helpers.js"; +import { + ANNOUNCE_COMPLETION_HARD_EXPIRY_MS, + safeRemoveAttachmentsDir, +} from "../registry/subagent-registry-helpers.js"; +import type { SubagentLifecycleController } from "../registry/subagent-registry-lifecycle.js"; import { subagentRuns } from "../registry/subagent-registry-memory.js"; import type { SubagentRunRecord } from "../registry/subagent-registry.types.js"; import { @@ -35,6 +39,12 @@ const SUSPENDED_RETENTION_MS = 7 * 24 * 60 * 60_000; const MAX_DELIVERY_GENERATION = 10; const CANONICAL_RESULT_PROMPT = "A completed subagent task is ready for parent review. The canonical result follows."; +type CompletionDeliveryRecoveryResult = { + ok: boolean; + reason?: string; + task?: TaskRecord; + duplicateRisk?: boolean; +}; function resolveTask(entry: SubagentRunRecord): TaskRecord | undefined { return findTaskByRunId(entry.taskRunId ?? entry.runId); @@ -236,12 +246,7 @@ export async function settleCorrelatedSubagentDelivery( export async function retrySubagentCompletionDelivery( taskId: string, databaseOptions?: OpenClawStateDatabaseOptions, -): Promise<{ - ok: boolean; - reason?: string; - task?: TaskRecord; - duplicateRisk?: boolean; -}> { +): Promise { const task = getTaskById(taskId); const current = task ? findSubagentForTask(task) : undefined; if (!task || !current || current.expectsCompletionMessage !== true) { @@ -309,11 +314,13 @@ export async function retrySubagentCompletionDelivery( return { ok: true, task: getTaskById(taskId), duplicateRisk: true }; } -export function dismissSubagentCompletionDelivery(taskId: string): { - ok: boolean; - reason?: string; - task?: TaskRecord; -} { +export async function dismissSubagentCompletionDelivery( + taskId: string, + options: { + discardTerminalDelivery: typeof SubagentLifecycleController.discardTerminalDelivery; + databaseOptions?: OpenClawStateDatabaseOptions; + }, +): Promise { const task = getTaskById(taskId); const current = task ? findSubagentForTask(task) : undefined; if (!task || !current || current.delivery?.status !== "suspended") { @@ -321,12 +328,6 @@ export function dismissSubagentCompletionDelivery(taskId: string): { } const now = Date.now(); const subagent = structuredClone(current); - const delivery = ensureDeliveryState(subagent); - delivery.status = "discarded"; - delivery.disposition = "intentional_non_delivery"; - delivery.dismissedAt = now; - delivery.queueId = undefined; - delivery.nextAttemptAt = undefined; const projectedTask: TaskRecord = { ...task, deliveryStatus: "dismissed", @@ -336,7 +337,15 @@ export function dismissSubagentCompletionDelivery(taskId: string): { cleanupAfter: Math.max(task.cleanupAfter ?? 0, now + SUSPENDED_RETENTION_MS), lastEventAt: now, }; - settleSubagentCompletionDelivery({ subagent, task: projectedTask }); + settleSubagentCompletionDelivery({ + subagent, + task: projectedTask, + databaseOptions: options.databaseOptions, + mutateSubagent: (entry) => options.discardTerminalDelivery(entry, now), + }); publishCommittedRecords(subagent, projectedTask); + if (subagent.cleanup === "delete" || !subagent.retainAttachmentsOnKeep) { + await safeRemoveAttachmentsDir(subagent); + } return { ok: true, task: getTaskById(taskId) }; } diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-announce-cleanup.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-announce-cleanup.ts new file mode 100644 index 000000000000..3ac89d558d22 --- /dev/null +++ b/src/agents/subagents/registry/subagent-registry-lifecycle-announce-cleanup.ts @@ -0,0 +1,627 @@ +import { defaultRuntime } from "../../../runtime.js"; +import { normalizeDeliveryContext } from "../../../utils/delivery-context.shared.js"; +import { + ensureCompletionState, + ensureDeliveryState, + getDeliveryLastError, + isDeliverySuspended, +} from "./subagent-delivery-state.js"; +import { SUBAGENT_ENDED_REASON_COMPLETE } from "./subagent-lifecycle-events.js"; +import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; +import { + resolveCleanupCompletionReason, + resolveDeferredCleanupDecision, +} from "./subagent-registry-cleanup.js"; +import { + ANNOUNCE_COMPLETION_HARD_EXPIRY_MS, + ANNOUNCE_EXPIRY_MS, + logAnnounceGiveUp, + MIN_ANNOUNCE_RETRY_DELAY_MS, + resolveAnnounceRetryDelayMs, + safeRemoveAttachmentsDir, +} from "./subagent-registry-helpers.js"; +import { + beginSubagentCleanup, + retireSupersededCleanupIfNeeded, + retireSupersededCleanupInBackground, + runDetachedCleanupAttempt, + scheduleResumeSubagentRun, + suspendPendingFinalDelivery, +} from "./subagent-registry-lifecycle-cleanup.js"; +import type { SubagentLifecycleAnnounceCleanupContext } from "./subagent-registry-lifecycle-context.js"; +import { + buildSafeLifecycleErrorMeta, + clearSubagentPendingDelivery, + emitCompletionEndedHookIfNeeded, + formatAnnounceDeliveryError, + hasPriorRequesterDeliveryMirror, + loadPendingFinalDeliveryPayload, + markPendingFinalDelivery, + maskLifecycleIdentifier, + recordAnnounceDeliveryResult, + safeMarkRequiredCompletionDeliveryBlocked, + safeSetSubagentTaskDeliveryStatus, +} from "./subagent-registry-lifecycle-delivery.js"; +import { loadSubagentSessionEntry } from "./subagent-session-reconciliation.js"; + +type RunSubagentAnnounceFlow = + (typeof import("../announce/subagent-announce.js"))["runSubagentAnnounceFlow"]; +type SubagentAnnounceFlowOutcome = Awaited>; +import type { SubagentRunRecord } from "./subagent-registry.types.js"; +import { deleteSubagentSessionForCleanup } from "./subagent-session-cleanup.js"; + +const shouldSuspendPendingFinalDelivery = (entry: SubagentRunRecord) => + entry.expectsCompletionMessage === true && + entry.endedReason === SUBAGENT_ENDED_REASON_COMPLETE && + entry.execution.outcome?.status === "ok"; + +export const finalizeResumedAnnounceGiveUp = async ( + context: SubagentLifecycleAnnounceCleanupContext, + giveUpParams: { + runId: string; + entry: SubagentRunRecord; + reason: "expiry" | "permanent_failure"; + cleanup?: "delete" | "keep"; + cleanupGeneration?: number; + retryCount?: number; + completedAt?: number; + }, +) => { + const params = context.options; + const { runId, entry, reason, cleanup, cleanupGeneration, retryCount, completedAt } = + giveUpParams; + if (shouldSuspendPendingFinalDelivery(entry)) { + suspendPendingFinalDelivery(context, { + runId, + entry, + reason, + error: getDeliveryLastError(entry), + }); + return; + } + const deliveryError = getDeliveryLastError(entry) ?? reason; + clearSubagentPendingDelivery(entry); + const failedDelivery = ensureDeliveryState(entry); + failedDelivery.status = "failed"; + failedDelivery.lastError = deliveryError; + if (retryCount != null) { + failedDelivery.attemptCount = retryCount; + failedDelivery.lastAttemptAt = completedAt ?? Date.now(); + } + safeSetSubagentTaskDeliveryStatus(params, { + entry, + deliveryStatus: "failed", + deliveryError, + }); + safeMarkRequiredCompletionDeliveryBlocked(params, { + entry, + reason: deliveryError, + }); + entry.wakeOnDescendantSettle = undefined; + const completion = ensureCompletionState(entry); + completion.fallbackResultText = undefined; + completion.fallbackCapturedAt = undefined; + if ((cleanup ?? entry.cleanup) === "delete" || !entry.retainAttachmentsOnKeep) { + await safeRemoveAttachmentsDir(entry); + } + if ( + cleanupGeneration !== undefined && + !context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration) + ) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + const completionReason = resolveCleanupCompletionReason(entry); + logAnnounceGiveUp(entry, reason); + // Retry-limit / expiry give-up should not leave cleanup stuck behind the + // best-effort ended hook. Mark the run cleaned first, then fire the hook. + context.completeCleanupBookkeeping({ + runId, + entry, + cleanup: cleanup ?? entry.cleanup, + completedAt: completedAt ?? Date.now(), + }); + if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { + await emitCompletionEndedHookIfNeeded( + params, + entry, + completionReason, + () => + context.isEndedHookOwnerCurrent(runId, entry) && + !shouldSuppressSubagentRecoverySessionEffects(entry), + ); + } +}; + +export const retryDeferredCompletedAnnounces = ( + context: SubagentLifecycleAnnounceCleanupContext, + excludeRunId?: string, +) => { + const params = context.options; + const now = Date.now(); + for (const [runId, entry] of params.runs.entries()) { + if (excludeRunId && runId === excludeRunId) { + continue; + } + if (typeof entry.execution.endedAt !== "number") { + continue; + } + if (entry.cleanupCompletedAt || entry.cleanupHandled) { + continue; + } + if (isDeliverySuspended(entry)) { + continue; + } + if (params.suppressAnnounceForSteerRestart(entry)) { + continue; + } + const endedAgo = now - (entry.execution.endedAt ?? now); + if (entry.expectsCompletionMessage !== true && endedAgo > ANNOUNCE_EXPIRY_MS) { + const cleanupGeneration = beginSubagentCleanup(context, runId); + if (cleanupGeneration === undefined) { + continue; + } + runDetachedCleanupAttempt(context, { + runId, + entry, + cleanupGeneration, + run: async () => { + await finalizeResumedAnnounceGiveUp(context, { + runId, + entry, + reason: "expiry", + }); + }, + }); + continue; + } + params.resumedRuns.delete(runId); + params.resumeSubagentRun(runId); + } +}; + +const finalizeSubagentCleanup = async ( + context: SubagentLifecycleAnnounceCleanupContext, + runId: string, + cleanup: "delete" | "keep", + announceOutcome: SubagentAnnounceFlowOutcome, + cleanupGeneration: number, + options?: { + skipAnnounce?: boolean; + skipDeliveryStatus?: boolean; + skipRequesterDelivery?: boolean; + }, +) => { + const params = context.options; + const entry = params.runs.get(runId); + if (!entry) { + return; + } + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + if (entry.expectsCompletionMessage === false || options?.skipRequesterDelivery) { + clearSubagentPendingDelivery(entry); + if (options?.skipRequesterDelivery) { + ensureDeliveryState(entry).status = "not_required"; + entry.suppressCompletionDelivery = undefined; + } + entry.wakeOnDescendantSettle = undefined; + const shouldDeleteAttachments = cleanup === "delete" || !entry.retainAttachmentsOnKeep; + if (shouldDeleteAttachments) { + await safeRemoveAttachmentsDir(entry); + } + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + context.completeCleanupBookkeeping({ + runId, + entry, + cleanup, + completedAt: Date.now(), + }); + if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { + await emitCompletionEndedHookIfNeeded( + params, + entry, + resolveCleanupCompletionReason(entry), + () => + context.isEndedHookOwnerCurrent(runId, entry) && + !shouldSuppressSubagentRecoverySessionEffects(entry), + ); + } + return; + } + if (announceOutcome === "delivered" || announceOutcome === "intentional_non_delivery") { + const delivery = ensureDeliveryState(entry); + const shouldCreditDelivery = + announceOutcome === "delivered" && + (!options?.skipAnnounce || + delivery.status === "delivered" || + typeof delivery.announcedAt === "number"); + if (shouldCreditDelivery) { + const deliveredAt = delivery.deliveredAt ?? delivery.announcedAt ?? Date.now(); + delivery.status = "delivered"; + delivery.deliveredAt = deliveredAt; + delivery.announcedAt = delivery.announcedAt ?? deliveredAt; + if (!options?.skipAnnounce) { + delivery.announcedAt = deliveredAt; + params.persist(runId); + } + } + if (announceOutcome === "delivered") { + clearSubagentPendingDelivery(entry); + } else { + // The requester-settle batch owns the real delivery now. Retire the + // per-child retry obligation without converting the handoff into success. + delivery.status = "pending"; + delivery.disposition = "intentional_non_delivery"; + delivery.payload = undefined; + delivery.createdAt = undefined; + delivery.attemptCount = undefined; + delivery.nextAttemptAt = undefined; + } + const finalDelivery = ensureDeliveryState(entry); + if (shouldCreditDelivery) { + finalDelivery.status = "delivered"; + finalDelivery.suspendedAt = undefined; + finalDelivery.suspendedReason = undefined; + } + if (shouldCreditDelivery && !options?.skipDeliveryStatus) { + safeSetSubagentTaskDeliveryStatus(params, { + entry, + deliveryStatus: "delivered", + }); + } else if (announceOutcome === "intentional_non_delivery" && !options?.skipDeliveryStatus) { + safeSetSubagentTaskDeliveryStatus(params, { + entry, + deliveryStatus: "pending", + }); + } + if (announceOutcome === "delivered") { + finalDelivery.lastError = undefined; + finalDelivery.lastDropReason = undefined; + } + entry.wakeOnDescendantSettle = undefined; + const completion = ensureCompletionState(entry); + completion.fallbackResultText = undefined; + completion.fallbackCapturedAt = undefined; + const completionReason = resolveCleanupCompletionReason(entry); + const shouldDeleteAttachments = cleanup === "delete" || !entry.retainAttachmentsOnKeep; + if (shouldDeleteAttachments) { + await safeRemoveAttachmentsDir(entry); + } + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + context.completeCleanupBookkeeping({ + runId, + entry, + cleanup, + completedAt: Date.now(), + }); + // Hook loading is best-effort; durable delivery and cleanup must already + // be terminal before plugin code can fail or stall. + if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { + await emitCompletionEndedHookIfNeeded( + params, + entry, + completionReason, + () => + context.isEndedHookOwnerCurrent(runId, entry) && + !shouldSuppressSubagentRecoverySessionEffects(entry), + ); + } + return; + } + + if (announceOutcome === "session_queued") { + // The correlated queue owns transport now. Settlement, not admission, + // decides delivered versus blocked and re-enters cleanup afterward. + entry.cleanupHandled = false; + params.resumedRuns.delete(runId); + params.persist(runId); + return; + } + + const now = Date.now(); + const deferredDecision = resolveDeferredCleanupDecision({ + entry, + now, + activeDescendantRuns: Math.max(0, params.countPendingDescendantRuns(entry.childSessionKey)), + announceExpiryMs: ANNOUNCE_EXPIRY_MS, + announceCompletionHardExpiryMs: ANNOUNCE_COMPLETION_HARD_EXPIRY_MS, + deferDescendantDelayMs: MIN_ANNOUNCE_RETRY_DELAY_MS, + resolveAnnounceRetryDelayMs, + }); + + if (deferredDecision.kind === "defer-descendants") { + ensureDeliveryState(entry).lastAttemptAt = now; + entry.wakeOnDescendantSettle = true; + entry.cleanupHandled = false; + params.resumedRuns.delete(runId); + params.persist(runId); + scheduleResumeSubagentRun(context, runId, entry, deferredDecision.delayMs); + return; + } + + if (deferredDecision.kind === "give-up") { + await finalizeResumedAnnounceGiveUp(context, { + runId, + entry, + reason: deferredDecision.reason, + cleanup, + cleanupGeneration, + retryCount: deferredDecision.retryCount, + completedAt: now, + }); + return; + } + + markPendingFinalDelivery({ + entry, + error: "announce deferred or direct delivery failed", + }); + const delivery = ensureDeliveryState(entry); + delivery.windowStartedAt ??= entry.execution.endedAt ?? now; + delivery.deadlineAt ??= delivery.windowStartedAt + ANNOUNCE_COMPLETION_HARD_EXPIRY_MS; + delivery.nextAttemptAt = now + (deferredDecision.resumeDelayMs ?? 0); + entry.cleanupHandled = false; + params.resumedRuns.delete(runId); + params.persist(runId); + if (deferredDecision.resumeDelayMs == null) { + return; + } + scheduleResumeSubagentRun(context, runId, entry, deferredDecision.resumeDelayMs); +}; + +export const startSubagentAnnounceCleanupFlow = ( + context: SubagentLifecycleAnnounceCleanupContext, + runId: string, + entry: SubagentRunRecord, +): boolean => { + const params = context.options; + if (entry.killReconciliation) { + // Restores and unrelated cleanup retries must not publish a provisional + // kill. The sweeper re-enters here after durable reconciliation. + return false; + } + const cleanup = entry.cleanup; + let suppressSessionEffects = shouldSuppressSubagentRecoverySessionEffects(entry); + if (typeof entry.delivery?.announcedAt === "number" || entry.delivery?.status === "delivered") { + const cleanupGeneration = beginSubagentCleanup(context, runId); + if (cleanupGeneration === undefined) { + return false; + } + runDetachedCleanupAttempt(context, { + runId, + entry, + cleanupGeneration, + run: async () => { + await finalizeSubagentCleanup(context, runId, cleanup, "delivered", cleanupGeneration, { + skipAnnounce: true, + }); + }, + }); + return true; + } + const cleanupGeneration = beginSubagentCleanup(context, runId); + if (cleanupGeneration === undefined) { + return false; + } + const cleanupSessionEntry = suppressSessionEffects + ? undefined + : loadSubagentSessionEntry({ childSessionKey: entry.childSessionKey }); + const cleanupSessionIdentity = + cleanupSessionEntry?.sessionId && cleanupSessionEntry.lifecycleRevision + ? { + sessionId: cleanupSessionEntry.sessionId, + lifecycleRevision: cleanupSessionEntry.lifecycleRevision, + } + : undefined; + const suppressChildSessionEffects = () => { + suppressSessionEffects = true; + if (entry.execution.suppressSessionEffects !== true) { + const previousExecution = entry.execution; + entry.execution = { + ...entry.execution, + suppressSessionEffects: true, + }; + try { + params.persistOrThrow(runId); + } catch (error) { + entry.execution = previousExecution; + suppressSessionEffects = false; + throw error; + } + } + }; + const childSessionEffectsAllowed = () => { + if (!suppressSessionEffects && shouldSuppressSubagentRecoverySessionEffects(entry)) { + suppressChildSessionEffects(); + } + return ( + !suppressSessionEffects && context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration) + ); + }; + const skipRequesterDelivery = entry.suppressCompletionDelivery === true; + if (entry.expectsCompletionMessage === false || skipRequesterDelivery) { + runDetachedCleanupAttempt(context, { + runId, + entry, + cleanupGeneration, + run: async () => { + // This driver is detached. Yield once so synchronous successor + // registration can invalidate it before sessions.delete is submitted. + await Promise.resolve(); + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + if (cleanup === "delete" && childSessionEffectsAllowed()) { + if (!cleanupSessionIdentity) { + // Without both lifecycle identities, key-only deletion could remove + // a successor that reused this child session after cleanup yielded. + suppressChildSessionEffects(); + } else { + // This durable boundary prevents a late yield from reviving a run + // after deletion may already have reached the gateway. + entry.deleteCleanupDispatchedAt ??= Date.now(); + params.persist(runId); + const sessionCleanup = await deleteSubagentSessionForCleanup({ + callGateway: params.callGateway, + childSessionKey: entry.childSessionKey, + spawnMode: entry.spawnMode, + expectedSessionId: cleanupSessionIdentity.sessionId, + expectedLifecycleRevision: cleanupSessionIdentity.lifecycleRevision, + onError: (error) => + params.warn("sessions.delete failed during subagent cleanup", { + error: buildSafeLifecycleErrorMeta(error), + runId: maskLifecycleIdentifier(runId, "run"), + childSessionKey: maskLifecycleIdentifier(entry.childSessionKey, "session"), + }), + }); + if (sessionCleanup === "failed") { + throw new Error("subagent session cleanup did not complete"); + } + if (sessionCleanup === "changed") { + suppressChildSessionEffects(); + } + } + } + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + await finalizeSubagentCleanup(context, runId, cleanup, "delivered", cleanupGeneration, { + skipAnnounce: true, + skipDeliveryStatus: true, + skipRequesterDelivery, + }); + }, + }); + return true; + } + const pendingPayload = loadPendingFinalDeliveryPayload(entry); + const requesterOrigin = normalizeDeliveryContext(pendingPayload.requesterOrigin); + let latestDeliveryError = getDeliveryLastError(entry); + const finalizeAnnounceCleanup = async (announceOutcome: SubagentAnnounceFlowOutcome) => { + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + const shouldCreditPriorDelivery = + announceOutcome !== "delivered" && (await hasPriorRequesterDeliveryMirror(params, entry)); + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration); + return; + } + if (shouldCreditPriorDelivery) { + latestDeliveryError = undefined; + } + if (announceOutcome !== "delivered" && latestDeliveryError) { + ensureDeliveryState(entry).lastError = latestDeliveryError; + } + await finalizeSubagentCleanup( + context, + runId, + cleanup, + shouldCreditPriorDelivery ? "delivered" : announceOutcome, + cleanupGeneration, + ); + }; + + const announceParams: Parameters[0] = { + childSessionKey: pendingPayload.childSessionKey, + childRunId: pendingPayload.childRunId, + requesterSessionKey: pendingPayload.requesterSessionKey, + requesterOrigin, + requesterDisplayKey: pendingPayload.requesterDisplayKey, + task: pendingPayload.task, + timeoutMs: params.subagentAnnounceTimeoutMs, + cleanup: suppressSessionEffects ? "keep" : cleanup, + roundOneReply: entry.completion?.resultText ?? undefined, + terminalReply: pendingPayload.terminalReply, + fallbackReply: entry.completion?.fallbackResultText ?? undefined, + waitForCompletion: false, + startedAt: pendingPayload.startedAt, + endedAt: pendingPayload.endedAt, + label: pendingPayload.label, + outcome: pendingPayload.outcome, + spawnMode: pendingPayload.spawnMode, + expectsCompletionMessage: pendingPayload.expectsCompletionMessage, + wakeOnDescendantSettle: pendingPayload.wakeOnDescendantSettle === true, + suppressChildSessionEffects: suppressSessionEffects, + isChildSessionEffectsAllowed: childSessionEffectsAllowed, + isCompletionDeliveryAllowed: () => + context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration), + isCompletionOwnedByRequesterYield: () => + entry.requesterTurnYielded === true || + entry.requesterSettleWake?.requesterYieldBatch === true, + onBeforeDeleteChildSession: + cleanup === "delete" + ? () => { + if (!childSessionEffectsAllowed()) { + return false; + } + // Announce owns delete submission; fence late yields at the + // exact handoff instead of when cleanup merely starts. + entry.deleteCleanupDispatchedAt ??= Date.now(); + params.persist(runId); + return true; + } + : undefined, + onDeliveryResult: (delivery) => { + if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + retireSupersededCleanupInBackground(context, runId, entry, cleanupGeneration); + return; + } + recordAnnounceDeliveryResult(entry, delivery); + if (delivery.delivered) { + const deliveryState = ensureDeliveryState(entry); + deliveryState.status = "delivered"; + deliveryState.announcedAt = deliveryState.deliveredAt ?? Date.now(); + deliveryState.lastError = undefined; + deliveryState.suspendedAt = undefined; + deliveryState.suspendedReason = undefined; + // Identified platform delivery precedes best-effort transcript + // mirroring; task ownership must become durable at that same edge. + params.persist(runId); + safeSetSubagentTaskDeliveryStatus(params, { + entry, + deliveryStatus: "delivered", + }); + latestDeliveryError = undefined; + return; + } + if (delivery.path === "none" && delivery.disposition !== "intentional_non_delivery") { + ensureDeliveryState(entry).lastDropReason = "sink_unavailable"; + } + latestDeliveryError = formatAnnounceDeliveryError(delivery); + if (ensureDeliveryState(entry).lastError !== latestDeliveryError) { + ensureDeliveryState(entry).lastError = latestDeliveryError; + params.persist(runId); + } + }, + }; + runDetachedCleanupAttempt(context, { + runId, + entry, + cleanupGeneration, + run: async () => { + let announceOutcome: SubagentAnnounceFlowOutcome = "retryable"; + try { + announceOutcome = await params.runSubagentAnnounceFlow(announceParams); + } catch (error) { + defaultRuntime.log( + `[warn] Subagent announce flow failed during cleanup for run ${runId}: ${String(error)}`, + ); + } + await finalizeAnnounceCleanup(announceOutcome); + }, + }); + return true; +}; diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-bookkeeping.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-bookkeeping.ts deleted file mode 100644 index 46910246a11c..000000000000 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-bookkeeping.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { runWithGatewayIndependentRootWorkAdmission } from "../../../process/gateway-work-admission.js"; -import { defaultRuntime } from "../../../runtime.js"; -import { retireSessionMcpRuntimeForSessionKey } from "../../agent-bundle-mcp-tools.js"; -import { removeInternalSessionEffectsSession } from "../../internal-session-effects.js"; -import { SUBAGENT_ENDED_REASON_KILLED } from "./subagent-lifecycle-events.js"; -import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; -import type { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; -import type { SubagentRegistryLifecycleParams } from "./subagent-registry-lifecycle-contracts.js"; -import type { createSubagentRegistryLifecycleRequesterWake } from "./subagent-registry-lifecycle-requester-wake.js"; -import type { SubagentRunRecord } from "./subagent-registry.types.js"; - -export function createSubagentRegistryLifecycleBookkeeping( - params: SubagentRegistryLifecycleParams, - common: ReturnType, - requesterWake: ReturnType, - retryDeferredCompletedAnnounces: (excludeRunId?: string) => void, -) { - const { buildSafeLifecycleErrorMeta, maskRunId, maskSessionKey, newerGenerationOwnsSession } = - common; - const { persistRequesterSettleWakePending, scheduleRequesterSettleWake } = requesterWake; - - const completeCleanupBookkeeping = (cleanupParams: { - runId: string; - entry: SubagentRunRecord; - cleanup: "delete" | "keep"; - completedAt: number; - preserveTranscript?: boolean; - provisionalKill?: boolean; - // Set by the suspended-delivery discard path: the settle wake already ran - // when the delivery was suspended, so a discard hours later must not - // re-evaluate the requester drain. - skipRequesterSettleWake?: boolean; - }) => { - const suppressSessionEffects = shouldSuppressSubagentRecoverySessionEffects( - cleanupParams.entry, - ); - const runCleanupTail = (label: string, run: () => Promise) => { - // These best-effort tails can outlive the durable registry transition, - // but they still mutate session-owned resources and must block snapshots. - void runWithGatewayIndependentRootWorkAdmission(run).catch((error: unknown) => { - defaultRuntime.log( - `[warn] subagent ${label} failed (${cleanupParams.runId}): ${String(error)}`, - ); - }); - }; - const scheduleCleanupTails = (options: { - allowRetiredRow: boolean; - isDeleteCleanup: boolean; - }) => { - // Retained bookkeeping requires the exact row. Immediate retirement - // removes it first, so absence remains ownership only while no newer - // child generation exists; any replacement blocks the stale cleanup. - const postBookkeepingEffectsAllowed = () => { - const current = params.runs.get(cleanupParams.runId); - const rowOwnershipMatches = - current === cleanupParams.entry || (options.allowRetiredRow && current === undefined); - return ( - rowOwnershipMatches && - !newerGenerationOwnsSession(cleanupParams.entry) && - !shouldSuppressSubagentRecoverySessionEffects(cleanupParams.entry) - ); - }; - if (postBookkeepingEffectsAllowed() && !cleanupParams.preserveTranscript) { - runCleanupTail("session cleanup", async () => { - if (!postBookkeepingEffectsAllowed()) { - return; - } - await removeInternalSessionEffectsSession(cleanupParams.entry.execution.transcriptTarget); - }); - } - if (postBookkeepingEffectsAllowed() && cleanupParams.entry.spawnMode !== "session") { - runCleanupTail("bundle MCP cleanup", async () => { - if (!postBookkeepingEffectsAllowed()) { - return; - } - await retireSessionMcpRuntimeForSessionKey({ - sessionKey: cleanupParams.entry.childSessionKey, - reason: "subagent-run-cleanup", - preserveActiveLeases: true, - onError: (error, sessionId) => { - params.warn("failed to retire subagent bundle MCP runtime", { - error: buildSafeLifecycleErrorMeta(error), - sessionId, - runId: maskRunId(cleanupParams.runId), - childSessionKey: maskSessionKey(cleanupParams.entry.childSessionKey), - }); - }, - }); - }); - } - if ( - !cleanupParams.provisionalKill && - postBookkeepingEffectsAllowed() && - (options.isDeleteCleanup || !cleanupParams.entry.collect) - ) { - runCleanupTail("context-engine cleanup", async () => { - if (!postBookkeepingEffectsAllowed()) { - return; - } - await params.notifyContextEngineSubagentEnded( - { - childSessionKey: cleanupParams.entry.childSessionKey, - reason: options.isDeleteCleanup ? "deleted" : "completed", - agentDir: cleanupParams.entry.agentDir, - workspaceDir: cleanupParams.entry.workspaceDir, - }, - { isCurrent: postBookkeepingEffectsAllowed }, - ); - }); - } - }; - if (cleanupParams.provisionalKill) { - // The provider result or bounded kill reconciliation owns terminal settle. - // Its kill marker was committed by the caller before reaching this tail. - scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup: false }); - return; - } - const isDeleteCleanup = cleanupParams.cleanup === "delete"; - if (isDeleteCleanup) { - params.clearPendingLifecycleError(cleanupParams.runId); - } - if (cleanupParams.entry.collect) { - // Delete-mode session cleanup already ran before this durable bookkeeping. - // Preserve only the collector result tombstone for waits and group caps. - const previousCleanupCompletedAt = cleanupParams.entry.cleanupCompletedAt; - const previousExecution = cleanupParams.entry.execution; - const previousRequesterSettleWake = cleanupParams.entry.requesterSettleWake; - const previousTerminalOwner = cleanupParams.entry.terminalOwner; - cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; - cleanupParams.entry.requesterSettleWake = undefined; - if (suppressSessionEffects) { - cleanupParams.entry.execution = { - ...cleanupParams.entry.execution, - restartRecovery: undefined, - suppressSessionEffects: true, - }; - cleanupParams.entry.terminalOwner = undefined; - } - try { - params.persistOrThrow(cleanupParams.runId); - } catch (error) { - cleanupParams.entry.cleanupCompletedAt = previousCleanupCompletedAt; - cleanupParams.entry.execution = previousExecution; - cleanupParams.entry.requesterSettleWake = previousRequesterSettleWake; - cleanupParams.entry.terminalOwner = previousTerminalOwner; - throw error; - } - scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup }); - retryDeferredCompletedAnnounces(cleanupParams.runId); - return; - } - const retireAfterSettle = - isDeleteCleanup || - (cleanupParams.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && - cleanupParams.entry.suppressAnnounceReason !== "killed"); - if (retireAfterSettle) { - // Reconciled keep-mode kills retire the registry row, not the child session. - if (!isDeleteCleanup) { - params.clearPendingLifecycleError(cleanupParams.runId); - } - if (cleanupParams.skipRequesterSettleWake) { - params.runs.delete(cleanupParams.runId); - try { - params.persistOrThrow(cleanupParams.runId); - } catch (error) { - params.runs.set(cleanupParams.runId, cleanupParams.entry); - throw error; - } - scheduleCleanupTails({ allowRetiredRow: true, isDeleteCleanup }); - retryDeferredCompletedAnnounces(cleanupParams.runId); - return; - } - persistRequesterSettleWakePending(cleanupParams.entry, { - cleanupCompletedAt: cleanupParams.completedAt, - retireAfterSettle: true, - retireInterruptedRecovery: suppressSessionEffects, - }); - // The settle wake may synchronously retire this durably marked row before - // the detached tails start. Absence is still stale-safe because any - // replacement row or newer child generation rejects the cleanup. - scheduleCleanupTails({ allowRetiredRow: true, isDeleteCleanup }); - retryDeferredCompletedAnnounces(cleanupParams.runId); - scheduleRequesterSettleWake(cleanupParams.runId, cleanupParams.entry); - return; - } - if (!cleanupParams.skipRequesterSettleWake) { - persistRequesterSettleWakePending(cleanupParams.entry, { - cleanupCompletedAt: cleanupParams.completedAt, - retireInterruptedRecovery: suppressSessionEffects, - }); - } else { - const previousCleanupCompletedAt = cleanupParams.entry.cleanupCompletedAt; - const previousExecution = cleanupParams.entry.execution; - const previousTerminalOwner = cleanupParams.entry.terminalOwner; - cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; - if (suppressSessionEffects) { - cleanupParams.entry.execution = { - ...cleanupParams.entry.execution, - restartRecovery: undefined, - suppressSessionEffects: true, - }; - cleanupParams.entry.terminalOwner = undefined; - } - try { - params.persistOrThrow(cleanupParams.runId); - } catch (error) { - cleanupParams.entry.cleanupCompletedAt = previousCleanupCompletedAt; - cleanupParams.entry.execution = previousExecution; - cleanupParams.entry.terminalOwner = previousTerminalOwner; - throw error; - } - } - scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup }); - retryDeferredCompletedAnnounces(cleanupParams.runId); - if (!cleanupParams.skipRequesterSettleWake) { - scheduleRequesterSettleWake(cleanupParams.runId, cleanupParams.entry); - } - }; - return { completeCleanupBookkeeping }; -} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-cleanup-base.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-cleanup-base.ts deleted file mode 100644 index b4b4a2070403..000000000000 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-cleanup-base.ts +++ /dev/null @@ -1,313 +0,0 @@ -import { runWithoutOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js"; -import { - isGatewayRestartDraining, - runWithGatewayIndependentRootWorkAdmission, -} from "../../../process/gateway-work-admission.js"; -import { defaultRuntime } from "../../../runtime.js"; -import { retireSessionMcpRuntimeForSessionKey } from "../../agent-bundle-mcp-tools.js"; -import { - ensureCompletionState, - ensureDeliveryState, - getDeliveryLastError, -} from "./subagent-delivery-state.js"; -import { - logAnnounceGiveUp, - MIN_ANNOUNCE_RETRY_DELAY_MS, - resolveAnnounceRetryDelayMs, -} from "./subagent-registry-helpers.js"; -import type { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; -import type { - SubagentRegistryLifecycleParams, - SubagentRegistryLifecycleState, -} from "./subagent-registry-lifecycle-contracts.js"; -import type { createSubagentRegistryLifecycleDelivery } from "./subagent-registry-lifecycle-delivery.js"; -import type { createSubagentRegistryLifecycleRequesterWake } from "./subagent-registry-lifecycle-requester-wake.js"; -import type { SubagentRunRecord } from "./subagent-registry.types.js"; - -const MAX_DETACHED_CLEANUP_RETRIES = 3; - -export function createSubagentRegistryLifecycleCleanupBase( - params: SubagentRegistryLifecycleParams, - state: SubagentRegistryLifecycleState, - common: ReturnType, - deliveryHelpers: ReturnType, - requesterWake: ReturnType, -) { - const { scheduledResumeTimers, cleanupGenerations, terminalGenerations } = state; - // Exhaustion intentionally leaves the durable row unlocked; only a process - // restart gets a fresh retry budget after this process stops scheduling it. - const cleanupFailureCounts = new WeakMap(); - const { buildSafeLifecycleErrorMeta, maskRunId, maskSessionKey, newerGenerationOwnsSession } = - common; - const { - markPendingFinalDelivery, - safeMarkRequiredCompletionDeliveryBlocked, - safeSetSubagentTaskDeliveryStatus, - } = deliveryHelpers; - const { markRequesterSettleWakePending, scheduleRequesterSettleWake } = requesterWake; - - const isCleanupGenerationCurrent = ( - runId: string, - entry: SubagentRunRecord, - generation: number, - ): boolean => - params.runs.get(runId) === entry && - entry.pauseReason !== "sessions_yield" && - cleanupGenerations.get(entry) === generation && - !newerGenerationOwnsSession(entry); - - const scheduleResumeSubagentRun = ( - runId: string, - entry: SubagentRunRecord, - delayMs: number, - cleanupGeneration?: number, - ) => { - const timer = setTimeout(() => { - scheduledResumeTimers.delete(timer); - void runWithGatewayIndependentRootWorkAdmission(async () => { - if (params.runs.get(runId) !== entry) { - return; - } - if (cleanupGeneration !== undefined) { - if (!isCleanupGenerationCurrent(runId, entry, cleanupGeneration)) { - return; - } - if (entry.cleanupHandled) { - entry.cleanupHandled = false; - params.persist(runId); - } - } - params.resumedRuns.delete(runId); - params.resumeSubagentRun(runId); - }).catch((err: unknown) => { - defaultRuntime.log(`[warn] subagent cleanup resume failed (${runId}): ${String(err)}`); - const current = params.runs.get(runId); - if ( - isGatewayRestartDraining() && - current === entry && - typeof current.cleanupCompletedAt !== "number" - ) { - scheduleResumeSubagentRun( - runId, - entry, - Math.max(delayMs, MIN_ANNOUNCE_RETRY_DELAY_MS), - cleanupGeneration, - ); - } - }); - }, delayMs); - timer.unref?.(); - scheduledResumeTimers.add(timer); - }; - - const runDetachedCleanupAttempt = (args: { - runId: string; - entry: SubagentRunRecord; - cleanupGeneration: number; - run: () => Promise; - }) => { - // Completion makes the task projection non-blocking before delivery and - // cleanup finish. This independent lease bridges that handoff and owns the - // full detached attempt, including its final durable registry write. - // Completion outlives the spawning attempt; inherited lock owners would - // reject requester transcript writes after that attempt is disposed. - runWithoutOwnedSessionTranscriptWrites(() => { - void runWithGatewayIndependentRootWorkAdmission(async () => { - try { - await args.run(); - cleanupFailureCounts.delete(args.entry); - } catch (err) { - defaultRuntime.log( - `[warn] subagent cleanup finalize failed (${args.runId}): ${String(err)}`, - ); - const current = params.runs.get(args.runId); - if ( - !current || - current.cleanupCompletedAt || - !isCleanupAttemptCurrent(args.runId, args.entry, args.cleanupGeneration) - ) { - return; - } - current.cleanupHandled = false; - params.resumedRuns.delete(args.runId); - params.persist(args.runId); - const failureCount = (cleanupFailureCounts.get(current) ?? 0) + 1; - cleanupFailureCounts.set(current, failureCount); - if (failureCount <= MAX_DETACHED_CLEANUP_RETRIES) { - scheduleResumeSubagentRun( - args.runId, - current, - resolveAnnounceRetryDelayMs(failureCount), - args.cleanupGeneration, - ); - } - } - }).catch((err: unknown) => { - defaultRuntime.log( - `[warn] subagent cleanup admission failed (${args.runId}): ${String(err)}`, - ); - if (isGatewayRestartDraining()) { - scheduleResumeSubagentRun( - args.runId, - args.entry, - MIN_ANNOUNCE_RETRY_DELAY_MS, - args.cleanupGeneration, - ); - } - }); - }); - }; - - const suspendPendingFinalDelivery = (args: { - runId: string; - entry: SubagentRunRecord; - reason: "expiry" | "permanent_failure"; - error?: string; - }) => { - const previousEntry = structuredClone(args.entry); - markPendingFinalDelivery({ - entry: args.entry, - error: args.error ?? getDeliveryLastError(args.entry) ?? args.reason, - }); - const now = Date.now(); - const delivery = ensureDeliveryState(args.entry); - delivery.status = "suspended"; - delivery.suspendedAt ??= now; - delivery.suspendedReason = args.reason; - args.entry.cleanupHandled = false; - args.entry.wakeOnDescendantSettle = undefined; - const completion = ensureCompletionState(args.entry); - completion.fallbackResultText = undefined; - completion.fallbackCapturedAt = undefined; - params.resumedRuns.delete(args.runId); - safeSetSubagentTaskDeliveryStatus({ - entry: args.entry, - deliveryStatus: "failed", - deliveryError: getDeliveryLastError(args.entry) ?? args.reason, - }); - safeMarkRequiredCompletionDeliveryBlocked({ - entry: args.entry, - reason: getDeliveryLastError(args.entry) ?? args.reason, - }); - logAnnounceGiveUp(args.entry, args.reason); - markRequesterSettleWakePending(args.entry); - try { - params.persistOrThrow(args.runId); - } catch (error) { - const mutableEntry = args.entry as unknown as Record; - for (const key of Object.keys(mutableEntry)) { - delete mutableEntry[key]; - } - Object.assign(args.entry, previousEntry); - throw error; - } - // Suspension is terminal for automatic retries, so it settles this child - // for requester-drain purposes even though cleanup stays incomplete. - scheduleRequesterSettleWake(args.runId, args.entry); - }; - - const beginSubagentCleanup = (runId: string) => { - const entry = params.runs.get(runId); - if (!entry) { - return false; - } - if (entry.cleanupCompletedAt || entry.cleanupHandled) { - return false; - } - entry.cleanupHandled = true; - cleanupGenerations.set(entry, (cleanupGenerations.get(entry) ?? 0) + 1); - params.persist(runId); - return true; - }; - - const isCleanupAttemptCurrent = ( - runId: string, - entry: SubagentRunRecord, - generation: number, - ): boolean => - entry.cleanupHandled === true && isCleanupGenerationCurrent(runId, entry, generation); - - const retireSupersededCleanupIfNeeded = async ( - runId: string, - entry: SubagentRunRecord, - generation: number, - ): Promise => { - if ( - params.runs.get(runId) !== entry || - cleanupGenerations.get(entry) !== generation || - !newerGenerationOwnsSession(entry) - ) { - return false; - } - // Cleanup can yield to attachment, mirror, or announce work. A successor - // registered while it was suspended owns every session-scoped side effect. - await params.retireSupersededRun(runId, entry); - return true; - }; - - const retireSupersededCleanupInBackground = ( - runId: string, - entry: SubagentRunRecord, - generation: number, - ) => { - // Delivery callbacks are synchronous and may arrive after their announce - // attempt returns. Give the async retirement tail its own snapshot blocker. - void runWithGatewayIndependentRootWorkAdmission(async () => { - await retireSupersededCleanupIfNeeded(runId, entry, generation); - }).catch((error: unknown) => { - defaultRuntime.log( - `[warn] subagent superseded cleanup retirement failed (${runId}): ${String(error)}`, - ); - }); - }; - - const isTerminalCallbackCurrent = ( - runId: string, - entry: SubagentRunRecord, - generation: number, - ): boolean => - params.runs.get(runId) === entry && - entry.pauseReason !== "sessions_yield" && - terminalGenerations.get(entry) === generation; - - const isEndedHookOwnerCurrent = (runId: string, entry: SubagentRunRecord): boolean => { - const current = params.runs.get(runId); - return (current === undefined || current === entry) && !newerGenerationOwnsSession(entry); - }; - - const retireRunModeBundleMcpRuntime = async (cleanupParams: { - runId: string; - entry: SubagentRunRecord; - reason: string; - }) => { - if (cleanupParams.entry.spawnMode === "session") { - return; - } - await retireSessionMcpRuntimeForSessionKey({ - sessionKey: cleanupParams.entry.childSessionKey, - reason: cleanupParams.reason, - preserveActiveLeases: true, - onError: (error, sessionId) => { - params.warn("failed to retire subagent bundle MCP runtime", { - error: buildSafeLifecycleErrorMeta(error), - sessionId, - runId: maskRunId(cleanupParams.runId), - childSessionKey: maskSessionKey(cleanupParams.entry.childSessionKey), - }); - }, - }); - }; - - return { - beginSubagentCleanup, - isCleanupAttemptCurrent, - isEndedHookOwnerCurrent, - isTerminalCallbackCurrent, - retireSupersededCleanupIfNeeded, - retireRunModeBundleMcpRuntime, - retireSupersededCleanupInBackground, - runDetachedCleanupAttempt, - scheduleResumeSubagentRun, - suspendPendingFinalDelivery, - }; -} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-cleanup.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-cleanup.ts index 7b26b87387e3..d59b867dab5b 100644 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-cleanup.ts +++ b/src/agents/subagents/registry/subagent-registry-lifecycle-cleanup.ts @@ -1,633 +1,621 @@ +import type { cleanupBrowserSessionsForLifecycleEnd } from "../../../browser-lifecycle-cleanup.js"; +import { runWithoutOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js"; +import { + isGatewayRestartDraining, + runWithGatewayIndependentRootWorkAdmission, +} from "../../../process/gateway-work-admission.js"; import { defaultRuntime } from "../../../runtime.js"; -import { normalizeDeliveryContext } from "../../../utils/delivery-context.shared.js"; +import { emitSessionLifecycleEvent } from "../../../sessions/session-lifecycle-events.js"; +import { recordSubagentTerminalState } from "../../../sessions/session-state-events.js"; +import { retireSessionMcpRuntimeForSessionKey } from "../../agent-bundle-mcp-tools.js"; +import { releaseSwarmRun } from "../swarm/swarm-scheduler.js"; import { ensureCompletionState, ensureDeliveryState, getDeliveryLastError, - isDeliverySuspended, } from "./subagent-delivery-state.js"; -import { SUBAGENT_ENDED_REASON_COMPLETE } from "./subagent-lifecycle-events.js"; +import { + SUBAGENT_ENDED_REASON_KILLED, + type SubagentLifecycleEndedReason, +} from "./subagent-lifecycle-events.js"; import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; import { - resolveCleanupCompletionReason, - resolveDeferredCleanupDecision, -} from "./subagent-registry-cleanup.js"; -import { - ANNOUNCE_COMPLETION_HARD_EXPIRY_MS, - ANNOUNCE_EXPIRY_MS, logAnnounceGiveUp, MIN_ANNOUNCE_RETRY_DELAY_MS, + persistSubagentSessionTiming, resolveAnnounceRetryDelayMs, - safeRemoveAttachmentsDir, } from "./subagent-registry-helpers.js"; -import { createSubagentRegistryLifecycleBookkeeping } from "./subagent-registry-lifecycle-bookkeeping.js"; -import type { createSubagentRegistryLifecycleCleanupBase } from "./subagent-registry-lifecycle-cleanup-base.js"; -import type { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; import type { - SubagentRegistryLifecycleParams, - SubagentRegistryLifecycleState, -} from "./subagent-registry-lifecycle-contracts.js"; -import type { createSubagentRegistryLifecycleDelivery } from "./subagent-registry-lifecycle-delivery.js"; -import type { createSubagentRegistryLifecycleRequesterWake } from "./subagent-registry-lifecycle-requester-wake.js"; -import { loadSubagentSessionEntry } from "./subagent-session-reconciliation.js"; + SubagentLifecycleCommonContext, + SubagentLifecycleCompletionContext, + SubagentLifecycleCleanupContext, + SubagentLifecycleWakeContext, +} from "./subagent-registry-lifecycle-context.js"; +import { + buildSafeLifecycleErrorMeta, + markPendingFinalDelivery, + maskLifecycleIdentifier, + safeMarkRequiredCompletionDeliveryBlocked, + safeSetSubagentTaskDeliveryStatus, +} from "./subagent-registry-lifecycle-delivery.js"; +import { + markRequesterSettleWakePending, + scheduleRequesterSettleWake, +} from "./subagent-registry-lifecycle-wake.js"; +import type { SubagentCompletionRequest, SubagentRunRecord } from "./subagent-registry.types.js"; -type RunSubagentAnnounceFlow = - (typeof import("../announce/subagent-announce.js"))["runSubagentAnnounceFlow"]; -type SubagentAnnounceFlowOutcome = Awaited>; -import type { SubagentRunRecord } from "./subagent-registry.types.js"; -import { deleteSubagentSessionForCleanup } from "./subagent-session-cleanup.js"; +const MAX_DETACHED_CLEANUP_RETRIES = 3; +type BrowserCleanup = typeof cleanupBrowserSessionsForLifecycleEnd; -export function createSubagentRegistryLifecycleCleanup( - params: SubagentRegistryLifecycleParams, - state: SubagentRegistryLifecycleState, - common: ReturnType, - deliveryHelpers: ReturnType, - requesterWake: ReturnType, - cleanupBase: ReturnType, -) { - const { cleanupGenerations } = state; - const { buildSafeLifecycleErrorMeta, maskRunId, maskSessionKey } = common; - const { - clearPendingFinalDelivery, - emitCompletionEndedHookIfNeeded, - formatAnnounceDeliveryError, - hasPriorRequesterDeliveryMirror, - loadPendingFinalDeliveryPayload, - markPendingFinalDelivery, - recordAnnounceDeliveryResult, - safeMarkRequiredCompletionDeliveryBlocked, - safeSetSubagentTaskDeliveryStatus, - } = deliveryHelpers; - const { - beginSubagentCleanup, - isCleanupAttemptCurrent, - isEndedHookOwnerCurrent, - retireRunModeBundleMcpRuntime, - retireSupersededCleanupIfNeeded, - retireSupersededCleanupInBackground, - runDetachedCleanupAttempt, - scheduleResumeSubagentRun, - suspendPendingFinalDelivery, - } = cleanupBase; - - const shouldSuspendPendingFinalDelivery = (entry: SubagentRunRecord) => - entry.expectsCompletionMessage === true && - entry.endedReason === SUBAGENT_ENDED_REASON_COMPLETE && - entry.execution.outcome?.status === "ok"; - - const finalizeAnnounceGiveUp = async (giveUpParams: { - runId: string; - entry: SubagentRunRecord; - reason: "expiry" | "permanent_failure"; - cleanup?: "delete" | "keep"; - cleanupGeneration?: number; - retryCount?: number; - completedAt?: number; - }) => { - const { runId, entry, reason, cleanup, cleanupGeneration, retryCount, completedAt } = - giveUpParams; - if (shouldSuspendPendingFinalDelivery(entry)) { - suspendPendingFinalDelivery({ - runId, - entry, - reason, - error: getDeliveryLastError(entry), - }); - return; - } - const deliveryError = getDeliveryLastError(entry) ?? reason; - clearPendingFinalDelivery(entry); - const failedDelivery = ensureDeliveryState(entry); - failedDelivery.status = "failed"; - failedDelivery.lastError = deliveryError; - if (retryCount != null) { - failedDelivery.attemptCount = retryCount; - failedDelivery.lastAttemptAt = completedAt ?? Date.now(); - } - safeSetSubagentTaskDeliveryStatus({ - entry, - deliveryStatus: "failed", - deliveryError, - }); - safeMarkRequiredCompletionDeliveryBlocked({ - entry, - reason: deliveryError, - }); - entry.wakeOnDescendantSettle = undefined; - const completion = ensureCompletionState(entry); - completion.fallbackResultText = undefined; - completion.fallbackCapturedAt = undefined; - if ((cleanup ?? entry.cleanup) === "delete" || !entry.retainAttachmentsOnKeep) { - await safeRemoveAttachmentsDir(entry); - } - if ( - cleanupGeneration !== undefined && - !isCleanupAttemptCurrent(runId, entry, cleanupGeneration) - ) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - const completionReason = resolveCleanupCompletionReason(entry); - logAnnounceGiveUp(entry, reason); - // Retry-limit / expiry give-up should not leave cleanup stuck behind the - // best-effort ended hook. Mark the run cleaned first, then fire the hook. - completeCleanupBookkeeping({ - runId, - entry, - cleanup: cleanup ?? entry.cleanup, - completedAt: completedAt ?? Date.now(), - }); - if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { - await emitCompletionEndedHookIfNeeded( - entry, - completionReason, - () => - isEndedHookOwnerCurrent(runId, entry) && - !shouldSuppressSubagentRecoverySessionEffects(entry), - ); - } - }; - - const retryDeferredCompletedAnnounces = (excludeRunId?: string) => { - const now = Date.now(); - for (const [runId, entry] of params.runs.entries()) { - if (excludeRunId && runId === excludeRunId) { - continue; +export function scheduleResumeSubagentRun( + context: SubagentLifecycleCleanupContext, + runId: string, + entry: SubagentRunRecord, + delayMs: number, + cleanupGeneration?: number, +): void { + const params = context.options; + const timer = setTimeout(() => { + context.deleteScheduledResumeTimer(timer); + void runWithGatewayIndependentRootWorkAdmission(async () => { + if (params.runs.get(runId) !== entry) { + return; } - if (typeof entry.execution.endedAt !== "number") { - continue; - } - if (entry.cleanupCompletedAt || entry.cleanupHandled) { - continue; - } - if (isDeliverySuspended(entry)) { - continue; - } - if (params.suppressAnnounceForSteerRestart(entry)) { - continue; - } - const endedAgo = now - (entry.execution.endedAt ?? now); - if (entry.expectsCompletionMessage !== true && endedAgo > ANNOUNCE_EXPIRY_MS) { - if (!beginSubagentCleanup(runId)) { - continue; + if (cleanupGeneration !== undefined) { + if (!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration)) { + return; + } + if (entry.cleanupHandled) { + entry.cleanupHandled = false; + params.persist(runId); } - runDetachedCleanupAttempt({ - runId, - entry, - cleanupGeneration: cleanupGenerations.get(entry)!, - run: async () => { - await finalizeAnnounceGiveUp({ - runId, - entry, - reason: "expiry", - }); - }, - }); - continue; } params.resumedRuns.delete(runId); params.resumeSubagentRun(runId); - } - }; - - const { completeCleanupBookkeeping } = createSubagentRegistryLifecycleBookkeeping( - params, - common, - requesterWake, - retryDeferredCompletedAnnounces, - ); - - const finalizeSubagentCleanup = async ( - runId: string, - cleanup: "delete" | "keep", - announceOutcome: SubagentAnnounceFlowOutcome, - cleanupGeneration: number, - options?: { - skipAnnounce?: boolean; - skipDeliveryStatus?: boolean; - skipRequesterDelivery?: boolean; - }, - ) => { - const entry = params.runs.get(runId); - if (!entry) { - return; - } - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - if (entry.expectsCompletionMessage === false || options?.skipRequesterDelivery) { - clearPendingFinalDelivery(entry); - if (options?.skipRequesterDelivery) { - ensureDeliveryState(entry).status = "not_required"; - entry.suppressCompletionDelivery = undefined; - } - entry.wakeOnDescendantSettle = undefined; - const shouldDeleteAttachments = cleanup === "delete" || !entry.retainAttachmentsOnKeep; - if (shouldDeleteAttachments) { - await safeRemoveAttachmentsDir(entry); - } - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - completeCleanupBookkeeping({ - runId, - entry, - cleanup, - completedAt: Date.now(), - }); - if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { - await emitCompletionEndedHookIfNeeded( + }).catch((err: unknown) => { + defaultRuntime.log(`[warn] subagent cleanup resume failed (${runId}): ${String(err)}`); + const current = params.runs.get(runId); + if ( + isGatewayRestartDraining() && + current === entry && + typeof current.cleanupCompletedAt !== "number" + ) { + scheduleResumeSubagentRun( + context, + runId, entry, - resolveCleanupCompletionReason(entry), - () => - isEndedHookOwnerCurrent(runId, entry) && - !shouldSuppressSubagentRecoverySessionEffects(entry), + Math.max(delayMs, MIN_ANNOUNCE_RETRY_DELAY_MS), + cleanupGeneration, ); } - return; - } - if (announceOutcome === "delivered" || announceOutcome === "intentional_non_delivery") { - const delivery = ensureDeliveryState(entry); - const shouldCreditDelivery = - announceOutcome === "delivered" && - (!options?.skipAnnounce || - delivery.status === "delivered" || - typeof delivery.announcedAt === "number"); - if (shouldCreditDelivery) { - const deliveredAt = delivery.deliveredAt ?? delivery.announcedAt ?? Date.now(); - delivery.status = "delivered"; - delivery.deliveredAt = deliveredAt; - delivery.announcedAt = delivery.announcedAt ?? deliveredAt; - if (!options?.skipAnnounce) { - delivery.announcedAt = deliveredAt; - params.persist(runId); - } - } - if (announceOutcome === "delivered") { - clearPendingFinalDelivery(entry); - } else { - // The requester-settle batch owns the real delivery now. Retire the - // per-child retry obligation without converting the handoff into success. - delivery.status = "pending"; - delivery.disposition = "intentional_non_delivery"; - delivery.payload = undefined; - delivery.createdAt = undefined; - delivery.attemptCount = undefined; - delivery.nextAttemptAt = undefined; - } - const finalDelivery = ensureDeliveryState(entry); - if (shouldCreditDelivery) { - finalDelivery.status = "delivered"; - finalDelivery.suspendedAt = undefined; - finalDelivery.suspendedReason = undefined; - } - if (shouldCreditDelivery && !options?.skipDeliveryStatus) { - safeSetSubagentTaskDeliveryStatus({ - entry, - deliveryStatus: "delivered", - }); - } else if (announceOutcome === "intentional_non_delivery" && !options?.skipDeliveryStatus) { - safeSetSubagentTaskDeliveryStatus({ entry, deliveryStatus: "pending" }); - } - if (announceOutcome === "delivered") { - finalDelivery.lastError = undefined; - finalDelivery.lastDropReason = undefined; - } - entry.wakeOnDescendantSettle = undefined; - const completion = ensureCompletionState(entry); - completion.fallbackResultText = undefined; - completion.fallbackCapturedAt = undefined; - const completionReason = resolveCleanupCompletionReason(entry); - const shouldDeleteAttachments = cleanup === "delete" || !entry.retainAttachmentsOnKeep; - if (shouldDeleteAttachments) { - await safeRemoveAttachmentsDir(entry); - } - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - completeCleanupBookkeeping({ - runId, - entry, - cleanup, - completedAt: Date.now(), - }); - // Hook loading is best-effort; durable delivery and cleanup must already - // be terminal before plugin code can fail or stall. - if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { - await emitCompletionEndedHookIfNeeded( - entry, - completionReason, - () => - isEndedHookOwnerCurrent(runId, entry) && - !shouldSuppressSubagentRecoverySessionEffects(entry), + }); + }, delayMs); + timer.unref?.(); + context.addScheduledResumeTimer(timer); +} + +export function runDetachedCleanupAttempt( + context: SubagentLifecycleCleanupContext, + args: { + runId: string; + entry: SubagentRunRecord; + cleanupGeneration: number; + run: () => Promise; + }, +): void { + const params = context.options; + // Completion makes the task projection non-blocking before delivery and + // cleanup finish. This independent lease bridges that handoff and owns the + // full detached attempt, including its final durable registry write. + // Completion outlives the spawning attempt; inherited lock owners would + // reject requester transcript writes after that attempt is disposed. + runWithoutOwnedSessionTranscriptWrites(() => { + void runWithGatewayIndependentRootWorkAdmission(async () => { + try { + await args.run(); + context.clearCleanupFailureCount(args.entry); + } catch (err) { + defaultRuntime.log( + `[warn] subagent cleanup finalize failed (${args.runId}): ${String(err)}`, ); - } - return; - } - - if (announceOutcome === "session_queued") { - // The correlated queue owns transport now. Settlement, not admission, - // decides delivered versus blocked and re-enters cleanup afterward. - entry.cleanupHandled = false; - params.resumedRuns.delete(runId); - params.persist(runId); - return; - } - - const now = Date.now(); - const deferredDecision = resolveDeferredCleanupDecision({ - entry, - now, - activeDescendantRuns: Math.max(0, params.countPendingDescendantRuns(entry.childSessionKey)), - announceExpiryMs: ANNOUNCE_EXPIRY_MS, - announceCompletionHardExpiryMs: ANNOUNCE_COMPLETION_HARD_EXPIRY_MS, - deferDescendantDelayMs: MIN_ANNOUNCE_RETRY_DELAY_MS, - resolveAnnounceRetryDelayMs, - }); - - if (deferredDecision.kind === "defer-descendants") { - ensureDeliveryState(entry).lastAttemptAt = now; - entry.wakeOnDescendantSettle = true; - entry.cleanupHandled = false; - params.resumedRuns.delete(runId); - params.persist(runId); - scheduleResumeSubagentRun(runId, entry, deferredDecision.delayMs); - return; - } - - if (deferredDecision.kind === "give-up") { - await finalizeAnnounceGiveUp({ - runId, - entry, - reason: deferredDecision.reason, - cleanup, - cleanupGeneration, - retryCount: deferredDecision.retryCount, - completedAt: now, - }); - return; - } - - markPendingFinalDelivery({ - entry, - error: "announce deferred or direct delivery failed", - }); - const delivery = ensureDeliveryState(entry); - delivery.windowStartedAt ??= entry.execution.endedAt ?? now; - delivery.deadlineAt ??= delivery.windowStartedAt + ANNOUNCE_COMPLETION_HARD_EXPIRY_MS; - delivery.nextAttemptAt = now + (deferredDecision.resumeDelayMs ?? 0); - entry.cleanupHandled = false; - params.resumedRuns.delete(runId); - params.persist(runId); - if (deferredDecision.resumeDelayMs == null) { - return; - } - scheduleResumeSubagentRun(runId, entry, deferredDecision.resumeDelayMs); - }; - - const startSubagentAnnounceCleanupFlow = (runId: string, entry: SubagentRunRecord): boolean => { - if (entry.killReconciliation) { - // Restores and unrelated cleanup retries must not publish a provisional - // kill. The sweeper re-enters here after durable reconciliation. - return false; - } - const cleanup = entry.cleanup; - let suppressSessionEffects = shouldSuppressSubagentRecoverySessionEffects(entry); - if (typeof entry.delivery?.announcedAt === "number" || entry.delivery?.status === "delivered") { - if (!beginSubagentCleanup(runId)) { - return false; - } - const cleanupGeneration = cleanupGenerations.get(entry)!; - runDetachedCleanupAttempt({ - runId, - entry, - cleanupGeneration, - run: async () => { - await finalizeSubagentCleanup(runId, cleanup, "delivered", cleanupGeneration, { - skipAnnounce: true, - }); - }, - }); - return true; - } - if (!beginSubagentCleanup(runId)) { - return false; - } - const cleanupGeneration = cleanupGenerations.get(entry)!; - const cleanupSessionEntry = suppressSessionEffects - ? undefined - : loadSubagentSessionEntry({ childSessionKey: entry.childSessionKey }); - const cleanupSessionIdentity = - cleanupSessionEntry?.sessionId && cleanupSessionEntry.lifecycleRevision - ? { - sessionId: cleanupSessionEntry.sessionId, - lifecycleRevision: cleanupSessionEntry.lifecycleRevision, - } - : undefined; - const suppressChildSessionEffects = () => { - suppressSessionEffects = true; - if (entry.execution.suppressSessionEffects !== true) { - const previousExecution = entry.execution; - entry.execution = { - ...entry.execution, - suppressSessionEffects: true, - }; - try { - params.persistOrThrow(runId); - } catch (error) { - entry.execution = previousExecution; - suppressSessionEffects = false; - throw error; - } - } - }; - const childSessionEffectsAllowed = () => { - if (!suppressSessionEffects && shouldSuppressSubagentRecoverySessionEffects(entry)) { - suppressChildSessionEffects(); - } - return !suppressSessionEffects && isCleanupAttemptCurrent(runId, entry, cleanupGeneration); - }; - const skipRequesterDelivery = entry.suppressCompletionDelivery === true; - if (entry.expectsCompletionMessage === false || skipRequesterDelivery) { - runDetachedCleanupAttempt({ - runId, - entry, - cleanupGeneration, - run: async () => { - // This driver is detached. Yield once so synchronous successor - // registration can invalidate it before sessions.delete is submitted. - await Promise.resolve(); - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - if (cleanup === "delete" && childSessionEffectsAllowed()) { - if (!cleanupSessionIdentity) { - // Without both lifecycle identities, key-only deletion could remove - // a successor that reused this child session after cleanup yielded. - suppressChildSessionEffects(); - } else { - // This durable boundary prevents a late yield from reviving a run - // after deletion may already have reached the gateway. - entry.deleteCleanupDispatchedAt ??= Date.now(); - params.persist(runId); - const sessionCleanup = await deleteSubagentSessionForCleanup({ - callGateway: params.callGateway, - childSessionKey: entry.childSessionKey, - spawnMode: entry.spawnMode, - expectedSessionId: cleanupSessionIdentity.sessionId, - expectedLifecycleRevision: cleanupSessionIdentity.lifecycleRevision, - onError: (error) => - params.warn("sessions.delete failed during subagent cleanup", { - error: buildSafeLifecycleErrorMeta(error), - runId: maskRunId(runId), - childSessionKey: maskSessionKey(entry.childSessionKey), - }), - }); - if (sessionCleanup === "failed") { - throw new Error("subagent session cleanup did not complete"); - } - if (sessionCleanup === "changed") { - suppressChildSessionEffects(); - } - } - } - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - await finalizeSubagentCleanup(runId, cleanup, "delivered", cleanupGeneration, { - skipAnnounce: true, - skipDeliveryStatus: true, - skipRequesterDelivery, - }); - }, - }); - return true; - } - const pendingPayload = loadPendingFinalDeliveryPayload(entry); - const requesterOrigin = normalizeDeliveryContext(pendingPayload.requesterOrigin); - let latestDeliveryError = getDeliveryLastError(entry); - const finalizeAnnounceCleanup = async (announceOutcome: SubagentAnnounceFlowOutcome) => { - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - const shouldCreditPriorDelivery = - announceOutcome !== "delivered" && (await hasPriorRequesterDeliveryMirror(entry)); - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); - return; - } - if (shouldCreditPriorDelivery) { - latestDeliveryError = undefined; - } - if (announceOutcome !== "delivered" && latestDeliveryError) { - ensureDeliveryState(entry).lastError = latestDeliveryError; - } - await finalizeSubagentCleanup( - runId, - cleanup, - shouldCreditPriorDelivery ? "delivered" : announceOutcome, - cleanupGeneration, - ); - }; - - const announceParams: Parameters[0] = { - childSessionKey: pendingPayload.childSessionKey, - childRunId: pendingPayload.childRunId, - requesterSessionKey: pendingPayload.requesterSessionKey, - requesterOrigin, - requesterDisplayKey: pendingPayload.requesterDisplayKey, - task: pendingPayload.task, - timeoutMs: params.subagentAnnounceTimeoutMs, - cleanup: suppressSessionEffects ? "keep" : cleanup, - roundOneReply: entry.completion?.resultText ?? undefined, - terminalReply: pendingPayload.terminalReply, - fallbackReply: entry.completion?.fallbackResultText ?? undefined, - waitForCompletion: false, - startedAt: pendingPayload.startedAt, - endedAt: pendingPayload.endedAt, - label: pendingPayload.label, - outcome: pendingPayload.outcome, - spawnMode: pendingPayload.spawnMode, - expectsCompletionMessage: pendingPayload.expectsCompletionMessage, - wakeOnDescendantSettle: pendingPayload.wakeOnDescendantSettle === true, - suppressChildSessionEffects: suppressSessionEffects, - isChildSessionEffectsAllowed: childSessionEffectsAllowed, - isCompletionDeliveryAllowed: () => isCleanupAttemptCurrent(runId, entry, cleanupGeneration), - isCompletionOwnedByRequesterYield: () => - entry.requesterTurnYielded === true || - entry.requesterSettleWake?.requesterYieldBatch === true, - onBeforeDeleteChildSession: - cleanup === "delete" - ? () => { - if (!childSessionEffectsAllowed()) { - return false; - } - // Announce owns delete submission; fence late yields at the - // exact handoff instead of when cleanup merely starts. - entry.deleteCleanupDispatchedAt ??= Date.now(); - params.persist(runId); - return true; - } - : undefined, - onDeliveryResult: (delivery) => { - if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { - retireSupersededCleanupInBackground(runId, entry, cleanupGeneration); + const current = params.runs.get(args.runId); + if ( + !current || + current.cleanupCompletedAt || + !context.isCleanupAttemptCurrent(args.runId, args.entry, args.cleanupGeneration) + ) { return; } - recordAnnounceDeliveryResult(entry, delivery); - if (delivery.delivered) { - const deliveryState = ensureDeliveryState(entry); - deliveryState.status = "delivered"; - deliveryState.announcedAt = deliveryState.deliveredAt ?? Date.now(); - deliveryState.lastError = undefined; - deliveryState.suspendedAt = undefined; - deliveryState.suspendedReason = undefined; - // Identified platform delivery precedes best-effort transcript - // mirroring; task ownership must become durable at that same edge. - params.persist(runId); - safeSetSubagentTaskDeliveryStatus({ entry, deliveryStatus: "delivered" }); - latestDeliveryError = undefined; - return; - } - if (delivery.path === "none" && delivery.disposition !== "intentional_non_delivery") { - ensureDeliveryState(entry).lastDropReason = "sink_unavailable"; - } - latestDeliveryError = formatAnnounceDeliveryError(delivery); - if (ensureDeliveryState(entry).lastError !== latestDeliveryError) { - ensureDeliveryState(entry).lastError = latestDeliveryError; - params.persist(runId); - } - }, - }; - runDetachedCleanupAttempt({ - runId, - entry, - cleanupGeneration, - run: async () => { - let announceOutcome: SubagentAnnounceFlowOutcome = "retryable"; - try { - announceOutcome = await params.runSubagentAnnounceFlow(announceParams); - } catch (error) { - defaultRuntime.log( - `[warn] Subagent announce flow failed during cleanup for run ${runId}: ${String(error)}`, + current.cleanupHandled = false; + params.resumedRuns.delete(args.runId); + params.persist(args.runId); + const failureCount = context.incrementCleanupFailureCount(current); + if (failureCount <= MAX_DETACHED_CLEANUP_RETRIES) { + scheduleResumeSubagentRun( + context, + args.runId, + current, + resolveAnnounceRetryDelayMs(failureCount), + args.cleanupGeneration, ); } - await finalizeAnnounceCleanup(announceOutcome); - }, + } + }).catch((err: unknown) => { + defaultRuntime.log( + `[warn] subagent cleanup admission failed (${args.runId}): ${String(err)}`, + ); + if (isGatewayRestartDraining()) { + scheduleResumeSubagentRun( + context, + args.runId, + args.entry, + MIN_ANNOUNCE_RETRY_DELAY_MS, + args.cleanupGeneration, + ); + } }); + }); +} + +export function suspendPendingFinalDelivery( + context: SubagentLifecycleCleanupContext & SubagentLifecycleWakeContext, + args: { + runId: string; + entry: SubagentRunRecord; + reason: "expiry" | "permanent_failure"; + error?: string; + }, +): void { + const params = context.options; + const previousEntry = structuredClone(args.entry); + markPendingFinalDelivery({ + entry: args.entry, + error: args.error ?? getDeliveryLastError(args.entry) ?? args.reason, + }); + const now = Date.now(); + const delivery = ensureDeliveryState(args.entry); + delivery.status = "suspended"; + delivery.suspendedAt ??= now; + delivery.suspendedReason = args.reason; + args.entry.cleanupHandled = false; + args.entry.wakeOnDescendantSettle = undefined; + const completion = ensureCompletionState(args.entry); + completion.fallbackResultText = undefined; + completion.fallbackCapturedAt = undefined; + params.resumedRuns.delete(args.runId); + safeSetSubagentTaskDeliveryStatus(params, { + entry: args.entry, + deliveryStatus: "failed", + deliveryError: getDeliveryLastError(args.entry) ?? args.reason, + }); + safeMarkRequiredCompletionDeliveryBlocked(params, { + entry: args.entry, + reason: getDeliveryLastError(args.entry) ?? args.reason, + }); + logAnnounceGiveUp(args.entry, args.reason); + markRequesterSettleWakePending(args.entry); + try { + params.persistOrThrow(args.runId); + } catch (error) { + const mutableEntry = args.entry as unknown as Record; + for (const key of Object.keys(mutableEntry)) { + delete mutableEntry[key]; + } + Object.assign(args.entry, previousEntry); + throw error; + } + // Suspension is terminal for automatic retries, so it settles this child + // for requester-drain purposes even though cleanup stays incomplete. + scheduleRequesterSettleWake(context, args.runId, args.entry); +} + +export function beginSubagentCleanup( + context: SubagentLifecycleCleanupContext, + runId: string, +): number | undefined { + const params = context.options; + const entry = params.runs.get(runId); + if (!entry || entry.cleanupCompletedAt || entry.cleanupHandled) { + return undefined; + } + entry.cleanupHandled = true; + const generation = context.bumpCleanupGeneration(entry); + params.persist(runId); + return generation; +} + +export async function retireSupersededCleanupIfNeeded( + context: SubagentLifecycleCleanupContext, + runId: string, + entry: SubagentRunRecord, + generation: number, +): Promise { + const params = context.options; + if ( + params.runs.get(runId) !== entry || + !context.isCleanupGeneration(entry, generation) || + !context.newerGenerationOwnsSession(entry) + ) { + return false; + } + // Cleanup can yield to attachment, mirror, or announce work. A successor + // registered while it was suspended owns every session-scoped side effect. + await params.retireSupersededRun(runId, entry); + return true; +} + +export function retireSupersededCleanupInBackground( + context: SubagentLifecycleCleanupContext, + runId: string, + entry: SubagentRunRecord, + generation: number, +): void { + // Delivery callbacks are synchronous and may arrive after their announce + // attempt returns. Give the async retirement tail its own snapshot blocker. + void runWithGatewayIndependentRootWorkAdmission(async () => { + await retireSupersededCleanupIfNeeded(context, runId, entry, generation); + }).catch((error: unknown) => { + defaultRuntime.log( + `[warn] subagent superseded cleanup retirement failed (${runId}): ${String(error)}`, + ); + }); +} + +async function retireRunModeBundleMcpRuntime( + context: SubagentLifecycleCommonContext, + cleanupParams: { runId: string; entry: SubagentRunRecord; reason: string }, +): Promise { + const params = context.options; + if (cleanupParams.entry.spawnMode === "session") { + return; + } + await retireSessionMcpRuntimeForSessionKey({ + sessionKey: cleanupParams.entry.childSessionKey, + reason: cleanupParams.reason, + preserveActiveLeases: true, + onError: (error, sessionId) => { + params.warn("failed to retire subagent bundle MCP runtime", { + error: buildSafeLifecycleErrorMeta(error), + sessionId, + runId: maskLifecycleIdentifier(cleanupParams.runId, "run"), + childSessionKey: maskLifecycleIdentifier(cleanupParams.entry.childSessionKey, "session"), + }); + }, + }); +} + +export async function completeTerminalEffects( + context: SubagentLifecycleCompletionContext, + args: { + completeParams: SubagentCompletionRequest; + completionReason: SubagentLifecycleEndedReason; + entry: SubagentRunRecord; + mutated: boolean; + sessionSuperseded: boolean; + suppressSessionEffects: boolean; + terminalGeneration: number; + loadCleanupBrowserSessionsForLifecycleEnd(): Promise; + }, +): Promise { + const params = context.options; + const { completeParams, completionReason, entry, mutated, terminalGeneration } = args; + let { sessionSuperseded, suppressSessionEffects } = args; + const isCurrentTerminalCallback = () => + context.isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration); + const isCurrentSessionEffectsOwner = () => + isCurrentTerminalCallback() && + !context.newerGenerationOwnsSession(entry) && + !shouldSuppressSubagentRecoverySessionEffects(entry); + const refreshSessionEffectsSuppression = () => { + if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { + return false; + } + suppressSessionEffects = true; + if (entry.execution.suppressSessionEffects !== true) { + const previousExecution = entry.execution; + entry.execution = { ...entry.execution, suppressSessionEffects: true }; + try { + params.persistOrThrow(completeParams.runId); + } catch (error) { + entry.execution = previousExecution; + suppressSessionEffects = false; + throw error; + } + } return true; }; - - return { - completeCleanupBookkeeping, - finalizeResumedAnnounceGiveUp: finalizeAnnounceGiveUp, - retireRunModeBundleMcpRuntime, - startSubagentAnnounceCleanupFlow, + if (!isCurrentTerminalCallback()) { + return; + } + const retireSupersededSession = async (currentEntry: SubagentRunRecord) => { + if (completionReason !== SUBAGENT_ENDED_REASON_KILLED) { + await params.retireSupersededRun(completeParams.runId, currentEntry); + } }; + sessionSuperseded ||= context.newerGenerationOwnsSession(entry); + if (sessionSuperseded) { + // This callback belongs to an older run that shared the session key. + // Update only its task projection; the newer generation owns all session effects. + await retireSupersededSession(entry); + return; + } + if (entry.collect) { + releaseSwarmRun(entry.schedulerSlotId ?? entry.runId); + } + refreshSessionEffectsSuppression(); + const isProvisionalKill = entry.killReconciliation !== undefined; + // Record only the current, non-superseded callback with a committed outcome; the + // run-terminal dedupe key is first-write-wins, so a provisional/stale status here + // would permanently mislabel the signal-log terminal kind. + const outcomeStatus = entry.execution.outcome?.status; + if ( + !suppressSessionEffects && + !isProvisionalKill && + outcomeStatus && + outcomeStatus !== "unknown" + ) { + recordSubagentTerminalState({ + childSessionKey: entry.childSessionKey, + runId: entry.runId, + requesterSessionKey: entry.requesterSessionKey, + outcomeStatus, + }); + } + + if (!suppressSessionEffects) { + try { + const assertSessionEffectsOwnerCurrent = () => { + if (!isCurrentSessionEffectsOwner()) { + throw new Error("subagent session-effects owner retired before session commit"); + } + }; + await persistSubagentSessionTiming(entry, { + // Recheck while patchSessionEntry owns its write lock so this old + // completion cannot commit after a synchronous ownership transfer. + isCurrentGeneration: isCurrentSessionEffectsOwner, + assertCommitAllowed: assertSessionEffectsOwnerCurrent, + }); + } catch (err) { + params.warn("failed to persist subagent session timing", { + err, + runId: entry.runId, + childSessionKey: entry.childSessionKey, + }); + } + } + refreshSessionEffectsSuppression(); + if (!isCurrentTerminalCallback()) { + return; + } + if (context.newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + + const suppressedForSteerRestart = params.suppressAnnounceForSteerRestart(entry); + // Recovery persists its terminal state before draining this callback, so an + // unchanged row still needs its first session-status and progress events. + const shouldPublishTerminalStatus = + mutated || + (completeParams.recoverInterrupted === true && + !isProvisionalKill && + !context.hasProgressEnded(entry)); + if (shouldPublishTerminalStatus && !suppressedForSteerRestart && !suppressSessionEffects) { + emitSessionLifecycleEvent({ + sessionKey: entry.childSessionKey, + reason: "subagent-status", + parentSessionKey: entry.requesterSessionKey, + label: entry.label, + }); + // The enclosing steer/session-effects guard admits only the real terminal generation. + if (!isProvisionalKill && !context.hasProgressEnded(entry)) { + context.markProgressEnded(entry); + await params.emitSubagentProgressEndedForRun(entry); + refreshSessionEffectsSuppression(); + if (!isCurrentTerminalCallback()) { + return; + } + } + } + const shouldEmitEndedHook = + !suppressedForSteerRestart && + !isProvisionalKill && + !suppressSessionEffects && + params.shouldEmitEndedHookForRun({ entry, reason: completionReason }); + const shouldDeferEndedHook = + shouldEmitEndedHook && + completeParams.triggerCleanup && + entry.expectsCompletionMessage === true && + !suppressedForSteerRestart; + if (!shouldDeferEndedHook && shouldEmitEndedHook) { + await params.emitSubagentEndedHookForRun({ + entry, + reason: completionReason, + sendFarewell: completeParams.sendFarewell, + accountId: completeParams.accountId, + isCurrent: isCurrentSessionEffectsOwner, + }); + refreshSessionEffectsSuppression(); + if (!isCurrentTerminalCallback()) { + return; + } + if (context.newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + } + + refreshSessionEffectsSuppression(); + await completeTerminalCleanup(context, { + completeParams, + entry, + isProvisionalKill, + retireSupersededSession, + suppressedForSteerRestart, + suppressSessionEffects, + terminalGeneration, + loadCleanupBrowserSessionsForLifecycleEnd: () => + args.loadCleanupBrowserSessionsForLifecycleEnd(), + }); +} + +async function completeTerminalCleanup( + context: SubagentLifecycleCompletionContext, + args: { + completeParams: SubagentCompletionRequest; + entry: SubagentRunRecord; + isProvisionalKill: boolean; + retireSupersededSession: (entry: SubagentRunRecord) => Promise; + suppressedForSteerRestart: boolean; + suppressSessionEffects: boolean; + terminalGeneration: number; + loadCleanupBrowserSessionsForLifecycleEnd(): Promise; + }, +): Promise { + const params = context.options; + const { + completeParams, + entry, + isProvisionalKill, + retireSupersededSession, + suppressedForSteerRestart, + terminalGeneration, + } = args; + let { suppressSessionEffects } = args; + // Session cleanup belongs to the exact registry row and child generation. + // A replacement may reuse either the run id or the child session key. + const isSessionEffectsOwnerCurrent = () => + context.isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration) && + !context.newerGenerationOwnsSession(entry); + const refreshSessionEffectsSuppression = () => { + if ( + suppressSessionEffects || + !isSessionEffectsOwnerCurrent() || + !shouldSuppressSubagentRecoverySessionEffects(entry) + ) { + return suppressSessionEffects; + } + const previousExecution = entry.execution; + entry.execution = { ...previousExecution, suppressSessionEffects: true }; + try { + params.persistOrThrow(completeParams.runId); + } catch (error) { + entry.execution = previousExecution; + throw error; + } + suppressSessionEffects = true; + return true; + }; + if (!completeParams.triggerCleanup || suppressedForSteerRestart) { + return; + } + refreshSessionEffectsSuppression(); + if (!context.isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + if (context.newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + + // registerSubagentRun fires both an in-process listener and a gateway + // waitForSubagentCompletion RPC; both can reach this point for the same + // runId in embedded mode. Dedupe only the browser driver tab-close IPC + // with a sync check-then-set. The retire + announce tail below must still + // run for every caller, so a slow or held first browser cleanup cannot + // strand a duplicate caller's completion behind it. + if (!suppressSessionEffects && entry.browserCleanupDispatchedAt === undefined) { + let dispatchedBrowserCleanup = false; + let cleanupBrowserSessions: typeof cleanupBrowserSessionsForLifecycleEnd | undefined = + params.cleanupBrowserSessionsForLifecycleEnd; + try { + cleanupBrowserSessions ??= await args.loadCleanupBrowserSessionsForLifecycleEnd(); + } catch (error) { + params.warn("failed to load browser cleanup for completed subagent", { + error: buildSafeLifecycleErrorMeta(error), + runId: maskLifecycleIdentifier(completeParams.runId, "run"), + childSessionKey: maskLifecycleIdentifier(entry.childSessionKey, "session"), + }); + } + if (cleanupBrowserSessions) { + if (!context.isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + if (context.newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + if (refreshSessionEffectsSuppression()) { + return; + } + // Claim only when this caller is about to dispatch. A concurrent caller + // may have claimed while the lazy browser module was loading. + if (entry.browserCleanupDispatchedAt === undefined) { + entry.browserCleanupDispatchedAt = Date.now(); + dispatchedBrowserCleanup = true; + try { + await cleanupBrowserSessions({ + sessionKeys: [entry.childSessionKey], + onWarn: (msg) => params.warn(msg, { runId: entry.runId }), + }); + } catch (error) { + params.warn("failed to cleanup browser sessions for completed subagent", { + error: buildSafeLifecycleErrorMeta(error), + runId: maskLifecycleIdentifier(completeParams.runId, "run"), + childSessionKey: maskLifecycleIdentifier(entry.childSessionKey, "session"), + }); + } + } + } + if (dispatchedBrowserCleanup) { + if (!context.isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + refreshSessionEffectsSuppression(); + if (context.newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + } + } + + if (!suppressSessionEffects) { + if (!context.isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + if (context.newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + try { + await retireRunModeBundleMcpRuntime(context, { + runId: completeParams.runId, + entry, + reason: "subagent-run-complete", + }); + } catch (error) { + params.warn("failed to retire subagent bundle MCP runtime after completion", { + error: buildSafeLifecycleErrorMeta(error), + runId: maskLifecycleIdentifier(completeParams.runId, "run"), + childSessionKey: maskLifecycleIdentifier(entry.childSessionKey, "session"), + }); + } + if (!context.isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + refreshSessionEffectsSuppression(); + if (context.newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + } + + if (isProvisionalKill) { + // Browser and MCP resources can close immediately, but completion delivery + // waits for the provider result or the killed tombstone reconciliation. + return; + } + + refreshSessionEffectsSuppression(); + context.startSubagentAnnounceCleanupFlow(completeParams.runId, entry); } diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-common.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-common.ts deleted file mode 100644 index 43177eb39149..000000000000 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-common.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; -import { formatErrorMessage, readErrorName } from "../../../infra/errors.js"; -import type { - SubagentRegistryLifecycleParams, - SubagentRegistryLifecycleState, -} from "./subagent-registry-lifecycle-contracts.js"; -import type { SubagentRunRecord } from "./subagent-registry.types.js"; -import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; - -export function createSubagentRegistryLifecycleCommon( - params: SubagentRegistryLifecycleParams, - state: SubagentRegistryLifecycleState, -) { - const { - scheduledResumeTimers, - scheduledRequesterSettleWakeTimers, - pendingRequesterSettleWakeRearms, - terminalCompletionLocks, - } = state; - - const newerGenerationOwnsSession = (entry: SubagentRunRecord): boolean => - entry.killReconciliation?.supersededAt !== undefined || - Array.from(params.runs.values()).some( - (candidate) => - candidate.runId !== entry.runId && - candidate.childSessionKey === entry.childSessionKey && - compareSubagentRunGeneration(candidate, entry) > 0, - ); - - const acquireTerminalCompletionLock = async (runId: string): Promise<() => void> => { - const previous = terminalCompletionLocks.get(runId) ?? Promise.resolve(); - let releaseLock = () => {}; - const current = new Promise((resolve) => { - releaseLock = resolve; - }); - terminalCompletionLocks.set(runId, current); - await previous; - return () => { - releaseLock(); - if (terminalCompletionLocks.get(runId) === current) { - terminalCompletionLocks.delete(runId); - } - }; - }; - - const clearScheduledResumeTimers = () => { - for (const timer of scheduledResumeTimers) { - clearTimeout(timer); - } - scheduledResumeTimers.clear(); - for (const scheduled of scheduledRequesterSettleWakeTimers.values()) { - clearTimeout(scheduled.timer); - } - scheduledRequesterSettleWakeTimers.clear(); - pendingRequesterSettleWakeRearms.clear(); - }; - - const maskRunId = (runId: string): string => { - const trimmed = runId.trim(); - if (!trimmed) { - return "unknown"; - } - if (trimmed.length <= 8) { - return "***"; - } - return `${sliceUtf16Safe(trimmed, 0, 4)}…${sliceUtf16Safe(trimmed, -4)}`; - }; - - const maskSessionKey = (sessionKey: string): string => { - const trimmed = sessionKey.trim(); - if (!trimmed) { - return "unknown"; - } - const prefix = trimmed.split(":").slice(0, 2).join(":") || "session"; - return `${prefix}:…`; - }; - - const buildSafeLifecycleErrorMeta = (err: unknown): Record => { - const message = formatErrorMessage(err); - const name = readErrorName(err); - return name ? { name, message } : { message }; - }; - - return { - acquireTerminalCompletionLock, - buildSafeLifecycleErrorMeta, - clearScheduledResumeTimers, - maskRunId, - maskSessionKey, - newerGenerationOwnsSession, - }; -} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-completion-support.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-completion-support.ts deleted file mode 100644 index b8682e693d45..000000000000 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-completion-support.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { createLazyImportLoader } from "../../../shared/lazy-promise.js"; -import type { SubagentRunOutcome } from "../announce/subagent-announce-output.js"; -import type { SubagentLifecycleEndedReason } from "./subagent-lifecycle-events.js"; -import type { SubagentRunRecord } from "./subagent-registry.types.js"; -import { - resolveSubagentRunDeadlineMs, - resolveSubagentRunEffectiveEndedAt, -} from "./subagent-run-timeout.js"; - -type BrowserCleanupModule = Pick< - typeof import("../../../browser-lifecycle-cleanup.js"), - "cleanupBrowserSessionsForLifecycleEnd" ->; - -const browserCleanupLoader = createLazyImportLoader( - () => import("../../../browser-lifecycle-cleanup.js"), -); - -export async function loadCleanupBrowserSessionsForLifecycleEnd(): Promise< - BrowserCleanupModule["cleanupBrowserSessionsForLifecycleEnd"] -> { - return (await browserCleanupLoader.load()).cleanupBrowserSessionsForLifecycleEnd; -} - -export function shouldPreservePublishedExplicitRunTimeout(params: { - entry: SubagentRunRecord; -}): boolean { - if ( - typeof params.entry.runTimeoutSeconds !== "number" || - !Number.isFinite(params.entry.runTimeoutSeconds) || - params.entry.runTimeoutSeconds <= 0 || - params.entry.execution.outcome?.status !== "timeout" || - typeof params.entry.execution.endedAt !== "number" - ) { - return false; - } - const deadlineMs = resolveSubagentRunDeadlineMs(params.entry); - if (deadlineMs === undefined || params.entry.execution.endedAt < deadlineMs) { - return false; - } - if ( - params.entry.cleanupHandled || - typeof params.entry.cleanupCompletedAt === "number" || - typeof params.entry.endedHookEmittedAt === "number" || - params.entry.delivery?.status === "delivered" || - typeof params.entry.delivery?.announcedAt === "number" - ) { - return true; - } - return false; -} - -export function resolveExpiredExplicitRunDeadlineMs(params: { - entry: SubagentRunRecord; - nextEndedAt: number; - observedStartedAt?: number; -}): number | undefined { - const effectiveEndedAt = resolveSubagentRunEffectiveEndedAt( - params.entry, - params.nextEndedAt, - params.observedStartedAt, - ); - return effectiveEndedAt < params.nextEndedAt ? effectiveEndedAt : undefined; -} - -export function isOlderEquivalentTerminalCallback(params: { - entry: SubagentRunRecord; - endedAt: number; - outcome: SubagentRunOutcome; - reason: SubagentLifecycleEndedReason; -}): boolean { - const current = params.entry.execution.outcome; - if ( - typeof params.entry.execution.endedAt !== "number" || - params.endedAt >= params.entry.execution.endedAt || - params.entry.endedReason !== params.reason || - current?.status !== params.outcome.status - ) { - return false; - } - return ( - current.status !== "error" || - params.outcome.status !== "error" || - current.error === params.outcome.error - ); -} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-completion.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-completion.ts index 55a172fba2e1..63a42fb6d3aa 100644 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-completion.ts +++ b/src/agents/subagents/registry/subagent-registry-lifecycle-completion.ts @@ -1,7 +1,6 @@ import { SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js"; import { isAgentEventLifecycleGenerationCurrent } from "../../../infra/agent-events.js"; -import { emitSessionLifecycleEvent } from "../../../sessions/session-lifecycle-events.js"; -import { recordSubagentTerminalState } from "../../../sessions/session-state-events.js"; +import { createLazyImportLoader } from "../../../shared/lazy-promise.js"; import type { DetachedTaskFindResult } from "../../../tasks/detached-task-runtime-contract.js"; import { isProvisionalSubagentKillTask } from "../../../tasks/task-cancellation-state.js"; import { mergeAgentRunTerminalReplySnapshot } from "../../agent-run-terminal-reply.js"; @@ -11,734 +10,622 @@ import { withSubagentOutcomeTiming, } from "../announce/subagent-announce-output.js"; import { updateSwarmCollectorCompletion } from "../swarm/swarm-collector.js"; -import { releaseSwarmRun } from "../swarm/swarm-scheduler.js"; import { clearDeliveryState, ensureCompletionState } from "./subagent-delivery-state.js"; import { SUBAGENT_ENDED_REASON_COMPLETE, SUBAGENT_ENDED_REASON_ERROR, SUBAGENT_ENDED_REASON_KILLED, + type SubagentLifecycleEndedReason, } from "./subagent-lifecycle-events.js"; import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; import { resolveKilledSubagentTaskEndedAt } from "./subagent-registry-completion.js"; +import { updateSubagentArchiveAtMs } from "./subagent-registry-helpers.js"; +import { completeTerminalEffects } from "./subagent-registry-lifecycle-cleanup.js"; +import type { SubagentLifecycleCompletionContext } from "./subagent-registry-lifecycle-context.js"; import { - persistSubagentSessionTiming, - updateSubagentArchiveAtMs, -} from "./subagent-registry-helpers.js"; -import type { createSubagentRegistryLifecycleCleanupBase } from "./subagent-registry-lifecycle-cleanup-base.js"; -import type { createSubagentRegistryLifecycleCleanup } from "./subagent-registry-lifecycle-cleanup.js"; -import type { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; -import { - isOlderEquivalentTerminalCallback, - resolveExpiredExplicitRunDeadlineMs, - shouldPreservePublishedExplicitRunTimeout, -} from "./subagent-registry-lifecycle-completion-support.js"; -import type { - SubagentRegistryLifecycleParams, - SubagentRegistryLifecycleState, -} from "./subagent-registry-lifecycle-contracts.js"; -import type { createSubagentRegistryLifecycleDelivery } from "./subagent-registry-lifecycle-delivery.js"; -import { createSubagentRegistryLifecycleTerminalCleanup } from "./subagent-registry-lifecycle-terminal-cleanup.js"; + freezeRunResultAtCompletion, + refreshPendingFinalDeliveryPayload, + safeFinalizeSubagentTaskRun, +} from "./subagent-registry-lifecycle-delivery.js"; import type { SubagentCompletionRequest, SubagentRunRecord } from "./subagent-registry.types.js"; +import { + resolveSubagentRunDeadlineMs, + resolveSubagentRunEffectiveEndedAt, +} from "./subagent-run-timeout.js"; -export function createSubagentRegistryLifecycleCompletion( - params: SubagentRegistryLifecycleParams, - state: SubagentRegistryLifecycleState, - common: ReturnType, - deliveryHelpers: ReturnType, - cleanupBase: ReturnType, - cleanup: ReturnType, -) { - const { cleanupGenerations, progressEndedEntries, terminalGenerations } = state; - const terminalCleanup = createSubagentRegistryLifecycleTerminalCleanup( - params, - common, - cleanupBase, - cleanup, +type BrowserCleanupModule = typeof import("../../../browser-lifecycle-cleanup.js"); +type BrowserCleanup = BrowserCleanupModule["cleanupBrowserSessionsForLifecycleEnd"]; + +const browserCleanupLoader = createLazyImportLoader( + () => import("../../../browser-lifecycle-cleanup.js"), +); + +async function loadCleanupBrowserSessionsForLifecycleEnd(): Promise { + return (await browserCleanupLoader.load()).cleanupBrowserSessionsForLifecycleEnd; +} + +function shouldPreservePublishedExplicitRunTimeout(params: { entry: SubagentRunRecord }): boolean { + if ( + typeof params.entry.runTimeoutSeconds !== "number" || + !Number.isFinite(params.entry.runTimeoutSeconds) || + params.entry.runTimeoutSeconds <= 0 || + params.entry.execution.outcome?.status !== "timeout" || + typeof params.entry.execution.endedAt !== "number" + ) { + return false; + } + const deadlineMs = resolveSubagentRunDeadlineMs(params.entry); + if (deadlineMs === undefined || params.entry.execution.endedAt < deadlineMs) { + return false; + } + return ( + params.entry.cleanupHandled === true || + typeof params.entry.cleanupCompletedAt === "number" || + typeof params.entry.endedHookEmittedAt === "number" || + params.entry.delivery?.status === "delivered" || + typeof params.entry.delivery?.announcedAt === "number" ); - const { acquireTerminalCompletionLock, newerGenerationOwnsSession } = common; - const { - freezeRunResultAtCompletion, - refreshPendingFinalDeliveryPayload, - safeFinalizeSubagentTaskRun, - } = deliveryHelpers; - const { isTerminalCallbackCurrent } = cleanupBase; +} - const completeSubagentRunAttempt = async (completeParams: SubagentCompletionRequest) => { - const releaseCompletionLock = await acquireTerminalCompletionLock(completeParams.runId); - let entry: SubagentRunRecord | undefined; - let terminalGeneration = 0; - let mutated = false; - let completionReason = completeParams.reason; - let sessionSuperseded = false; - let suppressSessionEffects = completeParams.suppressSessionEffects === true; - let suppressTaskFinalization: boolean; - let provisionalKillSnapshot: SubagentRunRecord | undefined; - let postCaptureTaskResolution: DetachedTaskFindResult | undefined; - let entrySnapshot: SubagentRunRecord | undefined; - try { - entry = params.runs.get(completeParams.runId); - if (!entry) { +function resolveExpiredExplicitRunDeadlineMs(params: { + entry: SubagentRunRecord; + nextEndedAt: number; + observedStartedAt?: number; +}): number | undefined { + const effectiveEndedAt = resolveSubagentRunEffectiveEndedAt( + params.entry, + params.nextEndedAt, + params.observedStartedAt, + ); + return effectiveEndedAt < params.nextEndedAt ? effectiveEndedAt : undefined; +} + +function isOlderEquivalentTerminalCallback(params: { + entry: SubagentRunRecord; + endedAt: number; + outcome: SubagentRunOutcome; + reason: SubagentLifecycleEndedReason; +}): boolean { + const current = params.entry.execution.outcome; + if ( + typeof params.entry.execution.endedAt !== "number" || + params.endedAt >= params.entry.execution.endedAt || + params.entry.endedReason !== params.reason || + current?.status !== params.outcome.status + ) { + return false; + } + return ( + current.status !== "error" || + params.outcome.status !== "error" || + current.error === params.outcome.error + ); +} + +export async function completeSubagentRunAttempt( + context: SubagentLifecycleCompletionContext, + completeParams: SubagentCompletionRequest, +): Promise { + const params = context.options; + const releaseCompletionLock = await context.acquireTerminalCompletionLock(completeParams.runId); + let entry: SubagentRunRecord | undefined; + let terminalGeneration = 0; + let mutated = false; + let completionReason = completeParams.reason; + let sessionSuperseded = false; + let suppressSessionEffects = completeParams.suppressSessionEffects === true; + let suppressTaskFinalization: boolean; + let provisionalKillSnapshot: SubagentRunRecord | undefined; + let postCaptureTaskResolution: DetachedTaskFindResult | undefined; + let entrySnapshot: SubagentRunRecord | undefined; + try { + entry = params.runs.get(completeParams.runId); + if (!entry) { + return; + } + if (completeParams.expectedEntry && entry !== completeParams.expectedEntry) { + return; + } + suppressSessionEffects ||= shouldSuppressSubagentRecoverySessionEffects(entry); + params.clearPendingLifecycleError(completeParams.runId); + const currentEntry = entry; + entrySnapshot = structuredClone(entry); + const restoreEntrySnapshot = (snapshot?: SubagentRunRecord) => { + if (!snapshot) { return; } - if (completeParams.expectedEntry && entry !== completeParams.expectedEntry) { - return; + const target = currentEntry as unknown as Record; + for (const key of Object.keys(target)) { + delete target[key]; } - suppressSessionEffects ||= shouldSuppressSubagentRecoverySessionEffects(entry); - params.clearPendingLifecycleError(completeParams.runId); - const currentEntry = entry; - entrySnapshot = structuredClone(entry); - const restoreEntrySnapshot = (snapshot?: SubagentRunRecord) => { - if (!snapshot) { - return; - } - const target = currentEntry as unknown as Record; - for (const key of Object.keys(target)) { - delete target[key]; - } - Object.assign(target, snapshot); - }; - const recoveryRequested = completeParams.recoverInterrupted === true; - if ( - !recoveryRequested && - (entry.terminalOwner === "interrupted-recovery" || - entry.execution.suppressSessionEffects === true) && - entry.killIntent === undefined - ) { - // Restart recovery already persisted the terminal winner for this exact - // run. Its sticky fence survives cleanup of the transient owner marker. - return; - } - if (recoveryRequested) { - const ownsInterruptedRecovery = entry.terminalOwner === "interrupted-recovery"; - // Mismatched partial terminal evidence is an existing winner and must - // not be overwritten. Exact normalized evidence may be the same recovery - // request deferred by restart admission, so drain it. - const hasTerminalEvidence = - entry.execution.status === "terminal" || - entry.endedReason !== undefined || - typeof entry.cleanupCompletedAt === "number"; - const expectedElapsedMs = - typeof currentEntry.execution.startedAt === "number" && - typeof completeParams.endedAt === "number" - ? Math.max(0, completeParams.endedAt - currentEntry.execution.startedAt) - : undefined; - const outcomeMatchesInterruptedRecovery = (outcome: SubagentRunOutcome | undefined) => - completeParams.outcome.status === "error" && - outcome?.status === "error" && - outcome.error === completeParams.outcome.error && - (outcome.startedAt === undefined || - outcome.startedAt === currentEntry.execution.startedAt) && - (outcome.endedAt === undefined || outcome.endedAt === completeParams.endedAt) && - (outcome.elapsedMs === undefined || outcome.elapsedMs === expectedElapsedMs); - const matchesRequestedInterruptedTerminal = - typeof completeParams.endedAt === "number" && - entry.execution.endedAt === completeParams.endedAt && - outcomeMatchesInterruptedRecovery(entry.execution.outcome) && - entry.endedReason === SUBAGENT_ENDED_REASON_ERROR; - if ( - !ownsInterruptedRecovery && - (entry.killReconciliation !== undefined || - entry.endedReason === SUBAGENT_ENDED_REASON_KILLED || - entry.pauseReason === "sessions_yield" || - typeof entry.cleanupCompletedAt === "number" || - (hasTerminalEvidence && !matchesRequestedInterruptedTerminal)) - ) { - return; - } - if (!ownsInterruptedRecovery) { - const endedAt = - typeof completeParams.endedAt === "number" ? completeParams.endedAt : Date.now(); - const outcome = withSubagentOutcomeTiming( - { status: "error", error: completeParams.outcome.error }, - { startedAt: entry.execution.startedAt, endedAt }, - ); - entry.endedReason = SUBAGENT_ENDED_REASON_ERROR; - entry.pauseReason = undefined; - entry.execution = { - ...entry.execution, - status: "terminal", - endedAt, - outcome, - interruptedAt: undefined, - interruptionReason: undefined, - suppressSessionEffects: suppressSessionEffects ? true : undefined, - }; - entry.completion = { - ...ensureCompletionState(entry), - resultText: null, - capturedAt: endedAt, - }; - entry.cleanupHandled = false; - entry.terminalOwner = "interrupted-recovery"; - mutated = true; - try { - params.persistOrThrow(completeParams.runId); - } catch (error) { - restoreEntrySnapshot(entrySnapshot); - throw error; - } - // Any later delivery-payload write rolls back to this durable owner, - // never to the pre-recovery running row. - entrySnapshot = structuredClone(entry); - mutated = false; - } - } - sessionSuperseded = newerGenerationOwnsSession(currentEntry); - if ( - completeParams.reason === SUBAGENT_ENDED_REASON_KILLED && - entry.killIntent === undefined && - entry.endedReason !== undefined && - entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED && - entry.execution.outcome !== undefined - ) { - // Any finalized provider outcome is canonical. A delayed abort listener - // must not replace success, failure, or timeout with a killed marker. - return; - } - let requestedEndedAt = - typeof completeParams.endedAt === "number" ? completeParams.endedAt : Date.now(); - if ( - shouldPreservePublishedExplicitRunTimeout({ - entry, - }) - ) { - return; - } - const shouldDrainExistingTerminal = - recoveryRequested || - isOlderEquivalentTerminalCallback({ - entry, - endedAt: requestedEndedAt, - outcome: completeParams.outcome, - reason: completeParams.reason, - }); - if (shouldDrainExistingTerminal) { - // Preserve the newer canonical timing while allowing this duplicate - // caller to rescue a stalled cleanup and delivery tail. - requestedEndedAt = entry.execution.endedAt!; - completionReason = entry.endedReason ?? completeParams.reason; - } - let endedAt = requestedEndedAt; - let completionOutcome = - shouldDrainExistingTerminal && entry.execution.outcome - ? entry.execution.outcome - : completeParams.outcome; - const liveStructuredOutput = entry.collect - ? (entry.structuredOutput ?? - peekSwarmStructuredOutput(entry.runId) ?? - (entry.swarmRunId ? peekSwarmStructuredOutput(entry.swarmRunId) : undefined)) - : undefined; - if (!entry.structuredOutput && liveStructuredOutput) { - entry.structuredOutput = liveStructuredOutput; - mutated = true; - } - if ( - liveStructuredOutput?.structured !== undefined && - completionOutcome.status === "error" && - completionOutcome.error === "completed" - ) { - // Tool-only collector turns use this runner sentinel after the result is - // durably recorded. Normalize before every task/session/hook projection. - completionOutcome = { status: "ok" }; - completionReason = SUBAGENT_ENDED_REASON_COMPLETE; - } - const observedStartedAt = - !shouldDrainExistingTerminal && - typeof completeParams.startedAt === "number" && - Number.isFinite(completeParams.startedAt) - ? completeParams.startedAt + Object.assign(target, snapshot); + }; + const recoveryRequested = completeParams.recoverInterrupted === true; + if ( + !recoveryRequested && + (entry.terminalOwner === "interrupted-recovery" || + entry.execution.suppressSessionEffects === true) && + entry.killIntent === undefined + ) { + // Restart recovery already persisted the terminal winner for this exact + // run. Its sticky fence survives cleanup of the transient owner marker. + return; + } + if (recoveryRequested) { + const ownsInterruptedRecovery = entry.terminalOwner === "interrupted-recovery"; + // Mismatched partial terminal evidence is an existing winner and must + // not be overwritten. Exact normalized evidence may be the same recovery + // request deferred by restart admission, so drain it. + const hasTerminalEvidence = + entry.execution.status === "terminal" || + entry.endedReason !== undefined || + typeof entry.cleanupCompletedAt === "number"; + const expectedElapsedMs = + typeof currentEntry.execution.startedAt === "number" && + typeof completeParams.endedAt === "number" + ? Math.max(0, completeParams.endedAt - currentEntry.execution.startedAt) : undefined; - const expiredDeadlineMs = recoveryRequested - ? undefined - : resolveExpiredExplicitRunDeadlineMs({ - entry, - nextEndedAt: endedAt, - observedStartedAt, - }); - if (expiredDeadlineMs !== undefined) { - endedAt = expiredDeadlineMs; - completionOutcome = { status: "timeout" }; - completionReason = SUBAGENT_ENDED_REASON_COMPLETE; - } - const killIntent = entry.killIntent; - if (killIntent) { - if (completionReason !== SUBAGENT_ENDED_REASON_KILLED && endedAt < killIntent.requestedAt) { - entry.killIntent = undefined; - } else { - const killOwnsCurrentLifecycle = - killIntent.lifecycleGeneration !== undefined && - isAgentEventLifecycleGenerationCurrent(killIntent.lifecycleGeneration); - completionReason = SUBAGENT_ENDED_REASON_KILLED; - completionOutcome = { status: "error", error: killIntent.reason }; - entry.killIntent = undefined; - if (killOwnsCurrentLifecycle) { - suppressSessionEffects = false; - entry.execution = { - ...entry.execution, - lifecycleGeneration: killIntent.lifecycleGeneration, - restartRecovery: undefined, - suppressSessionEffects: undefined, - }; - } - entry.killReconciliation = { - killedAt: killIntent.requestedAt, - suppressTaskDelivery: killIntent.suppressTaskDelivery === true ? true : undefined, - }; - } - mutated = true; - } + const outcomeMatchesInterruptedRecovery = (outcome: SubagentRunOutcome | undefined) => + completeParams.outcome.status === "error" && + outcome?.status === "error" && + outcome.error === completeParams.outcome.error && + (outcome.startedAt === undefined || + outcome.startedAt === currentEntry.execution.startedAt) && + (outcome.endedAt === undefined || outcome.endedAt === completeParams.endedAt) && + (outcome.elapsedMs === undefined || outcome.elapsedMs === expectedElapsedMs); + const matchesRequestedInterruptedTerminal = + typeof completeParams.endedAt === "number" && + entry.execution.endedAt === completeParams.endedAt && + outcomeMatchesInterruptedRecovery(entry.execution.outcome) && + entry.endedReason === SUBAGENT_ENDED_REASON_ERROR; if ( - completionReason !== SUBAGENT_ENDED_REASON_KILLED && - entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && - entry.killReconciliation === undefined + !ownsInterruptedRecovery && + (entry.killReconciliation !== undefined || + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED || + entry.pauseReason === "sessions_yield" || + typeof entry.cleanupCompletedAt === "number" || + (hasTerminalEvidence && !matchesRequestedInterruptedTerminal)) ) { - // Only current-version provisional kills carry reconciliation state. - // Legacy or already-stabilized killed rows are terminal cancellation. return; } - const isSteerRestartKill = - completeParams.reason === SUBAGENT_ENDED_REASON_KILLED && - entry.suppressAnnounceReason === "steer-restart"; - suppressTaskFinalization = isSteerRestartKill; - if (completionReason === SUBAGENT_ENDED_REASON_KILLED && !isSteerRestartKill) { - entry.suppressAnnounceReason = "killed"; - entry.killReconciliation ??= { - killedAt: requestedEndedAt, - }; - mutated = true; - } - - if ( - completionReason !== SUBAGENT_ENDED_REASON_KILLED && - entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && - entry.killReconciliation !== undefined - ) { - const killReconciliation = entry.killReconciliation; - const taskResolution = params.resolveSubagentTask(entry); - const stableTaskCancellation = - taskResolution.lookup === "available" && - taskResolution.task?.status === "cancelled" && - !isProvisionalSubagentKillTask(taskResolution.task); - const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(entry); - const completionPredatesCancellation = - typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; - if (stableTaskCancellation && !completionPredatesCancellation) { - // tasks.cancel promotes the provisional marker to durable operator - // intent. Only an already-durable earlier completion may reopen it. - return; - } - provisionalKillSnapshot = structuredClone(currentEntry); - // The sweeper uses marker identity to reject a concurrently replaced - // kill generation. A completion rollback must retain the same marker. - provisionalKillSnapshot.killReconciliation = killReconciliation; - // Completion capture yields. Stage the provider result off-registry so - // an unrelated persistence write cannot publish a tentative winner. - entry = structuredClone(currentEntry); - entry.suppressCompletionDelivery = - killReconciliation.suppressTaskDelivery === true ? true : undefined; - entry.suppressAnnounceReason = undefined; - entry.killReconciliation = undefined; - entry.cleanupHandled = false; - entry.cleanupCompletedAt = undefined; - clearDeliveryState(entry); - mutated = true; - } - - if (observedStartedAt !== undefined && entry.execution.startedAt !== observedStartedAt) { - entry.execution = { ...entry.execution, startedAt: observedStartedAt }; - if (typeof entry.sessionStartedAt !== "number") { - entry.sessionStartedAt = observedStartedAt; - } - mutated = true; - } - - if ( - completionReason === SUBAGENT_ENDED_REASON_COMPLETE && - completionOutcome.status !== "error" && - provisionalKillSnapshot !== undefined - ) { - // A killed lifecycle may freeze an empty result before the canonical end - // wins. Preserve any reply already captured by an earlier successful callback. - const completion = ensureCompletionState(entry); - const hasCapturedReply = - typeof completion.resultText === "string" && completion.resultText.trim().length > 0; - if ( - !hasCapturedReply && - (completion.resultText !== undefined || completion.capturedAt !== undefined) - ) { - completion.resultText = undefined; - completion.capturedAt = undefined; - mutated = true; - } - } - const outcome = - recoveryRequested && entry.execution.outcome - ? entry.execution.outcome - : withSubagentOutcomeTiming(completionOutcome, { - startedAt: entry.execution.startedAt, - endedAt, - }); - const executionOutcome = recoveryRequested ? (entry.execution.outcome ?? outcome) : outcome; - const retainedRestartRecovery = suppressSessionEffects - ? entry.execution.restartRecovery - : undefined; - if ( - entry.execution.status !== "terminal" || - entry.execution.endedAt !== endedAt || - entry.execution.outcome !== executionOutcome || - entry.execution.restartRecovery !== retainedRestartRecovery || - entry.execution.suppressSessionEffects !== (suppressSessionEffects ? true : undefined) - ) { + if (!ownsInterruptedRecovery) { + const endedAt = + typeof completeParams.endedAt === "number" ? completeParams.endedAt : Date.now(); + const outcome = withSubagentOutcomeTiming( + { status: "error", error: completeParams.outcome.error }, + { startedAt: entry.execution.startedAt, endedAt }, + ); + entry.endedReason = SUBAGENT_ENDED_REASON_ERROR; + entry.pauseReason = undefined; entry.execution = { ...entry.execution, status: "terminal", endedAt, - outcome: executionOutcome, - restartRecovery: retainedRestartRecovery, + outcome, + interruptedAt: undefined, + interruptionReason: undefined, suppressSessionEffects: suppressSessionEffects ? true : undefined, }; + entry.completion = { + ...ensureCompletionState(entry), + resultText: null, + capturedAt: endedAt, + }; + entry.cleanupHandled = false; + entry.terminalOwner = "interrupted-recovery"; mutated = true; - } - if (entry.endedReason !== completionReason) { - entry.endedReason = completionReason; - mutated = true; - } - if (completionReason === SUBAGENT_ENDED_REASON_KILLED && entry.terminalOwner !== undefined) { - entry.terminalOwner = undefined; - mutated = true; - } - if (entry.pauseReason !== undefined) { - entry.pauseReason = undefined; - mutated = true; - } - - if (completeParams.completionSnapshot) { - const completion = ensureCompletionState(entry); - if ( - completion.resultText !== completeParams.completionSnapshot.resultText || - completion.capturedAt !== completeParams.completionSnapshot.capturedAt - ) { - completion.resultText = completeParams.completionSnapshot.resultText; - completion.capturedAt = completeParams.completionSnapshot.capturedAt; - mutated = true; - } - } - - if (completeParams.terminalReply) { - const completion = ensureCompletionState(entry); - const terminalReply = mergeAgentRunTerminalReplySnapshot( - completion.terminalReply, - completeParams.terminalReply, - ); - if ( - terminalReply && - JSON.stringify(terminalReply) !== JSON.stringify(completion.terminalReply) - ) { - completion.terminalReply = terminalReply; - completion.resultText = - terminalReply.disposition === "visible" - ? terminalReply.text - : terminalReply.disposition === "silent" - ? SILENT_REPLY_TOKEN - : null; - completion.capturedAt = endedAt; - mutated = true; - } - } - - // A newer generation may share the session key. Its transcript/reply is - // not evidence for this older run, so reconcile only the terminal task state. - if (recoveryRequested || sessionSuperseded) { - const completion = ensureCompletionState(entry); - if (completion.resultText === undefined) { - completion.resultText = null; - completion.capturedAt = Date.now(); - mutated = true; - } - } else { - const didFreezeResult = await freezeRunResultAtCompletion(entry, executionOutcome); - sessionSuperseded = newerGenerationOwnsSession(entry); - if (sessionSuperseded) { - const completion = ensureCompletionState(entry); - completion.resultText = null; - completion.capturedAt = Date.now(); - mutated = true; - } else if (didFreezeResult) { - mutated = true; - } - } - if ( - entry.collect - ? updateSwarmCollectorCompletion(entry, params.getRuntimeConfig()) - : updateSubagentArchiveAtMs(entry, params.getRuntimeConfig()) - ) { - mutated = true; - } - if (provisionalKillSnapshot) { - // Keep the tombstone's superseded generation boundary through task - // commit. Clearing it on the canonical registry row must not let a - // late old-run result select a newer task sharing the session key. - const taskResolution = params.resolveSubagentTask(provisionalKillSnapshot); - postCaptureTaskResolution = taskResolution; - const stableTaskCancellation = - taskResolution.lookup === "available" && - taskResolution.task?.status === "cancelled" && - !isProvisionalSubagentKillTask(taskResolution.task); - const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(provisionalKillSnapshot); - const completionPredatesCancellation = - typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; - if (stableTaskCancellation && !completionPredatesCancellation) { - // Cancellation can become durable while completion capture yields. - // The provider transition is staged, so the live tombstone is intact. - return; - } - } - if (refreshPendingFinalDeliveryPayload(entry)) { - mutated = true; - } - - const opaqueTaskArbitration = - provisionalKillSnapshot !== undefined && - postCaptureTaskResolution?.lookup === "unavailable"; - // A steer abort ends one agent run but continues the same detached task. - // The successor must remain able to publish its eventual terminal state. - if (provisionalKillSnapshot) { - const finalizedTasks = safeFinalizeSubagentTaskRun({ - entry, - outcome: executionOutcome, - taskResolution: postCaptureTaskResolution, - }); - const taskWasAbsent = - postCaptureTaskResolution?.lookup === "available" && - postCaptureTaskResolution.task === undefined; - if ((!finalizedTasks || finalizedTasks.length === 0) && !taskWasAbsent) { - if (opaqueTaskArbitration) { - // The optional lookup cannot prove cancellation. Let the legacy - // runtime's own finalizer decide whether provider completion won. - return; - } - const latestTaskResolution = params.resolveSubagentTask(provisionalKillSnapshot); - const latestTask = latestTaskResolution.task; - const stableTaskCancellation = - latestTask?.status === "cancelled" && !isProvisionalSubagentKillTask(latestTask); - const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(provisionalKillSnapshot); - const completionPredatesCancellation = - typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; - if (stableTaskCancellation && !completionPredatesCancellation) { - return; - } - throw new Error("subagent task projection did not finalize"); - } - - // Task results do not auto-publish for subagents. Commit that durable, - // idempotent projection first: after a crash the persisted kill marker - // can replay it, while the inverse ordering could strand a provisional task. - entry.browserCleanupDispatchedAt ??= currentEntry.browserCleanupDispatchedAt; - if (currentEntry.killReconciliation?.suppressTaskDelivery === true) { - entry.suppressCompletionDelivery = true; - } - const liveBeforeCommit = structuredClone(currentEntry); - restoreEntrySnapshot(entry); - entry = currentEntry; try { params.persistOrThrow(completeParams.runId); - } catch (error) { - restoreEntrySnapshot(liveBeforeCommit); - throw error; - } - // A provider result supersedes provisional cleanup only after both - // durable owners accept it. Rejected callbacks leave the kill tail live. - cleanupGenerations.set(entry, (cleanupGenerations.get(entry) ?? 0) + 1); - } else { - try { - if (mutated) { - params.persistOrThrow(completeParams.runId); - } } catch (error) { restoreEntrySnapshot(entrySnapshot); throw error; } - if (!suppressTaskFinalization) { - safeFinalizeSubagentTaskRun({ entry, outcome: executionOutcome }); - } + // Any later delivery-payload write rolls back to this durable owner, + // never to the pre-recovery running row. + entrySnapshot = structuredClone(entry); + mutated = false; } - terminalGeneration = (terminalGenerations.get(entry) ?? 0) + 1; - terminalGenerations.set(entry, terminalGeneration); - } finally { - // Only the canonical state/capture transition is serialized. Cleanup - // remains re-entrant so a stalled browser close cannot strand a duplicate callback. - releaseCompletionLock(); } - - if (!entry) { - return; - } - const refreshSessionEffectsSuppression = () => { - if (!shouldSuppressSubagentRecoverySessionEffects(entry)) { - return false; - } - suppressSessionEffects = true; - if (entry.execution.suppressSessionEffects !== true) { - const previousExecution = entry.execution; - entry.execution = { - ...entry.execution, - suppressSessionEffects: true, - }; - try { - params.persistOrThrow(completeParams.runId); - } catch (error) { - entry.execution = previousExecution; - suppressSessionEffects = false; - throw error; - } - } - return true; - }; - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; - } - const retireSupersededSession = async (currentEntry: SubagentRunRecord) => { - if (completionReason !== SUBAGENT_ENDED_REASON_KILLED) { - await params.retireSupersededRun(completeParams.runId, currentEntry); - } - }; - sessionSuperseded = sessionSuperseded || newerGenerationOwnsSession(entry); - if (sessionSuperseded) { - // This callback belongs to an older run that shared the session key. - // Update only its task projection; the newer generation owns all session effects. - await retireSupersededSession(entry); - return; - } - if (entry.collect) { - releaseSwarmRun(entry.schedulerSlotId ?? entry.runId); - } - refreshSessionEffectsSuppression(); - const isProvisionalKill = entry.killReconciliation !== undefined; - // Record only the current, non-superseded callback with a committed outcome; the - // run-terminal dedupe key is first-write-wins, so a provisional/stale status here - // would permanently mislabel the signal-log terminal kind. - const outcomeStatus = entry.execution.outcome?.status; + sessionSuperseded = context.newerGenerationOwnsSession(currentEntry); if ( - !suppressSessionEffects && - !isProvisionalKill && - outcomeStatus && - outcomeStatus !== "unknown" + completeParams.reason === SUBAGENT_ENDED_REASON_KILLED && + entry.killIntent === undefined && + entry.endedReason !== undefined && + entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED && + entry.execution.outcome !== undefined ) { - recordSubagentTerminalState({ - childSessionKey: entry.childSessionKey, - runId: entry.runId, - requesterSessionKey: entry.requesterSessionKey, - outcomeStatus, + // Any finalized provider outcome is canonical. A delayed abort listener + // must not replace success, failure, or timeout with a killed marker. + return; + } + let requestedEndedAt = + typeof completeParams.endedAt === "number" ? completeParams.endedAt : Date.now(); + if ( + shouldPreservePublishedExplicitRunTimeout({ + entry, + }) + ) { + return; + } + const shouldDrainExistingTerminal = + recoveryRequested || + isOlderEquivalentTerminalCallback({ + entry, + endedAt: requestedEndedAt, + outcome: completeParams.outcome, + reason: completeParams.reason, }); + if (shouldDrainExistingTerminal) { + // Preserve the newer canonical timing while allowing this duplicate + // caller to rescue a stalled cleanup and delivery tail. + requestedEndedAt = entry.execution.endedAt!; + completionReason = entry.endedReason ?? completeParams.reason; + } + let endedAt = requestedEndedAt; + let completionOutcome = + shouldDrainExistingTerminal && entry.execution.outcome + ? entry.execution.outcome + : completeParams.outcome; + const liveStructuredOutput = entry.collect + ? (entry.structuredOutput ?? + peekSwarmStructuredOutput(entry.runId) ?? + (entry.swarmRunId ? peekSwarmStructuredOutput(entry.swarmRunId) : undefined)) + : undefined; + if (!entry.structuredOutput && liveStructuredOutput) { + entry.structuredOutput = liveStructuredOutput; + mutated = true; + } + if ( + liveStructuredOutput?.structured !== undefined && + completionOutcome.status === "error" && + completionOutcome.error === "completed" + ) { + // Tool-only collector turns use this runner sentinel after the result is + // durably recorded. Normalize before every task/session/hook projection. + completionOutcome = { status: "ok" }; + completionReason = SUBAGENT_ENDED_REASON_COMPLETE; + } + const observedStartedAt = + !shouldDrainExistingTerminal && + typeof completeParams.startedAt === "number" && + Number.isFinite(completeParams.startedAt) + ? completeParams.startedAt + : undefined; + const expiredDeadlineMs = recoveryRequested + ? undefined + : resolveExpiredExplicitRunDeadlineMs({ + entry, + nextEndedAt: endedAt, + observedStartedAt, + }); + if (expiredDeadlineMs !== undefined) { + endedAt = expiredDeadlineMs; + completionOutcome = { status: "timeout" }; + completionReason = SUBAGENT_ENDED_REASON_COMPLETE; + } + const killIntent = entry.killIntent; + if (killIntent) { + if (completionReason !== SUBAGENT_ENDED_REASON_KILLED && endedAt < killIntent.requestedAt) { + entry.killIntent = undefined; + } else { + const killOwnsCurrentLifecycle = + killIntent.lifecycleGeneration !== undefined && + isAgentEventLifecycleGenerationCurrent(killIntent.lifecycleGeneration); + completionReason = SUBAGENT_ENDED_REASON_KILLED; + completionOutcome = { status: "error", error: killIntent.reason }; + entry.killIntent = undefined; + if (killOwnsCurrentLifecycle) { + suppressSessionEffects = false; + entry.execution = { + ...entry.execution, + lifecycleGeneration: killIntent.lifecycleGeneration, + restartRecovery: undefined, + suppressSessionEffects: undefined, + }; + } + entry.killReconciliation = { + killedAt: killIntent.requestedAt, + suppressTaskDelivery: killIntent.suppressTaskDelivery === true ? true : undefined, + }; + } + mutated = true; + } + if ( + completionReason !== SUBAGENT_ENDED_REASON_KILLED && + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + entry.killReconciliation === undefined + ) { + // Only current-version provisional kills carry reconciliation state. + // Legacy or already-stabilized killed rows are terminal cancellation. + return; + } + const isSteerRestartKill = + completeParams.reason === SUBAGENT_ENDED_REASON_KILLED && + entry.suppressAnnounceReason === "steer-restart"; + suppressTaskFinalization = isSteerRestartKill; + if (completionReason === SUBAGENT_ENDED_REASON_KILLED && !isSteerRestartKill) { + entry.suppressAnnounceReason = "killed"; + entry.killReconciliation ??= { + killedAt: requestedEndedAt, + }; + mutated = true; } - if (!suppressSessionEffects) { - try { - const assertSessionEffectsOwnerCurrent = () => { - if ( - !isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration) || - newerGenerationOwnsSession(entry) || - shouldSuppressSubagentRecoverySessionEffects(entry) - ) { - throw new Error("subagent session-effects owner retired before session commit"); - } - }; - await persistSubagentSessionTiming(entry, { - // Recheck while patchSessionEntry owns its write lock so this old - // completion cannot commit after a synchronous ownership transfer. - isCurrentGeneration: () => - isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration) && - !newerGenerationOwnsSession(entry) && - !shouldSuppressSubagentRecoverySessionEffects(entry), - assertCommitAllowed: assertSessionEffectsOwnerCurrent, - }); - } catch (err) { - params.warn("failed to persist subagent session timing", { - err, - runId: entry.runId, - childSessionKey: entry.childSessionKey, - }); + if ( + completionReason !== SUBAGENT_ENDED_REASON_KILLED && + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + entry.killReconciliation !== undefined + ) { + const killReconciliation = entry.killReconciliation; + const taskResolution = params.resolveSubagentTask(entry); + const stableTaskCancellation = + taskResolution.lookup === "available" && + taskResolution.task?.status === "cancelled" && + !isProvisionalSubagentKillTask(taskResolution.task); + const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(entry); + const completionPredatesCancellation = + typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; + if (stableTaskCancellation && !completionPredatesCancellation) { + // tasks.cancel promotes the provisional marker to durable operator + // intent. Only an already-durable earlier completion may reopen it. + return; + } + provisionalKillSnapshot = structuredClone(currentEntry); + // The sweeper uses marker identity to reject a concurrently replaced + // kill generation. A completion rollback must retain the same marker. + provisionalKillSnapshot.killReconciliation = killReconciliation; + // Completion capture yields. Stage the provider result off-registry so + // an unrelated persistence write cannot publish a tentative winner. + entry = structuredClone(currentEntry); + entry.suppressCompletionDelivery = + killReconciliation.suppressTaskDelivery === true ? true : undefined; + entry.suppressAnnounceReason = undefined; + entry.killReconciliation = undefined; + entry.cleanupHandled = false; + entry.cleanupCompletedAt = undefined; + clearDeliveryState(entry); + mutated = true; + } + + if (observedStartedAt !== undefined && entry.execution.startedAt !== observedStartedAt) { + entry.execution = { ...entry.execution, startedAt: observedStartedAt }; + if (typeof entry.sessionStartedAt !== "number") { + entry.sessionStartedAt = observedStartedAt; + } + mutated = true; + } + + if ( + completionReason === SUBAGENT_ENDED_REASON_COMPLETE && + completionOutcome.status !== "error" && + provisionalKillSnapshot !== undefined + ) { + // A killed lifecycle may freeze an empty result before the canonical end + // wins. Preserve any reply already captured by an earlier successful callback. + const completion = ensureCompletionState(entry); + const hasCapturedReply = + typeof completion.resultText === "string" && completion.resultText.trim().length > 0; + if ( + !hasCapturedReply && + (completion.resultText !== undefined || completion.capturedAt !== undefined) + ) { + completion.resultText = undefined; + completion.capturedAt = undefined; + mutated = true; } } - refreshSessionEffectsSuppression(); - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; + const outcome = + recoveryRequested && entry.execution.outcome + ? entry.execution.outcome + : withSubagentOutcomeTiming(completionOutcome, { + startedAt: entry.execution.startedAt, + endedAt, + }); + const executionOutcome = recoveryRequested ? (entry.execution.outcome ?? outcome) : outcome; + const retainedRestartRecovery = suppressSessionEffects + ? entry.execution.restartRecovery + : undefined; + if ( + entry.execution.status !== "terminal" || + entry.execution.endedAt !== endedAt || + entry.execution.outcome !== executionOutcome || + entry.execution.restartRecovery !== retainedRestartRecovery || + entry.execution.suppressSessionEffects !== (suppressSessionEffects ? true : undefined) + ) { + entry.execution = { + ...entry.execution, + status: "terminal", + endedAt, + outcome: executionOutcome, + restartRecovery: retainedRestartRecovery, + suppressSessionEffects: suppressSessionEffects ? true : undefined, + }; + mutated = true; } - if (newerGenerationOwnsSession(entry)) { - await retireSupersededSession(entry); - return; + if (entry.endedReason !== completionReason) { + entry.endedReason = completionReason; + mutated = true; + } + if (completionReason === SUBAGENT_ENDED_REASON_KILLED && entry.terminalOwner !== undefined) { + entry.terminalOwner = undefined; + mutated = true; + } + if (entry.pauseReason !== undefined) { + entry.pauseReason = undefined; + mutated = true; } - const suppressedForSteerRestart = params.suppressAnnounceForSteerRestart(entry); - // Recovery persists its terminal state before draining this callback, so an - // unchanged row still needs its first session-status and progress events. - const shouldPublishTerminalStatus = - mutated || - (completeParams.recoverInterrupted === true && - !isProvisionalKill && - !progressEndedEntries.has(entry)); - if (shouldPublishTerminalStatus && !suppressedForSteerRestart && !suppressSessionEffects) { - emitSessionLifecycleEvent({ - sessionKey: entry.childSessionKey, - reason: "subagent-status", - parentSessionKey: entry.requesterSessionKey, - label: entry.label, + if (completeParams.completionSnapshot) { + const completion = ensureCompletionState(entry); + if ( + completion.resultText !== completeParams.completionSnapshot.resultText || + completion.capturedAt !== completeParams.completionSnapshot.capturedAt + ) { + completion.resultText = completeParams.completionSnapshot.resultText; + completion.capturedAt = completeParams.completionSnapshot.capturedAt; + mutated = true; + } + } + + if (completeParams.terminalReply) { + const completion = ensureCompletionState(entry); + const terminalReply = mergeAgentRunTerminalReplySnapshot( + completion.terminalReply, + completeParams.terminalReply, + ); + if ( + terminalReply && + JSON.stringify(terminalReply) !== JSON.stringify(completion.terminalReply) + ) { + completion.terminalReply = terminalReply; + completion.resultText = + terminalReply.disposition === "visible" + ? terminalReply.text + : terminalReply.disposition === "silent" + ? SILENT_REPLY_TOKEN + : null; + completion.capturedAt = endedAt; + mutated = true; + } + } + + // A newer generation may share the session key. Its transcript/reply is + // not evidence for this older run, so reconcile only the terminal task state. + if (recoveryRequested || sessionSuperseded) { + const completion = ensureCompletionState(entry); + if (completion.resultText === undefined) { + completion.resultText = null; + completion.capturedAt = Date.now(); + mutated = true; + } + } else { + const didFreezeResult = await freezeRunResultAtCompletion(context, entry, executionOutcome); + sessionSuperseded = context.newerGenerationOwnsSession(entry); + if (sessionSuperseded) { + const completion = ensureCompletionState(entry); + completion.resultText = null; + completion.capturedAt = Date.now(); + mutated = true; + } else if (didFreezeResult) { + mutated = true; + } + } + if ( + entry.collect + ? updateSwarmCollectorCompletion(entry, params.getRuntimeConfig()) + : updateSubagentArchiveAtMs(entry, params.getRuntimeConfig()) + ) { + mutated = true; + } + if (provisionalKillSnapshot) { + // Keep the tombstone's superseded generation boundary through task + // commit. Clearing it on the canonical registry row must not let a + // late old-run result select a newer task sharing the session key. + const taskResolution = params.resolveSubagentTask(provisionalKillSnapshot); + postCaptureTaskResolution = taskResolution; + const stableTaskCancellation = + taskResolution.lookup === "available" && + taskResolution.task?.status === "cancelled" && + !isProvisionalSubagentKillTask(taskResolution.task); + const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(provisionalKillSnapshot); + const completionPredatesCancellation = + typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; + if (stableTaskCancellation && !completionPredatesCancellation) { + // Cancellation can become durable while completion capture yields. + // The provider transition is staged, so the live tombstone is intact. + return; + } + } + if (refreshPendingFinalDeliveryPayload(entry)) { + mutated = true; + } + + const opaqueTaskArbitration = + provisionalKillSnapshot !== undefined && postCaptureTaskResolution?.lookup === "unavailable"; + // A steer abort ends one agent run but continues the same detached task. + // The successor must remain able to publish its eventual terminal state. + if (provisionalKillSnapshot) { + const finalizedTasks = safeFinalizeSubagentTaskRun(params, { + entry, + outcome: executionOutcome, + taskResolution: postCaptureTaskResolution, }); - // The enclosing steer/session-effects guard admits only the real terminal generation. - if (!isProvisionalKill && !progressEndedEntries.has(entry)) { - progressEndedEntries.add(entry); - await params.emitSubagentProgressEndedForRun(entry); - refreshSessionEffectsSuppression(); - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + const taskWasAbsent = + postCaptureTaskResolution?.lookup === "available" && + postCaptureTaskResolution.task === undefined; + if ((!finalizedTasks || finalizedTasks.length === 0) && !taskWasAbsent) { + if (opaqueTaskArbitration) { + // The optional lookup cannot prove cancellation. Let the legacy + // runtime's own finalizer decide whether provider completion won. return; } + const latestTaskResolution = params.resolveSubagentTask(provisionalKillSnapshot); + const latestTask = latestTaskResolution.task; + const stableTaskCancellation = + latestTask?.status === "cancelled" && !isProvisionalSubagentKillTask(latestTask); + const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(provisionalKillSnapshot); + const completionPredatesCancellation = + typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; + if (stableTaskCancellation && !completionPredatesCancellation) { + return; + } + throw new Error("subagent task projection did not finalize"); + } + + // Task results do not auto-publish for subagents. Commit that durable, + // idempotent projection first: after a crash the persisted kill marker + // can replay it, while the inverse ordering could strand a provisional task. + entry.browserCleanupDispatchedAt ??= currentEntry.browserCleanupDispatchedAt; + if (currentEntry.killReconciliation?.suppressTaskDelivery === true) { + entry.suppressCompletionDelivery = true; + } + const liveBeforeCommit = structuredClone(currentEntry); + restoreEntrySnapshot(entry); + entry = currentEntry; + try { + params.persistOrThrow(completeParams.runId); + } catch (error) { + restoreEntrySnapshot(liveBeforeCommit); + throw error; + } + // A provider result supersedes provisional cleanup only after both + // durable owners accept it. Rejected callbacks leave the kill tail live. + context.bumpCleanupGeneration(entry); + } else { + try { + if (mutated) { + params.persistOrThrow(completeParams.runId); + } + } catch (error) { + restoreEntrySnapshot(entrySnapshot); + throw error; + } + if (!suppressTaskFinalization) { + safeFinalizeSubagentTaskRun(params, { + entry, + outcome: executionOutcome, + }); } } - const shouldEmitEndedHook = - !suppressedForSteerRestart && - !isProvisionalKill && - !suppressSessionEffects && - params.shouldEmitEndedHookForRun({ - entry, - reason: completionReason, - }); - const shouldDeferEndedHook = - shouldEmitEndedHook && - completeParams.triggerCleanup && - entry.expectsCompletionMessage === true && - !suppressedForSteerRestart; - if (!shouldDeferEndedHook && shouldEmitEndedHook) { - await params.emitSubagentEndedHookForRun({ - entry, - reason: completionReason, - sendFarewell: completeParams.sendFarewell, - accountId: completeParams.accountId, - isCurrent: () => - isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration) && - !newerGenerationOwnsSession(entry) && - !shouldSuppressSubagentRecoverySessionEffects(entry), - }); - refreshSessionEffectsSuppression(); - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; - } - if (newerGenerationOwnsSession(entry)) { - await retireSupersededSession(entry); - return; - } - } + terminalGeneration = context.bumpTerminalGeneration(entry); + } finally { + // Only the canonical state/capture transition is serialized. Cleanup + // remains re-entrant so a stalled browser close cannot strand a duplicate callback. + releaseCompletionLock(); + } - refreshSessionEffectsSuppression(); - await terminalCleanup.complete({ - completeParams, - entry, - isProvisionalKill, - retireSupersededSession, - suppressedForSteerRestart, - suppressSessionEffects, - terminalGeneration, - }); - }; - - return { completeSubagentRunAttempt }; + if (!entry) { + return; + } + await completeTerminalEffects(context, { + completeParams, + completionReason, + entry, + mutated, + sessionSuperseded, + suppressSessionEffects, + terminalGeneration, + loadCleanupBrowserSessionsForLifecycleEnd: + params.loadCleanupBrowserSessionsForLifecycleEnd ?? loadCleanupBrowserSessionsForLifecycleEnd, + }); } diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-context.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-context.ts new file mode 100644 index 000000000000..a9d818fc1d87 --- /dev/null +++ b/src/agents/subagents/registry/subagent-registry-lifecycle-context.ts @@ -0,0 +1,119 @@ +// This type-only leaf exists solely to keep lifecycle sibling modules from importing the controller. +// Keeping the controller out of their dependency graph satisfies the architecture cycle gate. +import type { cleanupBrowserSessionsForLifecycleEnd } from "../../../browser-lifecycle-cleanup.js"; +import type { OpenClawConfig } from "../../../config/types.openclaw.js"; +import type { callGateway as defaultCallGateway } from "../../../gateway/call.js"; +import type { DetachedTaskFindResult } from "../../../tasks/detached-task-runtime-contract.js"; +import type { SubagentLifecycleEndedReason } from "./subagent-lifecycle-events.js"; +import type { SubagentRunRecord } from "./subagent-registry.types.js"; + +type CaptureSubagentCompletionReply = + (typeof import("../announce/subagent-announce.js"))["captureSubagentCompletionReply"]; +type RunSubagentAnnounceFlow = + (typeof import("../announce/subagent-announce.js"))["runSubagentAnnounceFlow"]; +type MaybeWakeRequesterAfterAllChildrenSettled = + (typeof import("../announce/subagent-announce.requester-settle-wake.js"))["maybeWakeRequesterAfterAllChildrenSettled"]; +type BrowserCleanup = typeof cleanupBrowserSessionsForLifecycleEnd; + +export type SubagentLifecycleOptions = { + runs: Map; + resumedRuns: Set; + subagentAnnounceTimeoutMs: number; + getRuntimeConfig(): OpenClawConfig; + persist(...runIds: string[]): void; + persistOrThrow(...runIds: string[]): void; + clearPendingLifecycleError(runId: string): void; + countPendingDescendantRuns(rootSessionKey: string): number; + suppressAnnounceForSteerRestart(entry?: SubagentRunRecord): boolean; + resolveSubagentTask(entry: SubagentRunRecord): DetachedTaskFindResult; + shouldEmitEndedHookForRun(args: { + entry: SubagentRunRecord; + reason: SubagentLifecycleEndedReason; + }): boolean; + emitSubagentEndedHookForRun(args: { + entry: SubagentRunRecord; + reason?: SubagentLifecycleEndedReason; + sendFarewell?: boolean; + accountId?: string; + isCurrent?: () => boolean; + }): Promise; + emitSubagentProgressEndedForRun(entry: SubagentRunRecord): Promise; + notifyContextEngineSubagentEnded( + args: { + childSessionKey: string; + reason: "completed" | "deleted"; + agentDir?: string; + workspaceDir?: string; + }, + options?: { isCurrent?: () => boolean }, + ): Promise; + retireSupersededRun(runId: string, entry: SubagentRunRecord): Promise; + resumeSubagentRun(runId: string): void; + callGateway: typeof defaultCallGateway; + captureSubagentCompletionReply: CaptureSubagentCompletionReply; + cleanupBrowserSessionsForLifecycleEnd?: BrowserCleanup; + loadCleanupBrowserSessionsForLifecycleEnd?: () => Promise; + runSubagentAnnounceFlow: RunSubagentAnnounceFlow; + maybeWakeRequesterAfterAllChildrenSettled: MaybeWakeRequesterAfterAllChildrenSettled; + warn(message: string, meta?: Record): void; +}; + +export interface SubagentLifecycleCommonContext { + readonly options: SubagentLifecycleOptions; + newerGenerationOwnsSession(entry: SubagentRunRecord): boolean; +} + +export interface SubagentLifecycleCompletionContext extends SubagentLifecycleCommonContext { + acquireTerminalCompletionLock(runId: string): Promise<() => void>; + bumpCleanupGeneration(entry: SubagentRunRecord): number; + bumpTerminalGeneration(entry: SubagentRunRecord): number; + hasProgressEnded(entry: SubagentRunRecord): boolean; + isTerminalCallbackCurrent(runId: string, entry: SubagentRunRecord, generation: number): boolean; + markProgressEnded(entry: SubagentRunRecord): void; + startSubagentAnnounceCleanupFlow(runId: string, entry: SubagentRunRecord): boolean; +} + +export interface SubagentLifecycleCleanupContext extends SubagentLifecycleCommonContext { + addScheduledResumeTimer(timer: ReturnType): void; + bumpCleanupGeneration(entry: SubagentRunRecord): number; + clearCleanupFailureCount(entry: SubagentRunRecord): void; + deleteScheduledResumeTimer(timer: ReturnType): void; + incrementCleanupFailureCount(entry: SubagentRunRecord): number; + isCleanupAttemptCurrent(runId: string, entry: SubagentRunRecord, generation: number): boolean; + isCleanupGeneration(entry: SubagentRunRecord, generation: number): boolean; + isCleanupGenerationCurrent(runId: string, entry: SubagentRunRecord, generation: number): boolean; + isEndedHookOwnerCurrent(runId: string, entry: SubagentRunRecord): boolean; + startSubagentAnnounceCleanupFlow(runId: string, entry: SubagentRunRecord): boolean; +} + +export interface SubagentLifecycleAnnounceCleanupContext + extends SubagentLifecycleCleanupContext, SubagentLifecycleWakeContext { + completeCleanupBookkeeping(args: CleanupBookkeepingParams): void; +} + +export interface SubagentLifecycleWakeContext extends SubagentLifecycleCommonContext { + deleteRequesterSettleWakeTimer(runId: string): void; + getRequesterSettleWakeTimer(runId: string): ScheduledRequesterSettleWake | undefined; + hasScheduledRequesterSettleWakeRun(runId: string): boolean; + markRequesterSettleWakeRearm(runId: string): void; + markRequesterSettleWakeRunScheduled(runId: string): void; + setRequesterSettleWakeTimer(runId: string, value: ScheduledRequesterSettleWake): void; + takeRequesterSettleWakeRearm(runId: string): boolean; + unmarkRequesterSettleWakeRunScheduled(runId: string): void; +} + +export type CleanupBookkeepingParams = { + runId: string; + entry: SubagentRunRecord; + cleanup: "delete" | "keep"; + completedAt: number; + preserveTranscript?: boolean; + provisionalKill?: boolean; + skipRequesterSettleWake?: boolean; +}; + +export type ScheduledRequesterSettleWake = { + timer: ReturnType; + deadline: number; + rearmGeneration?: number; +}; diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-contracts.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-contracts.ts deleted file mode 100644 index cff7b1ebcd25..000000000000 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-contracts.ts +++ /dev/null @@ -1,85 +0,0 @@ -import type { cleanupBrowserSessionsForLifecycleEnd } from "../../../browser-lifecycle-cleanup.js"; -import type { OpenClawConfig } from "../../../config/types.openclaw.js"; -import type { callGateway as defaultCallGateway } from "../../../gateway/call.js"; -import type { DetachedTaskFindResult } from "../../../tasks/detached-task-runtime-contract.js"; -import type { SubagentLifecycleEndedReason } from "./subagent-lifecycle-events.js"; -import type { SubagentRunRecord } from "./subagent-registry.types.js"; - -type CaptureSubagentCompletionReply = - (typeof import("../announce/subagent-announce.js"))["captureSubagentCompletionReply"]; -type RunSubagentAnnounceFlow = - (typeof import("../announce/subagent-announce.js"))["runSubagentAnnounceFlow"]; -type MaybeWakeRequesterAfterAllChildrenSettled = - (typeof import("../announce/subagent-announce.requester-settle-wake.js"))["maybeWakeRequesterAfterAllChildrenSettled"]; -export type SubagentRegistryLifecycleParams = { - runs: Map; - resumedRuns: Set; - subagentAnnounceTimeoutMs: number; - getRuntimeConfig(): OpenClawConfig; - persist(...runIds: string[]): void; - persistOrThrow(...runIds: string[]): void; - clearPendingLifecycleError(runId: string): void; - countPendingDescendantRuns(rootSessionKey: string): number; - suppressAnnounceForSteerRestart(entry?: SubagentRunRecord): boolean; - resolveSubagentTask(entry: SubagentRunRecord): DetachedTaskFindResult; - shouldEmitEndedHookForRun(args: { - entry: SubagentRunRecord; - reason: SubagentLifecycleEndedReason; - }): boolean; - emitSubagentEndedHookForRun(args: { - entry: SubagentRunRecord; - reason?: SubagentLifecycleEndedReason; - sendFarewell?: boolean; - accountId?: string; - isCurrent?: () => boolean; - }): Promise; - emitSubagentProgressEndedForRun(entry: SubagentRunRecord): Promise; - notifyContextEngineSubagentEnded( - args: { - childSessionKey: string; - reason: "completed" | "deleted"; - agentDir?: string; - workspaceDir?: string; - }, - options?: { isCurrent?: () => boolean }, - ): Promise; - retireSupersededRun(runId: string, entry: SubagentRunRecord): Promise; - resumeSubagentRun(runId: string): void; - callGateway: typeof defaultCallGateway; - captureSubagentCompletionReply: CaptureSubagentCompletionReply; - cleanupBrowserSessionsForLifecycleEnd?: typeof cleanupBrowserSessionsForLifecycleEnd; - runSubagentAnnounceFlow: RunSubagentAnnounceFlow; - maybeWakeRequesterAfterAllChildrenSettled: MaybeWakeRequesterAfterAllChildrenSettled; - warn(message: string, meta?: Record): void; -}; - -export type SubagentRegistryLifecycleState = { - scheduledResumeTimers: Set>; - pendingRequesterSettleWakeRearms: Set; - scheduledRequesterSettleWakeRuns: Set; - scheduledRequesterSettleWakeTimers: Map< - string, - { - timer: ReturnType; - deadline: number; - rearmGeneration?: number; - } - >; - terminalCompletionLocks: Map>; - terminalGenerations: WeakMap; - cleanupGenerations: WeakMap; - progressEndedEntries: WeakSet; -}; - -export function createSubagentRegistryLifecycleState(): SubagentRegistryLifecycleState { - return { - scheduledResumeTimers: new Set(), - pendingRequesterSettleWakeRearms: new Set(), - scheduledRequesterSettleWakeRuns: new Set(), - scheduledRequesterSettleWakeTimers: new Map(), - terminalCompletionLocks: new Map(), - terminalGenerations: new WeakMap(), - cleanupGenerations: new WeakMap(), - progressEndedEntries: new WeakSet(), - }; -} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-delivery.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-delivery.ts index 45c88e29f344..d4ab859efe4b 100644 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-delivery.ts +++ b/src/agents/subagents/registry/subagent-registry-lifecycle-delivery.ts @@ -1,10 +1,12 @@ import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { resolveStorePath } from "../../../config/sessions/paths.js"; import { loadSessionEntryReadOnly, type SessionTranscriptRuntimeTarget, } from "../../../config/sessions/session-accessor.js"; import { resolveSessionStorePathForScope } from "../../../config/sessions/session-store-path.js"; +import { formatErrorMessage, readErrorName } from "../../../infra/errors.js"; import { resolveAgentIdFromSessionKey } from "../../../routing/session-key.js"; import { extractTextFromChatContent } from "../../../shared/chat-content.js"; import type { DetachedTaskFindResult } from "../../../tasks/detached-task-runtime-contract.js"; @@ -31,475 +33,486 @@ import { import type { SubagentLifecycleEndedReason } from "./subagent-lifecycle-events.js"; import { resolveFinalizedSubagentTaskState } from "./subagent-registry-completion.js"; import { capFrozenResultText } from "./subagent-registry-helpers.js"; -import type { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; import type { - SubagentRegistryLifecycleParams, - SubagentRegistryLifecycleState, -} from "./subagent-registry-lifecycle-contracts.js"; + SubagentLifecycleCommonContext, + SubagentLifecycleOptions, +} from "./subagent-registry-lifecycle-context.js"; import type { PendingFinalDeliveryPayload, SubagentRunRecord } from "./subagent-registry.types.js"; import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; const DELIVERY_MIRROR_HISTORY_MAX_CHARS = 128 * 1024; -export function createSubagentRegistryLifecycleDelivery( - params: SubagentRegistryLifecycleParams, - _state: SubagentRegistryLifecycleState, - common: ReturnType, -) { - const { newerGenerationOwnsSession, buildSafeLifecycleErrorMeta, maskRunId, maskSessionKey } = - common; +export function buildSafeLifecycleErrorMeta(error: unknown): Record { + const message = formatErrorMessage(error); + const name = readErrorName(error); + return name ? { name, message } : { message }; +} - const formatAnnounceDeliveryError = (delivery: SubagentAnnounceDeliveryResult): string => { - const errors = [ - delivery.error, - delivery.reason, - ...(delivery.phases ?? []).map((phase) => - phase.error ? `${phase.phase}: ${phase.error}` : undefined, - ), - ] - .map((value) => value?.trim()) - .filter((value): value is string => Boolean(value)); - return errors.length > 0 - ? uniqueStrings(errors).join("; ") - : `delivery path ${delivery.path} did not complete`; - }; +export function maskLifecycleIdentifier(value: string, kind: "run" | "session"): string { + const trimmed = value.trim(); + if (!trimmed) { + return "unknown"; + } + return kind === "session" + ? `${trimmed.split(":").slice(0, 2).join(":") || "session"}:…` + : trimmed.length <= 8 + ? "***" + : `${sliceUtf16Safe(trimmed, 0, 4)}…${sliceUtf16Safe(trimmed, -4)}`; +} - const recordAnnounceDeliveryResult = ( - entry: SubagentRunRecord, - delivery: SubagentAnnounceDeliveryResult, - ) => { - const deliveryState = ensureDeliveryState(entry); - if (typeof delivery.enqueuedAt === "number") { - deliveryState.enqueuedAt ??= delivery.enqueuedAt; - } - if (delivery.delivered) { - const deliveredAt = - typeof delivery.deliveredAt === "number" ? delivery.deliveredAt : Date.now(); - deliveryState.deliveredAt = deliveredAt; - deliveryState.lastDropReason = undefined; - } - deliveryState.disposition = - delivery.disposition ?? (delivery.delivered ? "delivered" : "retryable"); - }; +export const formatAnnounceDeliveryError = (delivery: SubagentAnnounceDeliveryResult): string => { + const errors = [ + delivery.error, + delivery.reason, + ...(delivery.phases ?? []).map((phase) => + phase.error ? `${phase.phase}: ${phase.error}` : undefined, + ), + ] + .map((value) => value?.trim()) + .filter((value): value is string => Boolean(value)); + return errors.length > 0 + ? uniqueStrings(errors).join("; ") + : `delivery path ${delivery.path} did not complete`; +}; - const hasPriorRequesterDeliveryMirror = async (entry: SubagentRunRecord): Promise => { - const completion = ensureCompletionState(entry); - const expectedText = extractTextFromChatContent(completion.resultText, { joinWith: "" }); - if (entry.expectsCompletionMessage !== true || expectedText == null) { - return false; - } - const mirrorNotBefore = entry.execution.startedAt ?? entry.createdAt; - const mirrorNotAfter = Date.now() + 30_000; - const expectedIdempotencyKey = buildAnnounceIdempotencyKey( - buildAnnounceIdFromChildRun({ - childSessionKey: entry.childSessionKey, - childRunId: entry.runId, - }), - ); - const isExpectedMirrorIdempotencyKey = (value: unknown): boolean => - typeof value === "string" && - (value === expectedIdempotencyKey || - value.startsWith(`${expectedIdempotencyKey}:internal-source-reply:`) || - value.startsWith(`${expectedIdempotencyKey}:message-tool:internal-source-reply:`) || - value.startsWith(`${entry.runId}:message-tool:`) || - value.startsWith(`${entry.runId}:internal-source-reply:`)); - try { - const history = await params.callGateway<{ - messages?: unknown[]; - }>({ - method: "chat.history", - params: { - sessionKey: entry.requesterSessionKey, - limit: 25, - maxChars: DELIVERY_MIRROR_HISTORY_MAX_CHARS, - }, - timeoutMs: 5_000, - }); - const mirror = history.messages?.find((message) => { - if (!message || typeof message !== "object") { - return false; - } - const record = message as Record; - const timestamp = record.timestamp; - if ( - typeof timestamp !== "number" || - !Number.isFinite(timestamp) || - timestamp < mirrorNotBefore || - timestamp > mirrorNotAfter || - !isExpectedMirrorIdempotencyKey(record.idempotencyKey) - ) { - return false; - } - const text = extractTextFromChatContent(record.content, { joinWith: "" }); - return ( - record.role === "assistant" && - record.provider === "openclaw" && - record.model === "delivery-mirror" && - text === expectedText - ); - }); - if (mirror) { - ensureDeliveryState(entry).deliveredAt = (mirror as { timestamp: number }).timestamp; +export const recordAnnounceDeliveryResult = ( + entry: SubagentRunRecord, + delivery: SubagentAnnounceDeliveryResult, +) => { + const deliveryState = ensureDeliveryState(entry); + if (typeof delivery.enqueuedAt === "number") { + deliveryState.enqueuedAt ??= delivery.enqueuedAt; + } + if (delivery.delivered) { + const deliveredAt = + typeof delivery.deliveredAt === "number" ? delivery.deliveredAt : Date.now(); + deliveryState.deliveredAt = deliveredAt; + deliveryState.lastDropReason = undefined; + } + deliveryState.disposition = + delivery.disposition ?? (delivery.delivered ? "delivered" : "retryable"); +}; + +export const hasPriorRequesterDeliveryMirror = async ( + params: SubagentLifecycleOptions, + entry: SubagentRunRecord, +): Promise => { + const completion = ensureCompletionState(entry); + const expectedText = extractTextFromChatContent(completion.resultText, { joinWith: "" }); + if (entry.expectsCompletionMessage !== true || expectedText == null) { + return false; + } + const mirrorNotBefore = entry.execution.startedAt ?? entry.createdAt; + const mirrorNotAfter = Date.now() + 30_000; + const expectedIdempotencyKey = buildAnnounceIdempotencyKey( + buildAnnounceIdFromChildRun({ + childSessionKey: entry.childSessionKey, + childRunId: entry.runId, + }), + ); + const isExpectedMirrorIdempotencyKey = (value: unknown): boolean => + typeof value === "string" && + (value === expectedIdempotencyKey || + value.startsWith(`${expectedIdempotencyKey}:internal-source-reply:`) || + value.startsWith(`${expectedIdempotencyKey}:message-tool:internal-source-reply:`) || + value.startsWith(`${entry.runId}:message-tool:`) || + value.startsWith(`${entry.runId}:internal-source-reply:`)); + try { + const history = await params.callGateway<{ + messages?: unknown[]; + }>({ + method: "chat.history", + params: { + sessionKey: entry.requesterSessionKey, + limit: 25, + maxChars: DELIVERY_MIRROR_HISTORY_MAX_CHARS, + }, + timeoutMs: 5_000, + }); + const mirror = history.messages?.find((message) => { + if (!message || typeof message !== "object") { + return false; } - return Boolean(mirror); - } catch { - return false; + const record = message as Record; + const timestamp = record.timestamp; + if ( + typeof timestamp !== "number" || + !Number.isFinite(timestamp) || + timestamp < mirrorNotBefore || + timestamp > mirrorNotAfter || + !isExpectedMirrorIdempotencyKey(record.idempotencyKey) + ) { + return false; + } + const text = extractTextFromChatContent(record.content, { joinWith: "" }); + return ( + record.role === "assistant" && + record.provider === "openclaw" && + record.model === "delivery-mirror" && + text === expectedText + ); + }); + if (mirror) { + ensureDeliveryState(entry).deliveredAt = (mirror as { timestamp: number }).timestamp; } - }; + return Boolean(mirror); + } catch { + return false; + } +}; - const resolveSubagentTaskTarget = ( - entry: SubagentRunRecord, - resolution = params.resolveSubagentTask(entry), - ) => { - const durableTaskRunId = entry.taskRunId ?? entry.runId; - return { - runId: - resolution.lookup === "available" - ? (resolution.task?.runId ?? durableTaskRunId) - : durableTaskRunId, - sessionKey: - resolution.lookup === "available" - ? (resolution.task?.childSessionKey ?? entry.childSessionKey) - : entry.childSessionKey, - }; +const resolveSubagentTaskTarget = ( + params: SubagentLifecycleOptions, + entry: SubagentRunRecord, + resolution = params.resolveSubagentTask(entry), +) => { + const durableTaskRunId = entry.taskRunId ?? entry.runId; + return { + runId: + resolution.lookup === "available" + ? (resolution.task?.runId ?? durableTaskRunId) + : durableTaskRunId, + sessionKey: + resolution.lookup === "available" + ? (resolution.task?.childSessionKey ?? entry.childSessionKey) + : entry.childSessionKey, }; +}; - const safeSetSubagentTaskDeliveryStatus = (args: { +export const safeSetSubagentTaskDeliveryStatus = ( + params: SubagentLifecycleOptions, + args: { entry: SubagentRunRecord; deliveryStatus: Extract; deliveryError?: string; - }) => { - const target = resolveSubagentTaskTarget(args.entry); - try { - setDetachedTaskDeliveryStatusByRunId({ - runId: target.runId, - runtime: "subagent", - sessionKey: target.sessionKey, - deliveryStatus: args.deliveryStatus, - error: args.deliveryStatus === "failed" ? args.deliveryError : undefined, - }); - } catch (err) { - params.warn("failed to update subagent background task delivery state", { - error: buildSafeLifecycleErrorMeta(err), - runId: maskRunId(target.runId), - childSessionKey: maskSessionKey(target.sessionKey), - deliveryStatus: args.deliveryStatus, - }); - } - }; + }, +) => { + const target = resolveSubagentTaskTarget(params, args.entry); + try { + setDetachedTaskDeliveryStatusByRunId({ + runId: target.runId, + runtime: "subagent", + sessionKey: target.sessionKey, + deliveryStatus: args.deliveryStatus, + error: args.deliveryStatus === "failed" ? args.deliveryError : undefined, + }); + } catch (err) { + params.warn("failed to update subagent background task delivery state", { + error: buildSafeLifecycleErrorMeta(err), + runId: maskLifecycleIdentifier(target.runId, "run"), + childSessionKey: maskLifecycleIdentifier(target.sessionKey, "session"), + deliveryStatus: args.deliveryStatus, + }); + } +}; - const safeFinalizeSubagentTaskRun = (args: { +export const safeFinalizeSubagentTaskRun = ( + params: SubagentLifecycleOptions, + args: { entry: SubagentRunRecord; outcome: SubagentRunOutcome; taskResolution?: DetachedTaskFindResult; - }): ReturnType => { - const terminal = resolveFinalizedSubagentTaskState(args.entry); - if (!terminal) { - return []; - } - const target = resolveSubagentTaskTarget(args.entry, args.taskResolution); - const { status, error, terminalOutcome, ...details } = terminal; - const suppressDelivery = args.entry.suppressCompletionDelivery === true; - try { - if (status === "succeeded") { - return completeTaskRunByRunId({ - runId: target.runId, - runtime: "subagent", - sessionKey: target.sessionKey, - ...details, - terminalOutcome, - suppressDelivery, - }); - } - return failTaskRunByRunId({ + }, +): ReturnType => { + const terminal = resolveFinalizedSubagentTaskState(args.entry); + if (!terminal) { + return []; + } + const target = resolveSubagentTaskTarget(params, args.entry, args.taskResolution); + const { status, error, terminalOutcome, ...details } = terminal; + const suppressDelivery = args.entry.suppressCompletionDelivery === true; + try { + if (status === "succeeded") { + return completeTaskRunByRunId({ runId: target.runId, runtime: "subagent", sessionKey: target.sessionKey, ...details, - status, - error, + terminalOutcome, suppressDelivery, }); - } catch (err) { - params.warn("failed to finalize subagent background task state", { - error: buildSafeLifecycleErrorMeta(err), - runId: maskRunId(args.entry.runId), - childSessionKey: maskSessionKey(args.entry.childSessionKey), - outcomeStatus: args.outcome.status, - }); - return []; } - }; + return failTaskRunByRunId({ + runId: target.runId, + runtime: "subagent", + sessionKey: target.sessionKey, + ...details, + status, + error, + suppressDelivery, + }); + } catch (err) { + params.warn("failed to finalize subagent background task state", { + error: buildSafeLifecycleErrorMeta(err), + runId: maskLifecycleIdentifier(args.entry.runId, "run"), + childSessionKey: maskLifecycleIdentifier(args.entry.childSessionKey, "session"), + outcomeStatus: args.outcome.status, + }); + return []; + } +}; - const safeMarkRequiredCompletionDeliveryBlocked = (args: { +export const safeMarkRequiredCompletionDeliveryBlocked = ( + params: SubagentLifecycleOptions, + args: { entry: SubagentRunRecord; reason?: string; - }) => { - if ( - args.entry.expectsCompletionMessage !== true || - args.entry.execution.outcome?.status !== "ok" - ) { - return; - } - const endedAt = args.entry.execution.endedAt ?? Date.now(); - const terminalResult = resolveRequiredCompletionDeliveryFailureTerminalResult(args.reason); - const target = resolveSubagentTaskTarget(args.entry); - try { - completeTaskRunByRunId({ - runId: target.runId, - runtime: "subagent", - sessionKey: target.sessionKey, - endedAt, - lastEventAt: Date.now(), - progressSummary: resolveSubagentCompletionResultText(args.entry), - terminalSummary: terminalResult.terminalSummary, - terminalOutcome: terminalResult.terminalOutcome, - }); - } catch (err) { - params.warn("failed to mark subagent completion delivery blocked", { - error: buildSafeLifecycleErrorMeta(err), - runId: maskRunId(args.entry.runId), - childSessionKey: maskSessionKey(args.entry.childSessionKey), - }); - } - }; + }, +) => { + if ( + args.entry.expectsCompletionMessage !== true || + args.entry.execution.outcome?.status !== "ok" + ) { + return; + } + const endedAt = args.entry.execution.endedAt ?? Date.now(); + const terminalResult = resolveRequiredCompletionDeliveryFailureTerminalResult(args.reason); + const target = resolveSubagentTaskTarget(params, args.entry); + try { + completeTaskRunByRunId({ + runId: target.runId, + runtime: "subagent", + sessionKey: target.sessionKey, + endedAt, + lastEventAt: Date.now(), + progressSummary: resolveSubagentCompletionResultText(args.entry), + terminalSummary: terminalResult.terminalSummary, + terminalOutcome: terminalResult.terminalOutcome, + }); + } catch (err) { + params.warn("failed to mark subagent completion delivery blocked", { + error: buildSafeLifecycleErrorMeta(err), + runId: maskLifecycleIdentifier(args.entry.runId, "run"), + childSessionKey: maskLifecycleIdentifier(args.entry.childSessionKey, "session"), + }); + } +}; - const freezeRunResultAtCompletion = async ( - entry: SubagentRunRecord, - outcome: SubagentRunOutcome, - ): Promise => { - if (ensureCompletionState(entry).resultText !== undefined) { - return false; - } - if (outcome.status === "error") { - const completion = ensureCompletionState(entry); - completion.resultText = null; - completion.capturedAt = Date.now(); - return true; - } - let resultText: string | null; - try { - const transcriptTarget = entry.execution.transcriptTarget; - const agentId = - transcriptTarget?.agentId ?? resolveAgentIdFromSessionKey(entry.childSessionKey); - const sessionKey = transcriptTarget?.sessionKey ?? entry.childSessionKey; - const configuredStorePath = agentId - ? (transcriptTarget?.storePath ?? - resolveStorePath(params.getRuntimeConfig().session?.store, { agentId })) - : undefined; - const storePath = configuredStorePath - ? resolveSessionStorePathForScope({ - agentId, - sessionKey, - storePath: configuredStorePath, - }) - : undefined; - const sessionId = - transcriptTarget?.sessionId ?? - (agentId && storePath - ? loadSessionEntryReadOnly({ agentId, sessionKey, storePath })?.sessionId - : undefined); - const sessionTarget: SessionTranscriptRuntimeTarget | undefined = - agentId && sessionId && storePath - ? { agentId, sessionId, sessionKey, storePath } - : undefined; - const captured = await params.captureSubagentCompletionReply(entry.childSessionKey, { - waitForReply: entry.expectsCompletionMessage === true, - outcome, - ...(sessionTarget ? { sessionTarget } : {}), - }); - resultText = captured?.trim() ? capFrozenResultText(captured) : null; - } catch { - resultText = null; - } - const liveEntry = params.runs.get(entry.runId); - if ( - entry.pauseReason === "sessions_yield" || - liveEntry?.pauseReason === "sessions_yield" || - newerGenerationOwnsSession(entry) - ) { - return false; - } +export const freezeRunResultAtCompletion = async ( + context: SubagentLifecycleCommonContext, + entry: SubagentRunRecord, + outcome: SubagentRunOutcome, +): Promise => { + const params = context.options; + if (ensureCompletionState(entry).resultText !== undefined) { + return false; + } + if (outcome.status === "error") { const completion = ensureCompletionState(entry); - if (completion.resultText !== undefined) { - return false; - } - completion.resultText = resultText; + completion.resultText = null; completion.capturedAt = Date.now(); return true; - }; + } + let resultText: string | null; + try { + const transcriptTarget = entry.execution.transcriptTarget; + const agentId = + transcriptTarget?.agentId ?? resolveAgentIdFromSessionKey(entry.childSessionKey); + const sessionKey = transcriptTarget?.sessionKey ?? entry.childSessionKey; + const configuredStorePath = agentId + ? (transcriptTarget?.storePath ?? + resolveStorePath(params.getRuntimeConfig().session?.store, { agentId })) + : undefined; + const storePath = configuredStorePath + ? resolveSessionStorePathForScope({ + agentId, + sessionKey, + storePath: configuredStorePath, + }) + : undefined; + const sessionId = + transcriptTarget?.sessionId ?? + (agentId && storePath + ? loadSessionEntryReadOnly({ agentId, sessionKey, storePath })?.sessionId + : undefined); + const sessionTarget: SessionTranscriptRuntimeTarget | undefined = + agentId && sessionId && storePath ? { agentId, sessionId, sessionKey, storePath } : undefined; + const captured = await params.captureSubagentCompletionReply(entry.childSessionKey, { + waitForReply: entry.expectsCompletionMessage === true, + outcome, + ...(sessionTarget ? { sessionTarget } : {}), + }); + resultText = captured?.trim() ? capFrozenResultText(captured) : null; + } catch { + resultText = null; + } + const liveEntry = params.runs.get(entry.runId); + if ( + entry.pauseReason === "sessions_yield" || + liveEntry?.pauseReason === "sessions_yield" || + context.newerGenerationOwnsSession(entry) + ) { + return false; + } + const completion = ensureCompletionState(entry); + if (completion.resultText !== undefined) { + return false; + } + completion.resultText = resultText; + completion.capturedAt = Date.now(); + return true; +}; - const listPendingCompletionRunsForSession = (sessionKey: string): SubagentRunRecord[] => { - const key = sessionKey.trim(); - if (!key) { - return []; +const listPendingCompletionRunsForSession = ( + params: SubagentLifecycleOptions, + sessionKey: string, +): SubagentRunRecord[] => { + const key = sessionKey.trim(); + if (!key) { + return []; + } + const out: SubagentRunRecord[] = []; + for (const entry of params.runs.values()) { + if (entry.childSessionKey !== key) { + continue; } - const out: SubagentRunRecord[] = []; - for (const entry of params.runs.values()) { - if (entry.childSessionKey !== key) { - continue; - } - if (entry.expectsCompletionMessage !== true) { - continue; - } - if (typeof entry.execution.endedAt !== "number") { - continue; - } - if (typeof entry.cleanupCompletedAt === "number") { - continue; - } - // A paused row's result was deliberately cleared when it yielded; the text - // now in its session belongs to whatever turn runs next, not to the paused - // work. Refreezing it here would announce a stranger's output as this run's - // completion once the row finally settles. - if (entry.pauseReason === "sessions_yield") { - continue; - } - out.push(entry); + if (entry.expectsCompletionMessage !== true) { + continue; } - return out; - }; - - const refreshFrozenResultFromSession = async (sessionKey: string): Promise => { - const candidates = listPendingCompletionRunsForSession(sessionKey).filter( - (entry) => entry.execution.outcome?.status !== "error", - ); - const entry = candidates.toSorted(compareSubagentRunGeneration).at(-1); - if (!entry || newerGenerationOwnsSession(entry)) { - return false; + if (typeof entry.execution.endedAt !== "number") { + continue; } - const generation = entry.generation; - - let captured: string | undefined; - try { - captured = await params.captureSubagentCompletionReply(sessionKey); - } catch { - return false; + if (typeof entry.cleanupCompletedAt === "number") { + continue; } - const trimmed = captured?.trim(); - if (!trimmed || isSilentAgentReplyText(trimmed)) { - return false; - } - // Reply capture yields while registration can transfer session ownership. - // Only the exact row and generation that started capture may commit its text. - if ( - params.runs.get(entry.runId) !== entry || - entry.generation !== generation || - newerGenerationOwnsSession(entry) - ) { - return false; + // A paused row's result was deliberately cleared when it yielded; the text + // now in its session belongs to whatever turn runs next, not to the paused + // work. Refreezing it here would announce a stranger's output as this run's + // completion once the row finally settles. + if (entry.pauseReason === "sessions_yield") { + continue; } + out.push(entry); + } + return out; +}; - const nextFrozen = capFrozenResultText(trimmed); - const completion = ensureCompletionState(entry); - if (completion.resultText === nextFrozen) { - return false; - } - completion.resultText = nextFrozen; - completion.capturedAt = Date.now(); - params.persist(entry.runId); - return true; - }; +export const refreshFrozenResultFromSession = async ( + context: SubagentLifecycleCommonContext, + sessionKey: string, +): Promise => { + const params = context.options; + const candidates = listPendingCompletionRunsForSession(params, sessionKey).filter( + (entry) => entry.execution.outcome?.status !== "error", + ); + const entry = candidates.toSorted(compareSubagentRunGeneration).at(-1); + if (!entry || context.newerGenerationOwnsSession(entry)) { + return false; + } + const generation = entry.generation; - const emitCompletionEndedHookIfNeeded = async ( - entry: SubagentRunRecord, - reason: SubagentLifecycleEndedReason, - isCurrent?: () => boolean, - ) => { - if (params.shouldEmitEndedHookForRun({ entry, reason })) { - await params.emitSubagentEndedHookForRun({ - entry, - reason, - sendFarewell: true, - isCurrent, - }); - } - }; + let captured: string | undefined; + try { + captured = await params.captureSubagentCompletionReply(sessionKey); + } catch { + return false; + } + const trimmed = captured?.trim(); + if (!trimmed || isSilentAgentReplyText(trimmed)) { + return false; + } + // Reply capture yields while registration can transfer session ownership. + // Only the exact row and generation that started capture may commit its text. + if ( + params.runs.get(entry.runId) !== entry || + entry.generation !== generation || + context.newerGenerationOwnsSession(entry) + ) { + return false; + } - const clearPendingFinalDelivery = (entry: SubagentRunRecord) => { - const delivery = ensureDeliveryState(entry); - delivery.payload = undefined; - delivery.createdAt = undefined; - delivery.lastAttemptAt = undefined; - delivery.attemptCount = undefined; - delivery.lastError = undefined; - delivery.suspendedAt = undefined; - delivery.suspendedReason = undefined; - if (delivery.status !== "delivered" && delivery.status !== "failed") { - clearDeliveryState(entry); - } - }; + const nextFrozen = capFrozenResultText(trimmed); + const completion = ensureCompletionState(entry); + if (completion.resultText === nextFrozen) { + return false; + } + completion.resultText = nextFrozen; + completion.capturedAt = Date.now(); + params.persist(entry.runId); + return true; +}; - const loadPendingFinalDeliveryPayload = ( - entry: SubagentRunRecord, - ): PendingFinalDeliveryPayload => { - return { - requesterSessionKey: - entry.delivery?.payload?.requesterSessionKey ?? entry.requesterSessionKey, - requesterOrigin: entry.delivery?.payload?.requesterOrigin ?? entry.requesterOrigin, - requesterDisplayKey: - entry.delivery?.payload?.requesterDisplayKey ?? entry.requesterDisplayKey, - childSessionKey: entry.delivery?.payload?.childSessionKey ?? entry.childSessionKey, - childRunId: entry.delivery?.payload?.childRunId ?? entry.runId, - task: entry.delivery?.payload?.task ?? entry.task, - label: entry.delivery?.payload?.label ?? entry.label, - startedAt: entry.delivery?.payload?.startedAt ?? entry.execution.startedAt, - endedAt: entry.delivery?.payload?.endedAt ?? entry.execution.endedAt, - outcome: entry.delivery?.payload?.outcome ?? entry.execution.outcome, - expectsCompletionMessage: - entry.delivery?.payload?.expectsCompletionMessage ?? entry.expectsCompletionMessage, - spawnMode: entry.delivery?.payload?.spawnMode ?? entry.spawnMode, - wakeOnDescendantSettle: - entry.delivery?.payload?.wakeOnDescendantSettle ?? entry.wakeOnDescendantSettle, - terminalReply: entry.delivery?.payload?.terminalReply ?? entry.completion?.terminalReply, - }; - }; +export const emitCompletionEndedHookIfNeeded = async ( + params: SubagentLifecycleOptions, + entry: SubagentRunRecord, + reason: SubagentLifecycleEndedReason, + isCurrent?: () => boolean, +) => { + if (params.shouldEmitEndedHookForRun({ entry, reason })) { + await params.emitSubagentEndedHookForRun({ + entry, + reason, + sendFarewell: true, + isCurrent, + }); + } +}; - const markPendingFinalDelivery = (args: { entry: SubagentRunRecord; error?: string }) => { - const now = Date.now(); - const payload: PendingFinalDeliveryPayload = loadPendingFinalDeliveryPayload(args.entry); - - const delivery = ensureDeliveryState(args.entry); - delivery.status = "pending"; - delivery.createdAt ??= now; - delivery.lastAttemptAt = now; - delivery.attemptCount = (delivery.attemptCount ?? 0) + 1; - delivery.lastError = args.error ?? null; - delivery.payload = payload; - }; - - const refreshPendingFinalDeliveryPayload = (entry: SubagentRunRecord): boolean => { - const delivery = entry.delivery; - if ( - !delivery?.payload || - delivery.status === "delivered" || - typeof delivery.announcedAt === "number" - ) { - return false; - } - delivery.payload = { - ...delivery.payload, - startedAt: entry.execution.startedAt, - endedAt: entry.execution.endedAt, - outcome: entry.execution.outcome, - terminalReply: entry.completion?.terminalReply, - }; - return true; - }; +export const clearSubagentPendingDelivery = (entry: SubagentRunRecord) => { + const delivery = ensureDeliveryState(entry); + delivery.payload = undefined; + delivery.createdAt = undefined; + delivery.lastAttemptAt = undefined; + delivery.attemptCount = undefined; + delivery.lastError = undefined; + delivery.suspendedAt = undefined; + delivery.suspendedReason = undefined; + if (delivery.status !== "delivered" && delivery.status !== "failed") { + clearDeliveryState(entry); + } +}; +export const loadPendingFinalDeliveryPayload = ( + entry: SubagentRunRecord, +): PendingFinalDeliveryPayload => { return { - clearPendingFinalDelivery, - emitCompletionEndedHookIfNeeded, - formatAnnounceDeliveryError, - freezeRunResultAtCompletion, - hasPriorRequesterDeliveryMirror, - loadPendingFinalDeliveryPayload, - markPendingFinalDelivery, - recordAnnounceDeliveryResult, - refreshFrozenResultFromSession, - refreshPendingFinalDeliveryPayload, - safeFinalizeSubagentTaskRun, - safeMarkRequiredCompletionDeliveryBlocked, - safeSetSubagentTaskDeliveryStatus, + requesterSessionKey: entry.delivery?.payload?.requesterSessionKey ?? entry.requesterSessionKey, + requesterOrigin: entry.delivery?.payload?.requesterOrigin ?? entry.requesterOrigin, + requesterDisplayKey: entry.delivery?.payload?.requesterDisplayKey ?? entry.requesterDisplayKey, + childSessionKey: entry.delivery?.payload?.childSessionKey ?? entry.childSessionKey, + childRunId: entry.delivery?.payload?.childRunId ?? entry.runId, + task: entry.delivery?.payload?.task ?? entry.task, + label: entry.delivery?.payload?.label ?? entry.label, + startedAt: entry.delivery?.payload?.startedAt ?? entry.execution.startedAt, + endedAt: entry.delivery?.payload?.endedAt ?? entry.execution.endedAt, + outcome: entry.delivery?.payload?.outcome ?? entry.execution.outcome, + expectsCompletionMessage: + entry.delivery?.payload?.expectsCompletionMessage ?? entry.expectsCompletionMessage, + spawnMode: entry.delivery?.payload?.spawnMode ?? entry.spawnMode, + wakeOnDescendantSettle: + entry.delivery?.payload?.wakeOnDescendantSettle ?? entry.wakeOnDescendantSettle, + terminalReply: entry.delivery?.payload?.terminalReply ?? entry.completion?.terminalReply, }; -} +}; + +export const markPendingFinalDelivery = (args: { entry: SubagentRunRecord; error?: string }) => { + const now = Date.now(); + const payload: PendingFinalDeliveryPayload = loadPendingFinalDeliveryPayload(args.entry); + + const delivery = ensureDeliveryState(args.entry); + delivery.status = "pending"; + delivery.createdAt ??= now; + delivery.lastAttemptAt = now; + delivery.attemptCount = (delivery.attemptCount ?? 0) + 1; + delivery.lastError = args.error ?? null; + delivery.payload = payload; +}; + +export const refreshPendingFinalDeliveryPayload = (entry: SubagentRunRecord): boolean => { + const delivery = entry.delivery; + if ( + !delivery?.payload || + delivery.status === "delivered" || + typeof delivery.announcedAt === "number" + ) { + return false; + } + delivery.payload = { + ...delivery.payload, + startedAt: entry.execution.startedAt, + endedAt: entry.execution.endedAt, + outcome: entry.execution.outcome, + terminalReply: entry.completion?.terminalReply, + }; + return true; +}; diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-requester-wake.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-requester-wake.ts deleted file mode 100644 index 93b16700e8c5..000000000000 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-requester-wake.ts +++ /dev/null @@ -1,354 +0,0 @@ -import { runWithoutOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js"; -import { runWithGatewayIndependentRootWorkContinuation } from "../../../process/gateway-work-admission.js"; -import type { SubagentAnnounceDeliveryResult } from "../announce/subagent-announce-dispatch.js"; -import { ensureDeliveryState } from "./subagent-delivery-state.js"; -import type { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; -import type { - SubagentRegistryLifecycleParams, - SubagentRegistryLifecycleState, -} from "./subagent-registry-lifecycle-contracts.js"; -import type { createSubagentRegistryLifecycleDelivery } from "./subagent-registry-lifecycle-delivery.js"; -import type { RequesterSettleWakeState, SubagentRunRecord } from "./subagent-registry.types.js"; -import { hasSubagentRunEnded } from "./subagent-run-liveness.js"; - -type RequesterSettleWakeBatchState = - import("../announce/subagent-announce.requester-settle-wake.js").RequesterSettleWakeBatchState; - -export function createSubagentRegistryLifecycleRequesterWake( - params: SubagentRegistryLifecycleParams, - lifecycleState: SubagentRegistryLifecycleState, - common: ReturnType, - deliveryHelpers: ReturnType, -) { - const { - pendingRequesterSettleWakeRearms, - scheduledRequesterSettleWakeRuns, - scheduledRequesterSettleWakeTimers, - } = lifecycleState; - const { buildSafeLifecycleErrorMeta, maskRunId, maskSessionKey } = common; - const { safeMarkRequiredCompletionDeliveryBlocked, safeSetSubagentTaskDeliveryStatus } = - deliveryHelpers; - - const transitionRequesterSettleWakeBatch = ( - runIds: readonly string[], - state: RequesterSettleWakeBatchState, - ) => { - const entries = runIds - .map((runId) => params.runs.get(runId)) - .filter( - (entry): entry is SubagentRunRecord => - Boolean(entry?.requesterSettleWake) && - entry?.requesterSettleWake?.rearmGeneration === state.rearmGeneration, - ); - if (entries.length === 0) { - return; - } - const previousStates = entries.map((entry) => structuredClone(entry.requesterSettleWake)); - for (const entry of entries) { - entry.requesterSettleWake = { - ...state, - ...(entry.requesterSettleWake?.retireAfterSettle === true - ? { retireAfterSettle: true } - : {}), - }; - } - try { - params.persistOrThrow(...entries.map((entry) => entry.runId)); - } catch (error) { - entries.forEach((entry, index) => { - entry.requesterSettleWake = previousStates[index]; - }); - throw error; - } - }; - - const completeRequesterSettleWakeBatch = ( - runIds: readonly string[], - rearmGeneration?: number, - outcome?: SubagentAnnounceDeliveryResult, - ) => { - const entries = runIds - .map((runId) => [runId, params.runs.get(runId)] as const) - .filter( - (pair): pair is readonly [string, SubagentRunRecord] => - Boolean(pair[1]?.requesterSettleWake) && - pair[1]?.requesterSettleWake?.rearmGeneration === rearmGeneration, - ); - if (entries.length === 0) { - return; - } - const requesterSessionKeys = new Set(entries.map(([, entry]) => entry.requesterSessionKey)); - const previousStates = entries.map(([, entry]) => ({ - delivery: structuredClone(entry.delivery), - requesterSettleWake: structuredClone(entry.requesterSettleWake), - retireAfterRequesterTurn: entry.retireAfterRequesterTurn, - })); - const settledDeliveries: SubagentRunRecord[] = []; - for (const [runId, entry] of entries) { - if ( - outcome && - entry.expectsCompletionMessage === true && - entry.delivery?.status !== "delivered" - ) { - const delivery = ensureDeliveryState(entry); - if (outcome.delivered) { - const deliveredAt = outcome.deliveredAt ?? Date.now(); - delivery.status = "delivered"; - delivery.disposition = "delivered"; - delivery.deliveredAt = deliveredAt; - delivery.announcedAt = deliveredAt; - delivery.lastError = undefined; - delivery.lastDropReason = undefined; - } else { - delivery.status = "failed"; - delivery.disposition = outcome.disposition ?? delivery.disposition; - delivery.lastError = outcome.error ?? outcome.reason ?? "requester settle wake failed"; - delivery.deliveredAt = undefined; - delivery.announcedAt = undefined; - } - settledDeliveries.push(entry); - } - if (entry.requesterTurnRunId) { - entry.retireAfterRequesterTurn = - entry.retireAfterRequesterTurn === true || - entry.requesterSettleWake?.retireAfterSettle === true - ? true - : undefined; - entry.requesterSettleWake = undefined; - } else if (entry.requesterSettleWake?.retireAfterSettle === true) { - params.runs.delete(runId); - } else { - entry.requesterSettleWake = undefined; - } - } - try { - params.persistOrThrow(...entries.map(([runId]) => runId)); - } catch (error) { - entries.forEach(([runId, entry], index) => { - const previous = previousStates[index]; - params.runs.set(runId, entry); - entry.delivery = previous?.delivery; - entry.requesterSettleWake = previous?.requesterSettleWake; - entry.retireAfterRequesterTurn = previous?.retireAfterRequesterTurn; - }); - throw error; - } - for (const entry of settledDeliveries) { - if (outcome?.delivered) { - safeSetSubagentTaskDeliveryStatus({ entry, deliveryStatus: "delivered" }); - } else if (outcome) { - const error = outcome.error ?? outcome.reason ?? "requester settle wake failed"; - safeSetSubagentTaskDeliveryStatus({ - entry, - deliveryStatus: "failed", - deliveryError: error, - }); - safeMarkRequiredCompletionDeliveryBlocked({ entry, reason: error }); - } - } - for (const [runId, entry] of entries) { - const retryTimer = scheduledRequesterSettleWakeTimers.get(runId); - if (retryTimer) { - clearTimeout(retryTimer.timer); - scheduledRequesterSettleWakeTimers.delete(runId); - } - if (entry.requesterSettleWake === undefined || !params.runs.has(runId)) { - params.resumedRuns.delete(runId); - params.clearPendingLifecycleError(runId); - } - } - for (const [runId, entry] of params.runs) { - if (entry.requesterSettleWake && requesterSessionKeys.has(entry.requesterSessionKey)) { - scheduleRequesterSettleWake(runId, entry); - } - } - }; - - const markRequesterSettleWakePending = ( - entry: SubagentRunRecord, - options?: { retireAfterSettle?: boolean }, - ) => { - const existing = entry.requesterSettleWake; - entry.requesterSettleWake = { - status: existing?.status ?? "pending", - attemptCount: existing?.attemptCount ?? 0, - ...(existing?.replayCount !== undefined ? { replayCount: existing.replayCount } : {}), - ...(existing?.nextAttemptAt !== undefined ? { nextAttemptAt: existing.nextAttemptAt } : {}), - ...(existing?.batchRunIds ? { batchRunIds: [...existing.batchRunIds] } : {}), - ...(existing?.requesterYieldBatch === true ? { requesterYieldBatch: true } : {}), - ...(existing?.afterRequesterYield === true ? { afterRequesterYield: true } : {}), - ...(existing?.rearmGeneration !== undefined - ? { rearmGeneration: existing.rearmGeneration } - : {}), - ...(existing?.lastError !== undefined ? { lastError: existing.lastError } : {}), - ...(existing?.retireAfterSettle === true || options?.retireAfterSettle === true - ? { retireAfterSettle: true } - : {}), - } satisfies RequesterSettleWakeState; - }; - - const persistRequesterSettleWakePending = ( - entry: SubagentRunRecord, - options?: { - cleanupCompletedAt?: number; - retireAfterSettle?: boolean; - retireInterruptedRecovery?: boolean; - }, - ) => { - const previousCleanupCompletedAt = entry.cleanupCompletedAt; - const previousExecution = entry.execution; - const previousTerminalOwner = entry.terminalOwner; - const previousWake = structuredClone(entry.requesterSettleWake); - if (options?.cleanupCompletedAt !== undefined) { - entry.cleanupCompletedAt = options.cleanupCompletedAt; - } - if (options?.retireInterruptedRecovery) { - entry.execution = { - ...entry.execution, - restartRecovery: undefined, - suppressSessionEffects: true, - }; - entry.terminalOwner = undefined; - } - markRequesterSettleWakePending(entry, options); - try { - params.persistOrThrow(entry.runId); - } catch (error) { - entry.cleanupCompletedAt = previousCleanupCompletedAt; - entry.execution = previousExecution; - entry.terminalOwner = previousTerminalOwner; - entry.requesterSettleWake = previousWake; - throw error; - } - }; - - // Once a child reaches a terminal settle, let the announce layer decide - // whether its requester's batch has fully drained and, if so, wake the - // registry-less top-level requester to synthesize. Settle bookkeeping never - // blocks on the wake, but the wake must run as tracked root work: a live - // cleanup parent reserves the root synchronously, so restart or suspend - // cannot reach quiescence between scheduling and the wake's gateway turn. - // Failures are logged only. - function retainScheduledRequesterSettleWakeTimer( - runId: string, - deadline: number, - rearmGeneration?: number, - ): boolean { - const scheduled = scheduledRequesterSettleWakeTimers.get(runId); - if (!scheduled) { - return false; - } - const hasNewerGeneration = - rearmGeneration !== undefined && - (scheduled.rearmGeneration === undefined || rearmGeneration > scheduled.rearmGeneration); - if (!hasNewerGeneration && deadline >= scheduled.deadline) { - return true; - } - clearTimeout(scheduled.timer); - scheduledRequesterSettleWakeTimers.delete(runId); - return false; - } - - function scheduleRequesterSettleWakeRetry(runId: string, entry: SubagentRunRecord): void { - const nextAttemptAt = entry.requesterSettleWake?.nextAttemptAt; - if (nextAttemptAt === undefined || nextAttemptAt <= Date.now()) { - return; - } - const rearmGeneration = entry.requesterSettleWake?.rearmGeneration; - if (retainScheduledRequesterSettleWakeTimer(runId, nextAttemptAt, rearmGeneration)) { - return; - } - const timer = setTimeout( - () => { - if (scheduledRequesterSettleWakeTimers.get(runId)?.timer !== timer) { - return; - } - scheduledRequesterSettleWakeTimers.delete(runId); - const current = params.runs.get(runId); - if (current === entry && current.requesterSettleWake) { - scheduleRequesterSettleWake(runId, current); - } - }, - Math.max(0, nextAttemptAt - Date.now()), - ); - timer.unref?.(); - scheduledRequesterSettleWakeTimers.set(runId, { - timer, - deadline: nextAttemptAt, - rearmGeneration, - }); - } - - function scheduleRequesterSettleWake(runId: string, entry: SubagentRunRecord): void { - const requesterSessionKey = entry.requesterSessionKey?.trim(); - // A replayed lifecycle start can retain an older endedAt; require both - // terminal status and end evidence so a live child never wakes its requester. - if ( - entry.collect || - entry.execution.status === "running" || - !hasSubagentRunEnded(entry) || - !requesterSessionKey || - (entry.requesterTurnRunId && entry.requesterTurnYielded === true) || - scheduledRequesterSettleWakeRuns.has(runId) - ) { - return; - } - const now = Date.now(); - const nextAttemptAt = entry.requesterSettleWake?.nextAttemptAt; - const deadline = nextAttemptAt !== undefined && nextAttemptAt > now ? nextAttemptAt : now; - if ( - retainScheduledRequesterSettleWakeTimer( - runId, - deadline, - entry.requesterSettleWake?.rearmGeneration, - ) - ) { - return; - } - if (nextAttemptAt !== undefined && nextAttemptAt > now) { - scheduleRequesterSettleWakeRetry(runId, entry); - return; - } - scheduledRequesterSettleWakeRuns.add(runId); - // Wake turns outlive their spawning attempt; clear its owner before both - // dispatch and chained re-arms so transcript writes acquire a fresh lock. - runWithoutOwnedSessionTranscriptWrites(() => { - void runWithGatewayIndependentRootWorkContinuation(() => - params.maybeWakeRequesterAfterAllChildrenSettled({ - requesterSessionKey, - requesterOrigin: entry.requesterOrigin, - settledEntry: entry, - transitionBatch: transitionRequesterSettleWakeBatch, - completeBatch: completeRequesterSettleWakeBatch, - }), - ) - .catch((error: unknown) => { - params.warn("requester settle wake failed", { - error: buildSafeLifecycleErrorMeta(error), - runId: maskRunId(runId), - requesterSessionKey: maskSessionKey(requesterSessionKey), - }); - }) - .finally(() => { - scheduledRequesterSettleWakeRuns.delete(runId); - const wasRearmedWhileRunning = pendingRequesterSettleWakeRearms.delete(runId); - const current = params.runs.get(runId); - if (current === entry && current.requesterSettleWake) { - if (wasRearmedWhileRunning) { - // A requester yield can freeze a delivered batch while this run is - // resolving its earlier no-wake decision. Admit that durable update now. - scheduleRequesterSettleWake(runId, current); - } else { - scheduleRequesterSettleWakeRetry(runId, current); - } - } - }); - }); - } - - return { - markRequesterSettleWakePending, - persistRequesterSettleWakePending, - scheduleRequesterSettleWake, - }; -} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-terminal-cleanup.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-terminal-cleanup.ts deleted file mode 100644 index 05594af5fd3f..000000000000 --- a/src/agents/subagents/registry/subagent-registry-lifecycle-terminal-cleanup.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; -import type { createSubagentRegistryLifecycleCleanupBase } from "./subagent-registry-lifecycle-cleanup-base.js"; -import type { createSubagentRegistryLifecycleCleanup } from "./subagent-registry-lifecycle-cleanup.js"; -import type { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; -import { loadCleanupBrowserSessionsForLifecycleEnd } from "./subagent-registry-lifecycle-completion-support.js"; -import type { SubagentRegistryLifecycleParams } from "./subagent-registry-lifecycle-contracts.js"; -import type { SubagentCompletionRequest, SubagentRunRecord } from "./subagent-registry.types.js"; - -export function createSubagentRegistryLifecycleTerminalCleanup( - params: SubagentRegistryLifecycleParams, - common: ReturnType, - cleanupBase: ReturnType, - cleanup: ReturnType, -) { - const { buildSafeLifecycleErrorMeta, maskRunId, maskSessionKey, newerGenerationOwnsSession } = - common; - const { isTerminalCallbackCurrent } = cleanupBase; - const { retireRunModeBundleMcpRuntime, startSubagentAnnounceCleanupFlow } = cleanup; - - const complete = async (args: { - completeParams: SubagentCompletionRequest; - entry: SubagentRunRecord; - isProvisionalKill: boolean; - retireSupersededSession: (entry: SubagentRunRecord) => Promise; - suppressedForSteerRestart: boolean; - suppressSessionEffects: boolean; - terminalGeneration: number; - }) => { - const { - completeParams, - entry, - isProvisionalKill, - retireSupersededSession, - suppressedForSteerRestart, - terminalGeneration, - } = args; - let { suppressSessionEffects } = args; - // Session cleanup belongs to the exact registry row and child generation. - // A replacement may reuse either the run id or the child session key. - const isSessionEffectsOwnerCurrent = () => - isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration) && - !newerGenerationOwnsSession(entry); - const refreshSessionEffectsSuppression = () => { - if ( - suppressSessionEffects || - !isSessionEffectsOwnerCurrent() || - !shouldSuppressSubagentRecoverySessionEffects(entry) - ) { - return suppressSessionEffects; - } - const previousExecution = entry.execution; - entry.execution = { - ...previousExecution, - suppressSessionEffects: true, - }; - try { - params.persistOrThrow(completeParams.runId); - } catch (error) { - entry.execution = previousExecution; - throw error; - } - suppressSessionEffects = true; - return true; - }; - if (!completeParams.triggerCleanup || suppressedForSteerRestart) { - return; - } - refreshSessionEffectsSuppression(); - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; - } - if (newerGenerationOwnsSession(entry)) { - await retireSupersededSession(entry); - return; - } - - // registerSubagentRun fires both an in-process listener and a gateway - // waitForSubagentCompletion RPC; both can reach this point for the same - // runId in embedded mode. Dedupe only the browser driver tab-close IPC - // with a sync check-then-set. The retire + announce tail below must still - // run for every caller, so a slow or held first browser cleanup cannot - // strand a duplicate caller's completion behind it. - if (!suppressSessionEffects && entry.browserCleanupDispatchedAt === undefined) { - let dispatchedBrowserCleanup = false; - let cleanupBrowserSessions = params.cleanupBrowserSessionsForLifecycleEnd; - try { - cleanupBrowserSessions ??= await loadCleanupBrowserSessionsForLifecycleEnd(); - } catch (error) { - params.warn("failed to load browser cleanup for completed subagent", { - error: buildSafeLifecycleErrorMeta(error), - runId: maskRunId(completeParams.runId), - childSessionKey: maskSessionKey(entry.childSessionKey), - }); - } - if (cleanupBrowserSessions) { - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; - } - if (newerGenerationOwnsSession(entry)) { - await retireSupersededSession(entry); - return; - } - if (refreshSessionEffectsSuppression()) { - return; - } - // Claim only when this caller is about to dispatch. A concurrent caller - // may have claimed while the lazy browser module was loading. - if (entry.browserCleanupDispatchedAt === undefined) { - entry.browserCleanupDispatchedAt = Date.now(); - dispatchedBrowserCleanup = true; - try { - await cleanupBrowserSessions({ - sessionKeys: [entry.childSessionKey], - onWarn: (msg) => params.warn(msg, { runId: entry.runId }), - }); - } catch (error) { - params.warn("failed to cleanup browser sessions for completed subagent", { - error: buildSafeLifecycleErrorMeta(error), - runId: maskRunId(completeParams.runId), - childSessionKey: maskSessionKey(entry.childSessionKey), - }); - } - } - } - if (dispatchedBrowserCleanup) { - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; - } - refreshSessionEffectsSuppression(); - if (newerGenerationOwnsSession(entry)) { - await retireSupersededSession(entry); - return; - } - } - } - - if (!suppressSessionEffects) { - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; - } - if (newerGenerationOwnsSession(entry)) { - await retireSupersededSession(entry); - return; - } - try { - await retireRunModeBundleMcpRuntime({ - runId: completeParams.runId, - entry, - reason: "subagent-run-complete", - }); - } catch (error) { - params.warn("failed to retire subagent bundle MCP runtime after completion", { - error: buildSafeLifecycleErrorMeta(error), - runId: maskRunId(completeParams.runId), - childSessionKey: maskSessionKey(entry.childSessionKey), - }); - } - if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { - return; - } - refreshSessionEffectsSuppression(); - if (newerGenerationOwnsSession(entry)) { - await retireSupersededSession(entry); - return; - } - } - - if (isProvisionalKill) { - // Browser and MCP resources can close immediately, but completion delivery - // waits for the provider result or the killed tombstone reconciliation. - return; - } - - refreshSessionEffectsSuppression(); - startSubagentAnnounceCleanupFlow(completeParams.runId, entry); - }; - - return { complete }; -} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle-wake.ts b/src/agents/subagents/registry/subagent-registry-lifecycle-wake.ts new file mode 100644 index 000000000000..cdd155af23b1 --- /dev/null +++ b/src/agents/subagents/registry/subagent-registry-lifecycle-wake.ts @@ -0,0 +1,556 @@ +import { runWithoutOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js"; +import { runWithGatewayIndependentRootWorkContinuation } from "../../../process/gateway-work-admission.js"; +import { runWithGatewayIndependentRootWorkAdmission } from "../../../process/gateway-work-admission.js"; +import { defaultRuntime } from "../../../runtime.js"; +import { retireSessionMcpRuntimeForSessionKey } from "../../agent-bundle-mcp-tools.js"; +import { removeInternalSessionEffectsSession } from "../../internal-session-effects.js"; +import type { SubagentAnnounceDeliveryResult } from "../announce/subagent-announce-dispatch.js"; +import { ensureDeliveryState } from "./subagent-delivery-state.js"; +import { SUBAGENT_ENDED_REASON_KILLED } from "./subagent-lifecycle-events.js"; +import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; +import type { + CleanupBookkeepingParams, + SubagentLifecycleWakeContext, +} from "./subagent-registry-lifecycle-context.js"; +import { + buildSafeLifecycleErrorMeta, + maskLifecycleIdentifier, + safeMarkRequiredCompletionDeliveryBlocked, + safeSetSubagentTaskDeliveryStatus, +} from "./subagent-registry-lifecycle-delivery.js"; +import type { RequesterSettleWakeState, SubagentRunRecord } from "./subagent-registry.types.js"; +import { hasSubagentRunEnded } from "./subagent-run-liveness.js"; + +type RequesterSettleWakeBatchState = + import("../announce/subagent-announce.requester-settle-wake.js").RequesterSettleWakeBatchState; + +const transitionRequesterSettleWakeBatch = ( + context: SubagentLifecycleWakeContext, + runIds: readonly string[], + state: RequesterSettleWakeBatchState, +) => { + const params = context.options; + const entries = runIds + .map((runId) => params.runs.get(runId)) + .filter( + (entry): entry is SubagentRunRecord => + Boolean(entry?.requesterSettleWake) && + entry?.requesterSettleWake?.rearmGeneration === state.rearmGeneration, + ); + if (entries.length === 0) { + return; + } + const previousStates = entries.map((entry) => structuredClone(entry.requesterSettleWake)); + for (const entry of entries) { + entry.requesterSettleWake = { + ...state, + ...(entry.requesterSettleWake?.retireAfterSettle === true ? { retireAfterSettle: true } : {}), + }; + } + try { + params.persistOrThrow(...entries.map((entry) => entry.runId)); + } catch (error) { + entries.forEach((entry, index) => { + entry.requesterSettleWake = previousStates[index]; + }); + throw error; + } +}; + +const completeRequesterSettleWakeBatch = ( + context: SubagentLifecycleWakeContext, + runIds: readonly string[], + rearmGeneration?: number, + outcome?: SubagentAnnounceDeliveryResult, +) => { + const params = context.options; + const entries = runIds + .map((runId) => [runId, params.runs.get(runId)] as const) + .filter( + (pair): pair is readonly [string, SubagentRunRecord] => + Boolean(pair[1]?.requesterSettleWake) && + pair[1]?.requesterSettleWake?.rearmGeneration === rearmGeneration, + ); + if (entries.length === 0) { + return; + } + const requesterSessionKeys = new Set(entries.map(([, entry]) => entry.requesterSessionKey)); + const previousStates = entries.map(([, entry]) => ({ + delivery: structuredClone(entry.delivery), + requesterSettleWake: structuredClone(entry.requesterSettleWake), + retireAfterRequesterTurn: entry.retireAfterRequesterTurn, + })); + const settledDeliveries: SubagentRunRecord[] = []; + for (const [runId, entry] of entries) { + if ( + outcome && + entry.expectsCompletionMessage === true && + entry.delivery?.status !== "delivered" + ) { + const delivery = ensureDeliveryState(entry); + if (outcome.delivered) { + const deliveredAt = outcome.deliveredAt ?? Date.now(); + delivery.status = "delivered"; + delivery.disposition = "delivered"; + delivery.deliveredAt = deliveredAt; + delivery.announcedAt = deliveredAt; + delivery.lastError = undefined; + delivery.lastDropReason = undefined; + } else { + delivery.status = "failed"; + delivery.disposition = outcome.disposition ?? delivery.disposition; + delivery.lastError = outcome.error ?? outcome.reason ?? "requester settle wake failed"; + delivery.deliveredAt = undefined; + delivery.announcedAt = undefined; + } + settledDeliveries.push(entry); + } + if (entry.requesterTurnRunId) { + entry.retireAfterRequesterTurn = + entry.retireAfterRequesterTurn === true || + entry.requesterSettleWake?.retireAfterSettle === true + ? true + : undefined; + entry.requesterSettleWake = undefined; + } else if (entry.requesterSettleWake?.retireAfterSettle === true) { + params.runs.delete(runId); + } else { + entry.requesterSettleWake = undefined; + } + } + try { + params.persistOrThrow(...entries.map(([runId]) => runId)); + } catch (error) { + entries.forEach(([runId, entry], index) => { + const previous = previousStates[index]; + params.runs.set(runId, entry); + entry.delivery = previous?.delivery; + entry.requesterSettleWake = previous?.requesterSettleWake; + entry.retireAfterRequesterTurn = previous?.retireAfterRequesterTurn; + }); + throw error; + } + for (const entry of settledDeliveries) { + if (outcome?.delivered) { + safeSetSubagentTaskDeliveryStatus(params, { + entry, + deliveryStatus: "delivered", + }); + } else if (outcome) { + const error = outcome.error ?? outcome.reason ?? "requester settle wake failed"; + safeSetSubagentTaskDeliveryStatus(params, { + entry, + deliveryStatus: "failed", + deliveryError: error, + }); + safeMarkRequiredCompletionDeliveryBlocked(params, { entry, reason: error }); + } + } + for (const [runId, entry] of entries) { + const retryTimer = context.getRequesterSettleWakeTimer(runId); + if (retryTimer) { + clearTimeout(retryTimer.timer); + context.deleteRequesterSettleWakeTimer(runId); + } + if (entry.requesterSettleWake === undefined || !params.runs.has(runId)) { + params.resumedRuns.delete(runId); + params.clearPendingLifecycleError(runId); + } + } + for (const [runId, entry] of params.runs) { + if (entry.requesterSettleWake && requesterSessionKeys.has(entry.requesterSessionKey)) { + scheduleRequesterSettleWake(context, runId, entry); + } + } +}; + +export const markRequesterSettleWakePending = ( + entry: SubagentRunRecord, + options?: { retireAfterSettle?: boolean }, +) => { + const existing = entry.requesterSettleWake; + entry.requesterSettleWake = { + status: existing?.status ?? "pending", + attemptCount: existing?.attemptCount ?? 0, + ...(existing?.replayCount !== undefined ? { replayCount: existing.replayCount } : {}), + ...(existing?.nextAttemptAt !== undefined ? { nextAttemptAt: existing.nextAttemptAt } : {}), + ...(existing?.batchRunIds ? { batchRunIds: [...existing.batchRunIds] } : {}), + ...(existing?.requesterYieldBatch === true ? { requesterYieldBatch: true } : {}), + ...(existing?.afterRequesterYield === true ? { afterRequesterYield: true } : {}), + ...(existing?.rearmGeneration !== undefined + ? { rearmGeneration: existing.rearmGeneration } + : {}), + ...(existing?.lastError !== undefined ? { lastError: existing.lastError } : {}), + ...(existing?.retireAfterSettle === true || options?.retireAfterSettle === true + ? { retireAfterSettle: true } + : {}), + } satisfies RequesterSettleWakeState; +}; + +const persistRequesterSettleWakePending = ( + context: SubagentLifecycleWakeContext, + entry: SubagentRunRecord, + options?: { + cleanupCompletedAt?: number; + retireAfterSettle?: boolean; + retireInterruptedRecovery?: boolean; + }, +) => { + const params = context.options; + const previousCleanupCompletedAt = entry.cleanupCompletedAt; + const previousExecution = entry.execution; + const previousTerminalOwner = entry.terminalOwner; + const previousWake = structuredClone(entry.requesterSettleWake); + if (options?.cleanupCompletedAt !== undefined) { + entry.cleanupCompletedAt = options.cleanupCompletedAt; + } + if (options?.retireInterruptedRecovery) { + entry.execution = { + ...entry.execution, + restartRecovery: undefined, + suppressSessionEffects: true, + }; + entry.terminalOwner = undefined; + } + markRequesterSettleWakePending(entry, options); + try { + params.persistOrThrow(entry.runId); + } catch (error) { + entry.cleanupCompletedAt = previousCleanupCompletedAt; + entry.execution = previousExecution; + entry.terminalOwner = previousTerminalOwner; + entry.requesterSettleWake = previousWake; + throw error; + } +}; + +// Once a child reaches a terminal settle, let the announce layer decide +// whether its requester's batch has fully drained and, if so, wake the +// registry-less top-level requester to synthesize. Settle bookkeeping never +// blocks on the wake, but the wake must run as tracked root work: a live +// cleanup parent reserves the root synchronously, so restart or suspend +// cannot reach quiescence between scheduling and the wake's gateway turn. +// Failures are logged only. +function retainScheduledRequesterSettleWakeTimer( + context: SubagentLifecycleWakeContext, + runId: string, + deadline: number, + rearmGeneration?: number, +): boolean { + const scheduled = context.getRequesterSettleWakeTimer(runId); + if (!scheduled) { + return false; + } + const hasNewerGeneration = + rearmGeneration !== undefined && + (scheduled.rearmGeneration === undefined || rearmGeneration > scheduled.rearmGeneration); + if (!hasNewerGeneration && deadline >= scheduled.deadline) { + return true; + } + clearTimeout(scheduled.timer); + context.deleteRequesterSettleWakeTimer(runId); + return false; +} + +function scheduleRequesterSettleWakeRetry( + context: SubagentLifecycleWakeContext, + runId: string, + entry: SubagentRunRecord, +): void { + const params = context.options; + const nextAttemptAt = entry.requesterSettleWake?.nextAttemptAt; + if (nextAttemptAt === undefined || nextAttemptAt <= Date.now()) { + return; + } + const rearmGeneration = entry.requesterSettleWake?.rearmGeneration; + if (retainScheduledRequesterSettleWakeTimer(context, runId, nextAttemptAt, rearmGeneration)) { + return; + } + const timer = setTimeout( + () => { + if (context.getRequesterSettleWakeTimer(runId)?.timer !== timer) { + return; + } + context.deleteRequesterSettleWakeTimer(runId); + const current = params.runs.get(runId); + if (current === entry && current.requesterSettleWake) { + scheduleRequesterSettleWake(context, runId, current); + } + }, + Math.max(0, nextAttemptAt - Date.now()), + ); + timer.unref?.(); + context.setRequesterSettleWakeTimer(runId, { + timer, + deadline: nextAttemptAt, + rearmGeneration, + }); +} + +export function scheduleRequesterSettleWake( + context: SubagentLifecycleWakeContext, + runId: string, + entry: SubagentRunRecord, +): void { + const params = context.options; + const requesterSessionKey = entry.requesterSessionKey?.trim(); + // A replayed lifecycle start can retain an older endedAt; require both + // terminal status and end evidence so a live child never wakes its requester. + if ( + entry.collect || + entry.execution.status === "running" || + !hasSubagentRunEnded(entry) || + !requesterSessionKey || + (entry.requesterTurnRunId && entry.requesterTurnYielded === true) || + context.hasScheduledRequesterSettleWakeRun(runId) + ) { + return; + } + const now = Date.now(); + const nextAttemptAt = entry.requesterSettleWake?.nextAttemptAt; + const deadline = nextAttemptAt !== undefined && nextAttemptAt > now ? nextAttemptAt : now; + if ( + retainScheduledRequesterSettleWakeTimer( + context, + runId, + deadline, + entry.requesterSettleWake?.rearmGeneration, + ) + ) { + return; + } + if (nextAttemptAt !== undefined && nextAttemptAt > now) { + scheduleRequesterSettleWakeRetry(context, runId, entry); + return; + } + context.markRequesterSettleWakeRunScheduled(runId); + // Wake turns outlive their spawning attempt; clear its owner before both + // dispatch and chained re-arms so transcript writes acquire a fresh lock. + runWithoutOwnedSessionTranscriptWrites(() => { + void runWithGatewayIndependentRootWorkContinuation(() => + params.maybeWakeRequesterAfterAllChildrenSettled({ + requesterSessionKey, + requesterOrigin: entry.requesterOrigin, + settledEntry: entry, + transitionBatch: (runIds, state) => + transitionRequesterSettleWakeBatch(context, runIds, state), + completeBatch: (runIds, rearmGeneration, outcome) => + completeRequesterSettleWakeBatch(context, runIds, rearmGeneration, outcome), + }), + ) + .catch((error: unknown) => { + params.warn("requester settle wake failed", { + error: buildSafeLifecycleErrorMeta(error), + runId: maskLifecycleIdentifier(runId, "run"), + requesterSessionKey: maskLifecycleIdentifier(requesterSessionKey, "session"), + }); + }) + .finally(() => { + context.unmarkRequesterSettleWakeRunScheduled(runId); + const wasRearmedWhileRunning = context.takeRequesterSettleWakeRearm(runId); + const current = params.runs.get(runId); + if (current === entry && current.requesterSettleWake) { + if (wasRearmedWhileRunning) { + // A requester yield can freeze a delivered batch while this run is + // resolving its earlier no-wake decision. Admit that durable update now. + scheduleRequesterSettleWake(context, runId, current); + } else { + scheduleRequesterSettleWakeRetry(context, runId, current); + } + } + }); + }); +} + +export function completeCleanupBookkeeping( + context: SubagentLifecycleWakeContext, + cleanupParams: CleanupBookkeepingParams, + retryDeferredCompletedAnnounces: (excludeRunId?: string) => void, +): void { + const params = context.options; + const suppressSessionEffects = shouldSuppressSubagentRecoverySessionEffects(cleanupParams.entry); + const runCleanupTail = (label: string, run: () => Promise) => { + // These best-effort tails can outlive the durable registry transition, + // but they still mutate session-owned resources and must block snapshots. + void runWithGatewayIndependentRootWorkAdmission(run).catch((error: unknown) => { + defaultRuntime.log( + `[warn] subagent ${label} failed (${cleanupParams.runId}): ${String(error)}`, + ); + }); + }; + const scheduleCleanupTails = (options: { + allowRetiredRow: boolean; + isDeleteCleanup: boolean; + }) => { + // Retained bookkeeping requires the exact row. Immediate retirement + // removes it first, so absence remains ownership only while no newer + // child generation exists; any replacement blocks the stale cleanup. + const postBookkeepingEffectsAllowed = () => { + const current = params.runs.get(cleanupParams.runId); + const rowOwnershipMatches = + current === cleanupParams.entry || (options.allowRetiredRow && current === undefined); + return ( + rowOwnershipMatches && + !context.newerGenerationOwnsSession(cleanupParams.entry) && + !shouldSuppressSubagentRecoverySessionEffects(cleanupParams.entry) + ); + }; + if (postBookkeepingEffectsAllowed() && !cleanupParams.preserveTranscript) { + runCleanupTail("session cleanup", async () => { + if (!postBookkeepingEffectsAllowed()) { + return; + } + await removeInternalSessionEffectsSession(cleanupParams.entry.execution.transcriptTarget); + }); + } + if (postBookkeepingEffectsAllowed() && cleanupParams.entry.spawnMode !== "session") { + runCleanupTail("bundle MCP cleanup", async () => { + if (!postBookkeepingEffectsAllowed()) { + return; + } + await retireSessionMcpRuntimeForSessionKey({ + sessionKey: cleanupParams.entry.childSessionKey, + reason: "subagent-run-cleanup", + preserveActiveLeases: true, + onError: (error, sessionId) => { + params.warn("failed to retire subagent bundle MCP runtime", { + error: buildSafeLifecycleErrorMeta(error), + sessionId, + runId: maskLifecycleIdentifier(cleanupParams.runId, "run"), + childSessionKey: maskLifecycleIdentifier( + cleanupParams.entry.childSessionKey, + "session", + ), + }); + }, + }); + }); + } + if ( + !cleanupParams.provisionalKill && + postBookkeepingEffectsAllowed() && + (options.isDeleteCleanup || !cleanupParams.entry.collect) + ) { + runCleanupTail("context-engine cleanup", async () => { + if (!postBookkeepingEffectsAllowed()) { + return; + } + await params.notifyContextEngineSubagentEnded( + { + childSessionKey: cleanupParams.entry.childSessionKey, + reason: options.isDeleteCleanup ? "deleted" : "completed", + agentDir: cleanupParams.entry.agentDir, + workspaceDir: cleanupParams.entry.workspaceDir, + }, + { isCurrent: postBookkeepingEffectsAllowed }, + ); + }); + } + }; + if (cleanupParams.provisionalKill) { + // The provider result or bounded kill reconciliation owns terminal settle. + // Its kill marker was committed by the caller before reaching this tail. + scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup: false }); + return; + } + const isDeleteCleanup = cleanupParams.cleanup === "delete"; + if (isDeleteCleanup) { + params.clearPendingLifecycleError(cleanupParams.runId); + } + if (cleanupParams.entry.collect) { + // Delete-mode session cleanup already ran before this durable bookkeeping. + // Preserve only the collector result tombstone for waits and group caps. + const previousCleanupCompletedAt = cleanupParams.entry.cleanupCompletedAt; + const previousExecution = cleanupParams.entry.execution; + const previousRequesterSettleWake = cleanupParams.entry.requesterSettleWake; + const previousTerminalOwner = cleanupParams.entry.terminalOwner; + cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; + cleanupParams.entry.requesterSettleWake = undefined; + if (suppressSessionEffects) { + cleanupParams.entry.execution = { + ...cleanupParams.entry.execution, + restartRecovery: undefined, + suppressSessionEffects: true, + }; + cleanupParams.entry.terminalOwner = undefined; + } + try { + params.persistOrThrow(cleanupParams.runId); + } catch (error) { + cleanupParams.entry.cleanupCompletedAt = previousCleanupCompletedAt; + cleanupParams.entry.execution = previousExecution; + cleanupParams.entry.requesterSettleWake = previousRequesterSettleWake; + cleanupParams.entry.terminalOwner = previousTerminalOwner; + throw error; + } + scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup }); + retryDeferredCompletedAnnounces(cleanupParams.runId); + return; + } + const retireAfterSettle = + isDeleteCleanup || + (cleanupParams.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + cleanupParams.entry.suppressAnnounceReason !== "killed"); + if (retireAfterSettle) { + // Reconciled keep-mode kills retire the registry row, not the child session. + if (!isDeleteCleanup) { + params.clearPendingLifecycleError(cleanupParams.runId); + } + if (cleanupParams.skipRequesterSettleWake) { + params.runs.delete(cleanupParams.runId); + try { + params.persistOrThrow(cleanupParams.runId); + } catch (error) { + params.runs.set(cleanupParams.runId, cleanupParams.entry); + throw error; + } + scheduleCleanupTails({ allowRetiredRow: true, isDeleteCleanup }); + retryDeferredCompletedAnnounces(cleanupParams.runId); + return; + } + persistRequesterSettleWakePending(context, cleanupParams.entry, { + cleanupCompletedAt: cleanupParams.completedAt, + retireAfterSettle: true, + retireInterruptedRecovery: suppressSessionEffects, + }); + // The settle wake may synchronously retire this durably marked row before + // the detached tails start. Absence is still stale-safe because any + // replacement row or newer child generation rejects the cleanup. + scheduleCleanupTails({ allowRetiredRow: true, isDeleteCleanup }); + retryDeferredCompletedAnnounces(cleanupParams.runId); + scheduleRequesterSettleWake(context, cleanupParams.runId, cleanupParams.entry); + return; + } + if (!cleanupParams.skipRequesterSettleWake) { + persistRequesterSettleWakePending(context, cleanupParams.entry, { + cleanupCompletedAt: cleanupParams.completedAt, + retireInterruptedRecovery: suppressSessionEffects, + }); + } else { + const previousCleanupCompletedAt = cleanupParams.entry.cleanupCompletedAt; + const previousExecution = cleanupParams.entry.execution; + const previousTerminalOwner = cleanupParams.entry.terminalOwner; + cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; + if (suppressSessionEffects) { + cleanupParams.entry.execution = { + ...cleanupParams.entry.execution, + restartRecovery: undefined, + suppressSessionEffects: true, + }; + cleanupParams.entry.terminalOwner = undefined; + } + try { + params.persistOrThrow(cleanupParams.runId); + } catch (error) { + cleanupParams.entry.cleanupCompletedAt = previousCleanupCompletedAt; + cleanupParams.entry.execution = previousExecution; + cleanupParams.entry.terminalOwner = previousTerminalOwner; + throw error; + } + } + scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup }); + retryDeferredCompletedAnnounces(cleanupParams.runId); + if (!cleanupParams.skipRequesterSettleWake) { + scheduleRequesterSettleWake(context, cleanupParams.runId, cleanupParams.entry); + } +} diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle.test.ts b/src/agents/subagents/registry/subagent-registry-lifecycle.test.ts index 15cc8b328a7d..3ee4d6253d8e 100644 --- a/src/agents/subagents/registry/subagent-registry-lifecycle.test.ts +++ b/src/agents/subagents/registry/subagent-registry-lifecycle.test.ts @@ -29,12 +29,15 @@ import { SUBAGENT_ENDED_REASON_KILLED, } from "./subagent-lifecycle-events.js"; import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; -import { createSubagentRegistryLifecycleController } from "./subagent-registry-lifecycle.js"; +import { + SubagentLifecycleController, + type SubagentLifecycleOptions, +} from "./subagent-registry-lifecycle.js"; import { markSubagentRunPausedAfterYield } from "./subagent-registry-run-manager.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; -type LifecycleControllerParams = Parameters[0]; -type LifecycleController = ReturnType; +type LifecycleControllerParams = SubagentLifecycleOptions; +type LifecycleController = SubagentLifecycleController; type SubagentCompletionParams = Parameters[0]; type AnnounceFlowOutcome = Awaited< ReturnType @@ -109,10 +112,6 @@ const browserLifecycleCleanupMocks = vi.hoisted(() => ({ cleanupBrowserSessionsForLifecycleEnd: vi.fn(async () => {}), })); -const completionSupportMocks = vi.hoisted(() => ({ - loadCleanupBrowserSessionsForLifecycleEnd: vi.fn(), -})); - const bundleMcpRuntimeMocks = vi.hoisted(() => ({ retireSessionMcpRuntimeForSessionKey: vi.fn(async () => true), })); @@ -140,12 +139,6 @@ vi.mock("../../../browser-lifecycle-cleanup.js", () => ({ browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd, })); -vi.mock("./subagent-registry-lifecycle-completion-support.js", async (importOriginal) => ({ - ...(await importOriginal()), - loadCleanupBrowserSessionsForLifecycleEnd: - completionSupportMocks.loadCleanupBrowserSessionsForLifecycleEnd, -})); - vi.mock("../../agent-bundle-mcp-tools.js", () => ({ retireSessionMcpRuntimeForSessionKey: bundleMcpRuntimeMocks.retireSessionMcpRuntimeForSessionKey, })); @@ -373,7 +366,7 @@ function createLifecycleController({ }: { entry: SubagentRunRecord; runs?: Map; -} & Partial[0]>) { +} & Partial) { const params: LifecycleControllerParams = { runs, resumedRuns: new Set(), @@ -394,6 +387,8 @@ function createLifecycleController({ callGateway: async >(opts: CallGatewayOptions): Promise => (await gatewayMocks.callGateway(opts)) as T, captureSubagentCompletionReply: vi.fn(async () => "final completion reply"), + cleanupBrowserSessionsForLifecycleEnd: + browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd, runSubagentAnnounceFlow: vi.fn(async () => "delivered" as const), maybeWakeRequesterAfterAllChildrenSettled: vi.fn( async (wakeParams: { @@ -407,7 +402,7 @@ function createLifecycleController({ warn: vi.fn(), }; Object.assign(params, overrides); - return createSubagentRegistryLifecycleController(params); + return new SubagentLifecycleController(params); } function completeRun( @@ -484,9 +479,6 @@ describe("subagent registry lifecycle hardening", () => { gatewayMocks.callGateway.mockReset(); gatewayMocks.callGateway.mockResolvedValue({}); browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd.mockClear(); - completionSupportMocks.loadCleanupBrowserSessionsForLifecycleEnd - .mockReset() - .mockResolvedValue(browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd); bundleMcpRuntimeMocks.retireSessionMcpRuntimeForSessionKey.mockClear(); bundleMcpRuntimeMocks.retireSessionMcpRuntimeForSessionKey.mockResolvedValue(true); internalSessionEffectsMocks.removeInternalSessionEffectsSession.mockClear(); @@ -2684,14 +2676,17 @@ describe("subagent registry lifecycle hardening", () => { const browserLoaderRelease = new Promise((resolve) => { releaseBrowserLoader = resolve; }); - completionSupportMocks.loadCleanupBrowserSessionsForLifecycleEnd.mockImplementationOnce( - async () => { - markBrowserLoaderEntered(); - await browserLoaderRelease; - return browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd; - }, - ); - const controller = createLifecycleController({ entry, runs }); + const loadCleanupBrowserSessionsForLifecycleEnd = vi.fn(async () => { + markBrowserLoaderEntered(); + await browserLoaderRelease; + return browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd; + }); + const controller = createLifecycleController({ + entry, + runs, + cleanupBrowserSessionsForLifecycleEnd: undefined, + loadCleanupBrowserSessionsForLifecycleEnd, + }); const completion = completeRun(controller, entry, { triggerCleanup: true }); await browserLoaderEntered; @@ -2726,19 +2721,22 @@ describe("subagent registry lifecycle hardening", () => { const browserLoaderRelease = new Promise((resolve) => { releaseBrowserLoader = resolve; }); - completionSupportMocks.loadCleanupBrowserSessionsForLifecycleEnd.mockImplementationOnce( - async () => { - markBrowserLoaderEntered(); - await browserLoaderRelease; - return browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd; - }, - ); + const loadCleanupBrowserSessionsForLifecycleEnd = vi.fn(async () => { + markBrowserLoaderEntered(); + await browserLoaderRelease; + return browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd; + }); const persistOrThrow = vi.fn(() => { if (entry.execution.suppressSessionEffects === true) { throw new Error("suppression persistence failed"); } }); - const controller = createLifecycleController({ entry, persistOrThrow }); + const controller = createLifecycleController({ + entry, + persistOrThrow, + cleanupBrowserSessionsForLifecycleEnd: undefined, + loadCleanupBrowserSessionsForLifecycleEnd, + }); const completion = completeRun(controller, entry, { triggerCleanup: true }); await browserLoaderEntered; diff --git a/src/agents/subagents/registry/subagent-registry-lifecycle.ts b/src/agents/subagents/registry/subagent-registry-lifecycle.ts index db1de3e9f778..10679f10ec37 100644 --- a/src/agents/subagents/registry/subagent-registry-lifecycle.ts +++ b/src/agents/subagents/registry/subagent-registry-lifecycle.ts @@ -1,91 +1,245 @@ import { runWithGatewayIndependentRootWorkAdmission } from "../../../process/gateway-work-admission.js"; -/** - * Subagent registry lifecycle transitions. - * - * Completes/fails task runs, clears delivery state, emits lifecycle events, and cleans attached resources. - */ import type { AcceptedSessionSpawn } from "../../accepted-session-spawn.js"; -import { createSubagentRegistryLifecycleCleanupBase } from "./subagent-registry-lifecycle-cleanup-base.js"; -import { createSubagentRegistryLifecycleCleanup } from "./subagent-registry-lifecycle-cleanup.js"; -import { createSubagentRegistryLifecycleCommon } from "./subagent-registry-lifecycle-common.js"; -import { createSubagentRegistryLifecycleCompletion } from "./subagent-registry-lifecycle-completion.js"; import { - createSubagentRegistryLifecycleState, - type SubagentRegistryLifecycleParams, -} from "./subagent-registry-lifecycle-contracts.js"; -import { createSubagentRegistryLifecycleDelivery } from "./subagent-registry-lifecycle-delivery.js"; -import { createSubagentRegistryLifecycleRequesterWake } from "./subagent-registry-lifecycle-requester-wake.js"; + ensureCompletionState, + ensureDeliveryState, + getDeliveryLastError, +} from "./subagent-delivery-state.js"; +import { + finalizeResumedAnnounceGiveUp, + retryDeferredCompletedAnnounces, + startSubagentAnnounceCleanupFlow, +} from "./subagent-registry-lifecycle-announce-cleanup.js"; +import { completeSubagentRunAttempt } from "./subagent-registry-lifecycle-completion.js"; +import type { + CleanupBookkeepingParams, + ScheduledRequesterSettleWake, + SubagentLifecycleOptions, +} from "./subagent-registry-lifecycle-context.js"; +import { refreshFrozenResultFromSession } from "./subagent-registry-lifecycle-delivery.js"; +import { + completeCleanupBookkeeping, + scheduleRequesterSettleWake, +} from "./subagent-registry-lifecycle-wake.js"; import { settleRequesterTurnAfterSessionSpawns } from "./subagent-registry-requester-yield.js"; -import type { SubagentCompletionRequest } from "./subagent-registry.types.js"; +import type { SubagentCompletionRequest, SubagentRunRecord } from "./subagent-registry.types.js"; +import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; -export function createSubagentRegistryLifecycleController(params: SubagentRegistryLifecycleParams) { - const state = createSubagentRegistryLifecycleState(); - const common = createSubagentRegistryLifecycleCommon(params, state); - const delivery = createSubagentRegistryLifecycleDelivery(params, state, common); - const requesterWake = createSubagentRegistryLifecycleRequesterWake( - params, - state, - common, - delivery, - ); - const cleanupBase = createSubagentRegistryLifecycleCleanupBase( - params, - state, - common, - delivery, - requesterWake, - ); - const cleanup = createSubagentRegistryLifecycleCleanup( - params, - state, - common, - delivery, - requesterWake, - cleanupBase, - ); - const completion = createSubagentRegistryLifecycleCompletion( - params, - state, - common, - delivery, - cleanupBase, - cleanup, - ); - const completeSubagentRun = async (completeParams: SubagentCompletionRequest) => { +export type { SubagentLifecycleOptions } from "./subagent-registry-lifecycle-context.js"; + +export class SubagentLifecycleController { + private readonly scheduledResumeTimers = new Set>(); + private readonly pendingRequesterSettleWakeRearms = new Set(); + private readonly scheduledRequesterSettleWakeRuns = new Set(); + private readonly scheduledRequesterSettleWakeTimers = new Map< + string, + ScheduledRequesterSettleWake + >(); + private readonly terminalCompletionLocks = new Map>(); + private readonly terminalGenerations = new WeakMap(); + private readonly cleanupGenerations = new WeakMap(); + private readonly progressEndedEntries = new WeakSet(); + private readonly cleanupFailureCounts = new WeakMap(); + + constructor(readonly options: SubagentLifecycleOptions) {} + + newerGenerationOwnsSession(entry: SubagentRunRecord): boolean { + return ( + entry.killReconciliation?.supersededAt !== undefined || + Array.from(this.options.runs.values()).some( + (candidate) => + candidate.runId !== entry.runId && + candidate.childSessionKey === entry.childSessionKey && + compareSubagentRunGeneration(candidate, entry) > 0, + ) + ); + } + + async acquireTerminalCompletionLock(runId: string): Promise<() => void> { + const previous = this.terminalCompletionLocks.get(runId) ?? Promise.resolve(); + let releaseLock = () => {}; + const current = new Promise((resolve) => { + releaseLock = resolve; + }); + this.terminalCompletionLocks.set(runId, current); + await previous; + return () => { + releaseLock(); + if (this.terminalCompletionLocks.get(runId) === current) { + this.terminalCompletionLocks.delete(runId); + } + }; + } + + clearScheduledResumeTimers = () => { + for (const timer of this.scheduledResumeTimers) { + clearTimeout(timer); + } + this.scheduledResumeTimers.clear(); + for (const scheduled of this.scheduledRequesterSettleWakeTimers.values()) { + clearTimeout(scheduled.timer); + } + this.scheduledRequesterSettleWakeTimers.clear(); + this.pendingRequesterSettleWakeRearms.clear(); + }; + + addScheduledResumeTimer = (timer: ReturnType): void => + void this.scheduledResumeTimers.add(timer); + deleteScheduledResumeTimer = (timer: ReturnType): void => + void this.scheduledResumeTimers.delete(timer); + + bumpCleanupGeneration(entry: SubagentRunRecord): number { + const generation = (this.cleanupGenerations.get(entry) ?? 0) + 1; + this.cleanupGenerations.set(entry, generation); + return generation; + } + + isCleanupGeneration = (entry: SubagentRunRecord, generation: number): boolean => + this.cleanupGenerations.get(entry) === generation; + isCleanupGenerationCurrent = ( + runId: string, + entry: SubagentRunRecord, + generation: number, + ): boolean => + this.options.runs.get(runId) === entry && + entry.pauseReason !== "sessions_yield" && + this.isCleanupGeneration(entry, generation) && + !this.newerGenerationOwnsSession(entry); + isCleanupAttemptCurrent = ( + runId: string, + entry: SubagentRunRecord, + generation: number, + ): boolean => + entry.cleanupHandled === true && this.isCleanupGenerationCurrent(runId, entry, generation); + isEndedHookOwnerCurrent = (runId: string, entry: SubagentRunRecord): boolean => { + const current = this.options.runs.get(runId); + return (current === undefined || current === entry) && !this.newerGenerationOwnsSession(entry); + }; + + bumpTerminalGeneration(entry: SubagentRunRecord): number { + const generation = (this.terminalGenerations.get(entry) ?? 0) + 1; + this.terminalGenerations.set(entry, generation); + return generation; + } + + isTerminalCallbackCurrent = ( + runId: string, + entry: SubagentRunRecord, + generation: number, + ): boolean => + this.options.runs.get(runId) === entry && + entry.pauseReason !== "sessions_yield" && + this.terminalGenerations.get(entry) === generation; + hasProgressEnded = (entry: SubagentRunRecord): boolean => this.progressEndedEntries.has(entry); + markProgressEnded = (entry: SubagentRunRecord): void => void this.progressEndedEntries.add(entry); + clearCleanupFailureCount = (entry: SubagentRunRecord): void => + void this.cleanupFailureCounts.delete(entry); + + incrementCleanupFailureCount(entry: SubagentRunRecord): number { + const count = (this.cleanupFailureCounts.get(entry) ?? 0) + 1; + this.cleanupFailureCounts.set(entry, count); + return count; + } + + getRequesterSettleWakeTimer = (runId: string): ScheduledRequesterSettleWake | undefined => + this.scheduledRequesterSettleWakeTimers.get(runId); + setRequesterSettleWakeTimer = (runId: string, value: ScheduledRequesterSettleWake): void => + void this.scheduledRequesterSettleWakeTimers.set(runId, value); + deleteRequesterSettleWakeTimer = (runId: string): void => + void this.scheduledRequesterSettleWakeTimers.delete(runId); + hasScheduledRequesterSettleWakeRun = (runId: string): boolean => + this.scheduledRequesterSettleWakeRuns.has(runId); + markRequesterSettleWakeRunScheduled = (runId: string): void => + void this.scheduledRequesterSettleWakeRuns.add(runId); + unmarkRequesterSettleWakeRunScheduled = (runId: string): void => + void this.scheduledRequesterSettleWakeRuns.delete(runId); + markRequesterSettleWakeRearm = (runId: string): void => + void this.pendingRequesterSettleWakeRearms.add(runId); + takeRequesterSettleWakeRearm = (runId: string): boolean => + this.pendingRequesterSettleWakeRearms.delete(runId); + + completeSubagentRun = async (completeParams: SubagentCompletionRequest) => { // Task finalization can make the run disappear from suspension blockers // before browser/MCP retirement and cleanup delivery hand off. Own this // entire transition as an independent root so that boundary stays atomic. // Callers can detach while retaining parent ALS, so nesting is intentional. await runWithGatewayIndependentRootWorkAdmission(async () => { - await completion.completeSubagentRunAttempt(completeParams); + await completeSubagentRunAttempt(this, completeParams); }); }; - return { - clearScheduledResumeTimers: common.clearScheduledResumeTimers, - completeCleanupBookkeeping: cleanup.completeCleanupBookkeeping, - completeSubagentRun, - finalizeResumedAnnounceGiveUp: cleanup.finalizeResumedAnnounceGiveUp, - refreshFrozenResultFromSession: delivery.refreshFrozenResultFromSession, - settleRequesterTurnAfterSessionSpawns: (args: { - requesterSessionKey: string; - requesterTurnRunId: string; - requesterYielded: boolean; - acceptedSessionSpawns: readonly AcceptedSessionSpawn[]; - }) => - settleRequesterTurnAfterSessionSpawns({ - ...args, - runs: params.runs, - persistOrThrow: (...runIds) => params.persistOrThrow(...runIds), - schedule: (runId, entry) => { - if (state.scheduledRequesterSettleWakeRuns.has(runId)) { - state.pendingRequesterSettleWakeRearms.add(runId); - return; - } - requesterWake.scheduleRequesterSettleWake(runId, entry); - }, - }), - resumeRequesterSettleWake: requesterWake.scheduleRequesterSettleWake, - startSubagentAnnounceCleanupFlow: cleanup.startSubagentAnnounceCleanupFlow, + completeCleanupBookkeeping = (params: CleanupBookkeepingParams) => { + completeCleanupBookkeeping(this, params, (excludeRunId) => + retryDeferredCompletedAnnounces(this, excludeRunId), + ); }; + + static discardTerminalDelivery( + this: void, + entry: SubagentRunRecord, + completedAt: number, + reason: "dismissed" | "expired" = "dismissed", + ): void { + const delivery = ensureDeliveryState(entry); + const payload = delivery.payload; + if (reason === "dismissed") { + delivery.disposition = "intentional_non_delivery"; + delivery.dismissedAt = completedAt; + } else { + delivery.discardedAt = completedAt; + delivery.discardReason = "expired"; + delivery.discardedPayloadSummary = { + requesterSessionKey: payload?.requesterSessionKey ?? entry.requesterSessionKey, + childSessionKey: payload?.childSessionKey ?? entry.childSessionKey, + childRunId: payload?.childRunId ?? entry.runId, + endedAt: payload?.endedAt ?? entry.execution.endedAt, + status: payload?.outcome?.status ?? entry.execution.outcome?.status, + lastError: getDeliveryLastError(entry) ?? null, + }; + } + Object.assign(delivery, { status: "discarded", queueId: undefined, nextAttemptAt: undefined }); + delivery.payload = undefined; + Object.assign(delivery, { createdAt: undefined, lastAttemptAt: undefined }); + Object.assign(delivery, { + attemptCount: undefined, + lastError: undefined, + announcedAt: undefined, + }); + Object.assign(delivery, { suspendedAt: undefined, suspendedReason: undefined }); + Object.assign(entry, { wakeOnDescendantSettle: undefined, cleanupHandled: true }); + const completion = ensureCompletionState(entry); + Object.assign(completion, { fallbackResultText: undefined, fallbackCapturedAt: undefined }); + entry.cleanupCompletedAt = completedAt; + } + + finalizeResumedAnnounceGiveUp = (params: Parameters[1]) => + finalizeResumedAnnounceGiveUp(this, params); + + refreshFrozenResultFromSession = (sessionKey: string) => + refreshFrozenResultFromSession(this, sessionKey); + + resumeRequesterSettleWake = (runId: string, entry: SubagentRunRecord) => + scheduleRequesterSettleWake(this, runId, entry); + + settleRequesterTurnAfterSessionSpawns = (args: { + requesterSessionKey: string; + requesterTurnRunId: string; + requesterYielded: boolean; + acceptedSessionSpawns: readonly AcceptedSessionSpawn[]; + }) => + settleRequesterTurnAfterSessionSpawns({ + ...args, + runs: this.options.runs, + persistOrThrow: (...runIds) => this.options.persistOrThrow(...runIds), + schedule: (runId, entry) => { + if (this.hasScheduledRequesterSettleWakeRun(runId)) { + this.markRequesterSettleWakeRearm(runId); + return; + } + scheduleRequesterSettleWake(this, runId, entry); + }, + }); + + startSubagentAnnounceCleanupFlow = (runId: string, entry: SubagentRunRecord): boolean => + startSubagentAnnounceCleanupFlow(this, runId, entry); } diff --git a/src/agents/subagents/registry/subagent-registry-public-api.ts b/src/agents/subagents/registry/subagent-registry-public-api.ts index d0b6e47941a4..790c74a68c9e 100644 --- a/src/agents/subagents/registry/subagent-registry-public-api.ts +++ b/src/agents/subagents/registry/subagent-registry-public-api.ts @@ -4,7 +4,7 @@ import { releaseLeasedAgentSteeringItemsFromSubagentRuns, } from "../../agent-steering-queue.js"; import type { SubagentRegistryDeps } from "./subagent-registry-deps.js"; -import type { createSubagentRegistryLifecycleController } from "./subagent-registry-lifecycle.js"; +import type { SubagentLifecycleController } from "./subagent-registry-lifecycle.js"; import { getSubagentRunsForChildSession } from "./subagent-registry-memory.js"; import { countActiveRunsForSessionFromRuns, @@ -20,9 +20,7 @@ export function createSubagentRegistryPublicApi(config: { persistOrThrow: (...runIds: string[]) => void; restoreOnce: () => void; startAnnounceCleanup: (runId: string, entry: SubagentRunRecord) => boolean; - settleRequesterTurn: ReturnType< - typeof createSubagentRegistryLifecycleController - >["settleRequesterTurnAfterSessionSpawns"]; + settleRequesterTurn: SubagentLifecycleController["settleRequesterTurnAfterSessionSpawns"]; }) { const { runs, diff --git a/src/agents/subagents/registry/subagent-registry-restore.ts b/src/agents/subagents/registry/subagent-registry-restore.ts index 8065e10b4c96..73a7e102fed4 100644 --- a/src/agents/subagents/registry/subagent-registry-restore.ts +++ b/src/agents/subagents/registry/subagent-registry-restore.ts @@ -18,7 +18,7 @@ import { reconcileOrphanedRestoredRuns, updateSubagentArchiveAtMs, } from "./subagent-registry-helpers.js"; -import type { createSubagentRegistryLifecycleController } from "./subagent-registry-lifecycle.js"; +import type { SubagentLifecycleController } from "./subagent-registry-lifecycle.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; import { deleteSubagentSessionForCleanup } from "./subagent-session-cleanup.js"; import { @@ -53,9 +53,7 @@ export function createSubagentRegistryRestorer(config: { deps: () => SubagentRegistryDeps; persist: (...runIds: string[]) => void; persistOrThrow: (...runIds: string[]) => void; - settleRequesterTurn: ReturnType< - typeof createSubagentRegistryLifecycleController - >["settleRequesterTurnAfterSessionSpawns"]; + settleRequesterTurn: SubagentLifecycleController["settleRequesterTurnAfterSessionSpawns"]; ensureListener: () => void; startSweeper: () => void; resumeRun: (runId: string) => void; diff --git a/src/agents/subagents/registry/subagent-registry-suspended-delivery.ts b/src/agents/subagents/registry/subagent-registry-suspended-delivery.ts index f01d7133086e..72e3ccddb80a 100644 --- a/src/agents/subagents/registry/subagent-registry-suspended-delivery.ts +++ b/src/agents/subagents/registry/subagent-registry-suspended-delivery.ts @@ -1,15 +1,11 @@ -import { - ensureCompletionState, - ensureDeliveryState, - getDeliveryLastError, - isDeliverySuspended, -} from "./subagent-delivery-state.js"; +import { isDeliverySuspended } from "./subagent-delivery-state.js"; import { SUBAGENT_ENDED_REASON_COMPLETE, type SubagentLifecycleEndedReason, } from "./subagent-lifecycle-events.js"; import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; import { safeRemoveAttachmentsDir } from "./subagent-registry-helpers.js"; +import type { SubagentLifecycleController } from "./subagent-registry-lifecycle.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; const SUBAGENT_SUSPENDED_DELIVERY_RETENTION_MS = 7 * 24 * 60 * 60_000; @@ -32,6 +28,7 @@ export async function discardSuspendedPendingFinalDelivery(params: { resumedRuns: Set; clearPendingLifecycleError: (runId: string) => void; clearPendingLifecycleTimeout: (runId: string) => void; + discardTerminalDelivery: typeof SubagentLifecycleController.discardTerminalDelivery; completeCleanupBookkeeping: (params: { runId: string; entry: SubagentRunRecord; @@ -53,32 +50,7 @@ export async function discardSuspendedPendingFinalDelivery(params: { const { runId, entry, now, reason, resumedRuns } = params; const snapshot = structuredClone(entry); const wasResumed = resumedRuns.has(runId); - const delivery = ensureDeliveryState(entry); - const payload = delivery.payload; - delivery.status = "discarded"; - delivery.discardedAt = now; - delivery.discardReason = reason; - delivery.discardedPayloadSummary = { - requesterSessionKey: payload?.requesterSessionKey ?? entry.requesterSessionKey, - childSessionKey: payload?.childSessionKey ?? entry.childSessionKey, - childRunId: payload?.childRunId ?? entry.runId, - endedAt: payload?.endedAt ?? entry.execution.endedAt, - status: payload?.outcome?.status ?? entry.execution.outcome?.status, - lastError: getDeliveryLastError(entry) ?? null, - }; - delivery.payload = undefined; - delivery.createdAt = undefined; - delivery.lastAttemptAt = undefined; - delivery.attemptCount = undefined; - delivery.lastError = undefined; - delivery.suspendedAt = undefined; - delivery.suspendedReason = undefined; - entry.wakeOnDescendantSettle = undefined; - const completion = ensureCompletionState(entry); - completion.fallbackResultText = undefined; - completion.fallbackCapturedAt = undefined; - entry.cleanupHandled = true; - delivery.announcedAt = undefined; + params.discardTerminalDelivery(entry, now, reason); const suppressSessionEffects = shouldSuppressSubagentRecoverySessionEffects(entry); const completionReason = entry.endedReason ?? SUBAGENT_ENDED_REASON_COMPLETE; try { diff --git a/src/agents/subagents/registry/subagent-registry-sweeper-recovery.test.ts b/src/agents/subagents/registry/subagent-registry-sweeper-recovery.test.ts index 8ef24d032198..faed61f5d0d1 100644 --- a/src/agents/subagents/registry/subagent-registry-sweeper-recovery.test.ts +++ b/src/agents/subagents/registry/subagent-registry-sweeper-recovery.test.ts @@ -124,6 +124,7 @@ function createHarness(runtime: { current?: GatewayRecoveryRuntime }) { resumeRequesterSettleWake: vi.fn(), startSubagentAnnounceCleanupFlow: vi.fn(() => true), completeCleanupBookkeeping, + discardTerminalDelivery: vi.fn(), shouldEmitEndedHookForRun: vi.fn(() => false), emitSubagentEndedHookForRun, callGateway, diff --git a/src/agents/subagents/registry/subagent-registry-sweeper.ts b/src/agents/subagents/registry/subagent-registry-sweeper.ts index c13de7ed5a1b..d7de56a7b020 100644 --- a/src/agents/subagents/registry/subagent-registry-sweeper.ts +++ b/src/agents/subagents/registry/subagent-registry-sweeper.ts @@ -9,7 +9,10 @@ import { SUBAGENT_ENDED_REASON_ERROR } from "./subagent-lifecycle-events.js"; import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js"; import type { createSubagentRegistryCompletionRuntime } from "./subagent-registry-completion-runtime.js"; import { reconcileOrphanedRun, safeRemoveAttachmentsDir } from "./subagent-registry-helpers.js"; -import type { createSubagentRegistryLifecycleController } from "./subagent-registry-lifecycle.js"; +import type { + SubagentLifecycleController, + SubagentLifecycleOptions, +} from "./subagent-registry-lifecycle.js"; import { createInterruptedRecoveryCoordinator } from "./subagent-registry-restart-recovery-coordinator.js"; import { isRestoredQueuedFailureSettlementClaimed } from "./subagent-registry-restore.js"; import type { createSubagentRunManager } from "./subagent-registry-run-manager.js"; @@ -46,9 +49,6 @@ const restartRecoveryLoader = createLazyImportLoader( ); const killRuntimeLoader = createLazyImportLoader(() => import("./subagent-control.runtime.js")); -type LifecycleController = ReturnType; -type LifecycleOptions = Parameters[0]; - export function createSubagentRegistrySweeper(params: { runs: Map; resumedRuns: Set; @@ -91,11 +91,12 @@ export function createSubagentRegistrySweeper(params: { finalizeInterruptedSubagentRun: ReturnType< typeof createSubagentRegistryCompletionRuntime >["finalizeInterruptedSubagentRun"]; - resumeRequesterSettleWake: LifecycleController["resumeRequesterSettleWake"]; - startSubagentAnnounceCleanupFlow: LifecycleController["startSubagentAnnounceCleanupFlow"]; - completeCleanupBookkeeping: LifecycleController["completeCleanupBookkeeping"]; - shouldEmitEndedHookForRun: LifecycleOptions["shouldEmitEndedHookForRun"]; - emitSubagentEndedHookForRun: LifecycleOptions["emitSubagentEndedHookForRun"]; + resumeRequesterSettleWake: SubagentLifecycleController["resumeRequesterSettleWake"]; + startSubagentAnnounceCleanupFlow: SubagentLifecycleController["startSubagentAnnounceCleanupFlow"]; + completeCleanupBookkeeping: SubagentLifecycleController["completeCleanupBookkeeping"]; + discardTerminalDelivery: typeof SubagentLifecycleController.discardTerminalDelivery; + shouldEmitEndedHookForRun: SubagentLifecycleOptions["shouldEmitEndedHookForRun"]; + emitSubagentEndedHookForRun: SubagentLifecycleOptions["emitSubagentEndedHookForRun"]; callGateway: typeof callGateway; cleanupCollectorLaunchResources: (entry: SubagentRunRecord) => Promise; runContextEngineSubagentEnded: (params: ContextEngineSubagentEndedParams) => Promise; @@ -306,6 +307,7 @@ export function createSubagentRegistrySweeper(params: { resumedRuns, clearPendingLifecycleError: params.clearPendingLifecycleError, clearPendingLifecycleTimeout: params.clearPendingLifecycleTimeout, + discardTerminalDelivery: params.discardTerminalDelivery, completeCleanupBookkeeping: params.completeCleanupBookkeeping, shouldEmitEndedHookForRun: params.shouldEmitEndedHookForRun, emitSubagentEndedHookForRun: params.emitSubagentEndedHookForRun, diff --git a/src/agents/subagents/registry/subagent-registry.persistence.resume.test.ts b/src/agents/subagents/registry/subagent-registry.persistence.resume.test.ts index 2062de97e1d7..0e5664bad7d8 100644 --- a/src/agents/subagents/registry/subagent-registry.persistence.resume.test.ts +++ b/src/agents/subagents/registry/subagent-registry.persistence.resume.test.ts @@ -180,6 +180,46 @@ describe("subagent registry persistence resume", () => { }); }); + it("keeps dismissed terminal delivery dormant and TTL-eligible after restore", async () => { + tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-")); + const stateDir = tempStateDir; + await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => { + const now = Date.now(); + const run: SubagentRunRecord = { + runId: "run-dismissed-delivery", + childSessionKey: "agent:main:subagent:dismissed-delivery", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "retain no delivery obligation", + cleanup: "keep", + createdAt: now - 10 * 60_000, + endedReason: "subagent-complete", + execution: { + status: "terminal", + startedAt: now - 9 * 60_000, + endedAt: now - 8 * 60_000, + outcome: { status: "ok" }, + }, + expectsCompletionMessage: true, + completion: { required: true, resultText: "done", capturedAt: now - 8 * 60_000 }, + delivery: { + status: "discarded", + disposition: "intentional_non_delivery", + dismissedAt: now - 6 * 60_000, + }, + cleanupHandled: true, + cleanupCompletedAt: now - 6 * 60_000, + }; + saveSubagentRegistryToSqlite(new Map([[run.runId, run]])); + + mod.initSubagentRegistry(); + await mod.testing.sweepOnceForTests(); + + expect(announceSpy).not.toHaveBeenCalled(); + expect(mod.getSubagentRunByRunId(run.runId)).toBeUndefined(); + }); + }); + it.each([false, true])( "settles a restored steered requester turn (yielded: %s)", async (requesterYielded) => { diff --git a/src/agents/subagents/registry/subagent-registry.ts b/src/agents/subagents/registry/subagent-registry.ts index 60d347f5b4aa..b1b2e02aca57 100644 --- a/src/agents/subagents/registry/subagent-registry.ts +++ b/src/agents/subagents/registry/subagent-registry.ts @@ -21,7 +21,7 @@ import { type SubagentRegistryDeps, } from "./subagent-registry-deps.js"; import { ANNOUNCE_EXPIRY_MS, reconcileOrphanedRun } from "./subagent-registry-helpers.js"; -import { createSubagentRegistryLifecycleController } from "./subagent-registry-lifecycle.js"; +import { SubagentLifecycleController } from "./subagent-registry-lifecycle.js"; import { createSubagentRegistryListener } from "./subagent-registry-listener.js"; import { getSubagentRunsForChildSession, @@ -55,20 +55,11 @@ import { export type { SubagentRunRecord } from "./subagent-registry.types.js"; const log = createSubsystemLogger("agents/subagent-registry"); -type SubagentRegistryRestorer = ReturnType; -type SubagentRegistryBootstrapState = { +const subagentRegistryBootstrapState: { pending?: boolean; ready?: boolean; - restorer?: SubagentRegistryRestorer; -}; - -function getSubagentRegistryBootstrapState(): SubagentRegistryBootstrapState { - const owner = getSubagentRegistryBootstrapState as typeof getSubagentRegistryBootstrapState & { - state?: SubagentRegistryBootstrapState; - }; - owner.state ??= {}; - return owner.state; -} + restorer?: ReturnType; +} = {}; const resumeRetryTimers = new Set>(); const SUBAGENT_ANNOUNCE_TIMEOUT_MS = 120_000; @@ -132,7 +123,7 @@ const contextCleanup = createSubagentRegistryContextCleanup({ warn: (message, meta) => log.warn(message, meta), }); -const subagentLifecycleController = createSubagentRegistryLifecycleController({ +const subagentLifecycleController = new SubagentLifecycleController({ runs: subagentRuns, resumedRuns, subagentAnnounceTimeoutMs: SUBAGENT_ANNOUNCE_TIMEOUT_MS, @@ -140,7 +131,9 @@ const subagentLifecycleController = createSubagentRegistryLifecycleController({ persist: persistSubagentRuns, persistOrThrow: persistSubagentRunsOrThrow, clearPendingLifecycleError, - countPendingDescendantRuns, + // Lifecycle wiring precedes publicApi construction; inject this read query + // as a late-bound callback instead of threading a partially built API object. + countPendingDescendantRuns: (rootSessionKey) => countPendingDescendantRuns(rootSessionKey), suppressAnnounceForSteerRestart: contextCleanup.suppressAnnounceForSteerRestart, resolveSubagentTask: findSubagentTaskForRun, shouldEmitEndedHookForRun: contextCleanup.shouldEmitEndedHookForRun, @@ -408,6 +401,7 @@ const subagentSweeper = createSubagentRegistrySweeper({ resumeRequesterSettleWake, startSubagentAnnounceCleanupFlow, completeCleanupBookkeeping, + discardTerminalDelivery: SubagentLifecycleController.discardTerminalDelivery, shouldEmitEndedHookForRun: contextCleanup.shouldEmitEndedHookForRun, emitSubagentEndedHookForRun: contextCleanup.emitSubagentEndedHookForRun, callGateway: (request) => subagentRegistryDeps.callGateway(request), @@ -569,6 +563,7 @@ function addSubagentRunForTests(entry: SubagentRunRecord) { } export const markSubagentRunTerminated = subagentRunManager.markSubagentRunTerminated; +export const discardSubagentTerminalDelivery = SubagentLifecycleController.discardTerminalDelivery; export { prependAgentSteeringPrompt }; @@ -593,7 +588,7 @@ export const listSwarmRunsForGroup = publicApi.listSwarmRunsForGroup; export const getSwarmRunByLaunchReplayKey = publicApi.getSwarmRunByLaunchReplayKey; export const countActiveRunsForSession = publicApi.countActiveRunsForSession; export function initSubagentRegistry() { - const state = getSubagentRegistryBootstrapState(); + const state = subagentRegistryBootstrapState; if (!state.ready || !state.restorer) { state.pending = true; return; @@ -603,7 +598,7 @@ export function initSubagentRegistry() { export const settleRequesterAfterSessionSpawns = publicApi.settleRequesterAfterSessionSpawns; export const markRequesterTurnYielded = publicApi.markRequesterTurnYielded; -const bootstrapState = getSubagentRegistryBootstrapState(); +const bootstrapState = subagentRegistryBootstrapState; bootstrapState.restorer = subagentRestorer; bootstrapState.ready = true; if (bootstrapState.pending) { diff --git a/src/gateway/server-methods/tasks.ts b/src/gateway/server-methods/tasks.ts index 0dc88e478a0e..d1385fa90758 100644 --- a/src/gateway/server-methods/tasks.ts +++ b/src/gateway/server-methods/tasks.ts @@ -161,20 +161,24 @@ export const tasksHandlers: GatewayRequestHandlers = { } respond(true, { results }); }, - "tasks.dismiss": ({ params, respond }) => { + "tasks.dismiss": async ({ params, respond }) => { if (!assertValidParams(params, validateTasksRecoveryParams, "tasks.dismiss", respond)) { return; } - respond(true, { - results: params.taskIds.map((taskId) => { - const result = dismissSubagentCompletionDelivery(taskId); - return { - taskId, - ok: result.ok, - ...(result.reason ? { reason: result.reason } : {}), - ...(result.task ? { task: mapTaskSummary(result.task, { includePrompt: true }) } : {}), - }; - }), - }); + const { discardSubagentTerminalDelivery } = + await import("../../agents/subagents/registry/subagent-registry.js"); + const results = []; + for (const taskId of params.taskIds) { + const result = await dismissSubagentCompletionDelivery(taskId, { + discardTerminalDelivery: discardSubagentTerminalDelivery, + }); + results.push({ + taskId, + ok: result.ok, + ...(result.reason ? { reason: result.reason } : {}), + ...(result.task ? { task: mapTaskSummary(result.task, { includePrompt: true }) } : {}), + }); + } + respond(true, { results }); }, }; diff --git a/src/gateway/session-message-events.test.ts b/src/gateway/session-message-events.test.ts index fe7a6bdbe282..e854bcf6b053 100644 --- a/src/gateway/session-message-events.test.ts +++ b/src/gateway/session-message-events.test.ts @@ -15,7 +15,7 @@ import { } from "../../packages/gateway-protocol/src/client-info.js"; import { SESSION_VIEWER_PRESENCE_MAX_KEYS } from "../../packages/gateway-protocol/src/schema/sessions-viewer-presence.js"; import { SUBAGENT_ENDED_REASON_ERROR } from "../agents/subagents/registry/subagent-lifecycle-events.js"; -import { createSubagentRegistryLifecycleController } from "../agents/subagents/registry/subagent-registry-lifecycle.js"; +import { SubagentLifecycleController } from "../agents/subagents/registry/subagent-registry-lifecycle.js"; import type { SubagentRunRecord } from "../agents/subagents/registry/subagent-registry.types.js"; import { formatSqliteSessionFileMarker } from "../config/sessions/legacy-sqlite-marker.js"; import { @@ -646,7 +646,7 @@ describe("session.message websocket events", () => { }); const emitSubagentProgressEndedForRun = vi.fn(async () => {}); - const controller = createSubagentRegistryLifecycleController({ + const controller = new SubagentLifecycleController({ runs: new Map([[entry.runId, entry]]), resumedRuns: new Set(), subagentAnnounceTimeoutMs: 1_000,