refactor(auto-reply): split reply agent runner (#113072)

* refactor(auto-reply): split reply agent runner

* fix(auto-reply): keep split contracts acyclic

* fix(auto-reply): isolate result contracts
This commit is contained in:
Peter Steinberger
2026-07-23 09:58:29 -07:00
committed by GitHub
parent 0cd7ea0089
commit b09dd1fec9
11 changed files with 3770 additions and 3087 deletions
-1
View File
@@ -567,7 +567,6 @@ src/auto-reply/reply/agent-runner-memory.ts
src/auto-reply/reply/agent-runner-payloads.test.ts
src/auto-reply/reply/agent-runner.misc.runreplyagent.test.ts
src/auto-reply/reply/agent-runner.runreplyagent.e2e.test.ts
src/auto-reply/reply/agent-runner.ts
src/auto-reply/reply/commands-acp.test.ts
src/auto-reply/reply/commands-approve.test.ts
src/auto-reply/reply/commands-models.ts
+547
View File
@@ -0,0 +1,547 @@
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { hasSessionAutoModelFallbackProvenance } from "../../agents/agent-scope.js";
import { hasVisibleCommittedMessagingToolDeliveryEvidence } from "../../agents/embedded-agent-runner/delivery-evidence.js";
import { enqueueCommitmentExtraction } from "../../commitments/runtime.js";
import type { OpenClawConfig } from "../../config/config.js";
import {
resolveSessionPluginStatusLines,
resolveSessionPluginTraceLines,
type SessionEntry,
} from "../../config/sessions.js";
import { loadSessionEntryReadOnly } from "../../config/sessions/session-accessor.js";
import {
formatSqliteSessionFileMarker,
sqliteSessionFileMarkerMatchesSession,
} from "../../config/sessions/sqlite-marker.js";
import { parseSessionThreadInfoFast } from "../../config/sessions/thread-info.js";
import type { TypingMode } from "../../config/types.js";
import { logVerbose } from "../../globals.js";
import { CommandLaneClearedError, GatewayDrainingError } from "../../process/command-queue.js";
import { resolveSendPolicy } from "../../sessions/send-policy.js";
import {
type DeliveryContext,
normalizeDeliveryContext,
} from "../../utils/delivery-context.shared.js";
import { resolveFallbackTransition } from "../fallback-state.js";
import { stripHeartbeatToken } from "../heartbeat.js";
import {
isReplyPayloadStatusNotice,
markReplyPayloadForSourceSuppressionDelivery,
setReplyPayloadMetadata,
} from "../reply-payload.js";
import type { TemplateContext } from "../templating.js";
import type { VerboseLevel } from "../thinking.js";
import { SILENT_REPLY_TOKEN } from "../tokens.js";
import type { GetReplyOptions, ReplyPayload } from "../types.js";
import { buildKnownAgentRunFailureReplyPayload } from "./agent-runner-failure-reply.js";
import type { BlockReplyPipeline } from "./block-reply-pipeline.js";
import { resolveEffectiveReplyRoute } from "./effective-reply-route.js";
import type { InternalGetReplyOptions } from "./get-reply.types.js";
import { normalizeReplyPayload } from "./normalize-reply.js";
import { resolveOriginMessageTo } from "./origin-routing.js";
import {
buildPendingFinalDeliveryText,
sanitizePendingFinalDeliveryText,
} from "./pending-final-delivery.js";
import { type FollowupRun, type QueueSettings, scheduleFollowupDrain } from "./queue.js";
import { normalizeReplyPayloadDirectives } from "./reply-delivery.js";
import { type ReplyOperation, runAfterReplyOperationClear } from "./reply-run-registry.js";
import { resolveSourceReplyVisibilityPolicy } from "./source-reply-delivery-mode.js";
import type { TypingController } from "./typing.js";
export const BLOCK_REPLY_SEND_TIMEOUT_MS = 15_000;
const RESTART_LIFECYCLE_REPLY_TEXT =
"⚠️ Gateway is restarting. Please wait a few seconds and try again.";
export function scheduleFollowupDrainAfterReplyOperationClear(params: {
operation: ReplyOperation;
queueKey: string;
runFollowup: (run: FollowupRun) => Promise<void>;
}): void {
runAfterReplyOperationClear(params.operation, (admissionSessionId) => {
const completedSessionId = params.operation.sessionId;
const runFollowupAfterClear =
admissionSessionId === completedSessionId
? params.runFollowup
: (queued: FollowupRun) =>
params.runFollowup(
queued.run.sessionId === completedSessionId
? { ...queued, admissionSessionId }
: queued,
);
scheduleFollowupDrain(params.queueKey, runFollowupAfterClear);
});
}
export function markBeforeAgentRunBlockedPayloads(payloads: ReplyPayload[]): ReplyPayload[] {
return payloads.map((payload) =>
setReplyPayloadMetadata(payload, { beforeAgentRunBlocked: true }),
);
}
export function resolvePendingFinalDeliveryRetryText(params: {
isHeartbeat: boolean;
payload: ReplyPayload;
}): string {
const pendingText = buildPendingFinalDeliveryText([params.payload]);
if (!params.isHeartbeat) {
return pendingText;
}
const stripped = stripHeartbeatToken(pendingText, { mode: "message" });
return stripped.shouldSkip ? "" : stripped.text || pendingText;
}
export function buildSilentFallbackFailurePayload(params: {
fallbackTransition: ReturnType<typeof resolveFallbackTransition>;
fallbackFailureKnown: boolean;
isHeartbeat: boolean;
hasSuccessfulTerminalDelivery: boolean;
allowEmptyAssistantReplyAsSilent?: boolean;
silentExpected?: boolean;
}): ReplyPayload | undefined {
if (
params.isHeartbeat ||
params.allowEmptyAssistantReplyAsSilent === true ||
params.silentExpected === true ||
params.hasSuccessfulTerminalDelivery ||
!params.fallbackTransition.fallbackActive ||
!params.fallbackFailureKnown
) {
return undefined;
}
return markReplyPayloadForSourceSuppressionDelivery({
text:
`⚠️ I couldn't reach the configured model backend ${params.fallbackTransition.selectedModelRef}. ` +
`Fallback used ${params.fallbackTransition.activeModelRef}, but it produced no visible reply.`,
isError: true,
});
}
export function resolveSourceReplyPolicy(params: {
cfg: OpenClawConfig;
sessionCtx: TemplateContext;
sessionEntry?: SessionEntry;
sessionKey: string;
runtimePolicySessionKey?: string;
opts?: GetReplyOptions;
}): ReturnType<typeof resolveSourceReplyVisibilityPolicy> {
const sendPolicy = resolveSendPolicy({
cfg: params.cfg,
entry: params.sessionEntry,
sessionKey: params.runtimePolicySessionKey ?? params.sessionKey,
channel:
params.sessionCtx.OriginatingChannel ??
params.sessionCtx.Surface ??
params.sessionCtx.Provider ??
params.sessionEntry?.channel,
chatType: params.sessionEntry?.chatType,
});
return resolveSourceReplyVisibilityPolicy({
cfg: params.cfg,
ctx: params.sessionCtx,
requested: params.opts?.sourceReplyDeliveryMode,
sendPolicy,
});
}
export function resolveReplyRunDeliveryContext(params: {
cfg: OpenClawConfig;
sessionCtx: TemplateContext;
sessionEntry?: SessionEntry;
sessionKey: string;
runtimePolicySessionKey?: string;
opts?: GetReplyOptions;
}): DeliveryContext | undefined {
const sourceReplyPolicy = resolveSourceReplyPolicy(params);
if (
params.sessionCtx.InboundEventKind === "room_event" ||
sourceReplyPolicy.sendPolicyDenied ||
(sourceReplyPolicy.suppressDelivery &&
sourceReplyPolicy.sourceReplyDeliveryMode !== "message_tool_only")
) {
return undefined;
}
const threadId =
normalizeOptionalString(params.sessionCtx.MessageThreadId) ??
normalizeOptionalString(params.sessionCtx.TransportThreadId) ??
normalizeOptionalString(
parseSessionThreadInfoFast(params.sessionCtx.SessionKey ?? params.sessionKey).threadId,
);
return normalizeDeliveryContext({
...resolveEffectiveReplyRoute({
ctx: params.sessionCtx,
entry: params.sessionEntry,
}),
threadId,
});
}
export function hasSuccessfulSourceReplyDelivery(params: {
blockReplyPipeline: { didStream: () => boolean; isAborted: () => boolean } | null;
directlySentBlockKeys?: Set<string>;
messagingToolSentTexts?: string[];
messagingToolSentMediaUrls?: string[];
messagingToolSentTargets?: unknown[];
}): boolean {
return (
(params.blockReplyPipeline?.didStream() && !params.blockReplyPipeline.isAborted()) ||
(params.directlySentBlockKeys?.size ?? 0) > 0 ||
hasVisibleCommittedMessagingToolDeliveryEvidence(params)
);
}
export function hasSuccessfulTerminalSourceReplyDelivery(params: {
blockReplyPipeline: {
didStreamTerminalReply?: () => boolean;
isAborted: () => boolean;
} | null;
directlySentBlockPayloads?: ReplyPayload[];
}): boolean {
const sentTerminalBlock = params.directlySentBlockPayloads?.some(
(payload) =>
payload.isReasoning !== true &&
payload.isCommentary !== true &&
!isReplyPayloadStatusNotice(payload) &&
normalizeReplyPayload(payload, { applyChannelTransforms: false }) !== null,
);
return (
(params.blockReplyPipeline?.didStreamTerminalReply?.() === true &&
!params.blockReplyPipeline.isAborted()) ||
sentTerminalBlock === true
);
}
export function resolveConfiguredFallbackModel(params: {
run: FollowupRun["run"];
fallbackStateEntry?: SessionEntry;
}): { provider: string; model: string; persistedAutoFallback: boolean } {
const entry = params.fallbackStateEntry;
const isAutoFallbackOverride =
entry?.modelOverrideSource === "auto" ||
(entry !== undefined &&
entry.modelOverrideSource === undefined &&
hasSessionAutoModelFallbackProvenance(entry));
if (isAutoFallbackOverride && entry !== undefined) {
const originProvider = normalizeOptionalString(entry.modelOverrideFallbackOriginProvider);
const originModel = normalizeOptionalString(entry.modelOverrideFallbackOriginModel);
if (originProvider && originModel) {
return { provider: originProvider, model: originModel, persistedAutoFallback: true };
}
}
return {
provider: params.run.provider,
model: params.run.model,
persistedAutoFallback: false,
};
}
export function buildInlinePluginStatusPayload(params: {
entry: SessionEntry | undefined;
includeTraceLines: boolean;
}): ReplyPayload | undefined {
const statusLines =
params.entry?.verboseLevel && params.entry.verboseLevel !== "off"
? resolveSessionPluginStatusLines(params.entry)
: [];
const traceLines =
params.includeTraceLines &&
(params.entry?.traceLevel === "on" || params.entry?.traceLevel === "raw")
? resolveSessionPluginTraceLines(params.entry)
: [];
const lines = [...statusLines, ...traceLines];
if (lines.length === 0) {
return undefined;
}
return { text: lines.join("\n") };
}
function joinCommitmentAssistantText(payloads: ReplyPayload[]): string {
return payloads
.filter(
(payload) => !payload.isError && !payload.isReasoning && !isReplyPayloadStatusNotice(payload),
)
.map((payload) => payload.text?.trim())
.filter((text): text is string => Boolean(text))
.join("\n")
.trim();
}
export function normalizeAssistantFinalDeliveryText(text: string): string {
const parsed = normalizeReplyPayloadDirectives({
payload: { text },
trimLeadingWhitespace: true,
parseMode: "auto",
});
return sanitizePendingFinalDeliveryText(parsed.payload.text ?? "");
}
export function enqueueCommitmentExtractionForTurn(params: {
cfg: OpenClawConfig;
commandBody: string;
isHeartbeat: boolean;
followupRun: FollowupRun;
sessionCtx: TemplateContext;
sessionKey?: string;
replyToChannel?: string;
payloads: ReplyPayload[];
runId: string;
}): void {
if (params.isHeartbeat) {
return;
}
const userText =
params.commandBody.trim() ||
params.sessionCtx.BodyStripped?.trim() ||
params.sessionCtx.BodyForCommands?.trim() ||
params.sessionCtx.CommandBody?.trim() ||
params.sessionCtx.RawBody?.trim() ||
params.sessionCtx.Body?.trim() ||
"";
const assistantText = joinCommitmentAssistantText(params.payloads);
const sessionKey = params.sessionKey ?? params.followupRun.run.sessionKey;
const channel =
params.replyToChannel ??
params.followupRun.run.messageProvider ??
params.sessionCtx.Surface ??
params.sessionCtx.Provider;
if (!userText || !assistantText || !sessionKey || !channel) {
return;
}
const to = resolveOriginMessageTo({
originatingTo: params.sessionCtx.OriginatingTo,
to: params.sessionCtx.To,
});
enqueueCommitmentExtraction({
cfg: params.cfg,
agentId: params.followupRun.run.agentId,
sessionKey,
channel,
...(params.sessionCtx.AccountId ? { accountId: params.sessionCtx.AccountId } : {}),
...(to ? { to } : {}),
...(params.sessionCtx.MessageThreadId !== undefined
? { threadId: String(params.sessionCtx.MessageThreadId) }
: {}),
...(params.followupRun.run.senderId ? { senderId: params.followupRun.run.senderId } : {}),
userText,
assistantText,
...(params.sessionCtx.MessageSidFull || params.sessionCtx.MessageSid
? { sourceMessageId: params.sessionCtx.MessageSidFull ?? params.sessionCtx.MessageSid }
: {}),
sourceRunId: params.runId,
});
}
export function refreshSessionEntryFromStore(params: {
storePath?: string;
sessionKey?: string;
fallbackEntry?: SessionEntry;
activeSessionStore?: Record<string, SessionEntry>;
}): SessionEntry | undefined {
const { storePath, sessionKey, fallbackEntry, activeSessionStore } = params;
if (!storePath || !sessionKey) {
return fallbackEntry;
}
try {
const latestEntry = loadSessionEntryReadOnly({
storePath,
sessionKey,
});
if (!latestEntry) {
return fallbackEntry;
}
if (activeSessionStore) {
activeSessionStore[sessionKey] = latestEntry;
}
return latestEntry;
} catch {
return fallbackEntry;
}
}
export function resolveAdmittedRunSessionFile(params: {
agentId: string;
sessionId: string;
sessionFile?: string;
storePath?: string;
}): string | undefined {
if (
params.sessionFile &&
sqliteSessionFileMarkerMatchesSession(params.sessionFile, params.sessionId)
) {
return params.sessionFile;
}
if (params.storePath) {
return formatSqliteSessionFileMarker({
agentId: params.agentId,
sessionId: params.sessionId,
storePath: params.storePath,
});
}
return params.sessionFile;
}
export async function handleReplyAgentRunError(
error: unknown,
context: {
cfg: OpenClawConfig;
isRestartRecoveryArmed: () => boolean;
replyOperation: ReplyOperation;
resolvedVerboseLevel: VerboseLevel;
returnWithQueuedFollowupDrain: <T>(value: T) => T;
sessionCtx: TemplateContext;
},
): Promise<ReplyPayload | undefined> {
const {
cfg,
isRestartRecoveryArmed,
replyOperation,
resolvedVerboseLevel,
returnWithQueuedFollowupDrain,
sessionCtx,
} = context;
if (
replyOperation.result?.kind === "aborted" &&
replyOperation.result.code === "aborted_by_user"
) {
return returnWithQueuedFollowupDrain({ text: SILENT_REPLY_TOKEN });
}
if (
replyOperation.result?.kind === "aborted" &&
replyOperation.result.code === "aborted_for_restart"
) {
if (isRestartRecoveryArmed()) {
return returnWithQueuedFollowupDrain({ text: SILENT_REPLY_TOKEN });
}
return returnWithQueuedFollowupDrain(
markReplyPayloadForSourceSuppressionDelivery({
text: RESTART_LIFECYCLE_REPLY_TEXT,
}),
);
}
if (error instanceof GatewayDrainingError) {
replyOperation.fail("gateway_draining", error);
return returnWithQueuedFollowupDrain(
markReplyPayloadForSourceSuppressionDelivery({
text: RESTART_LIFECYCLE_REPLY_TEXT,
}),
);
}
if (error instanceof CommandLaneClearedError) {
replyOperation.fail("command_lane_cleared", error);
return returnWithQueuedFollowupDrain(
markReplyPayloadForSourceSuppressionDelivery({
text: RESTART_LIFECYCLE_REPLY_TEXT,
}),
);
}
const knownFailurePayload = buildKnownAgentRunFailureReplyPayload({
err: error,
sessionCtx,
resolvedVerboseLevel,
cfg,
});
if (knownFailurePayload) {
replyOperation.fail("run_failed", error);
return returnWithQueuedFollowupDrain(knownFailurePayload);
}
replyOperation.fail("run_failed", error);
// Keep the followup queue moving even when an unexpected exception escapes
// the run path; the caller still receives the original error.
returnWithQueuedFollowupDrain(undefined);
throw error;
}
export async function cleanupReplyAgentRun(context: {
blockReplyPipeline: BlockReplyPipeline | null;
clearRestartRecoveryDeliveryClaim: () => Promise<void>;
providedReplyOperation: ReplyOperation | undefined;
queueKey: string;
replyOperation: ReplyOperation;
runFollowupTurn: (queued: FollowupRun) => Promise<void>;
sessionKey: string | undefined;
shouldDrainQueuedFollowupsAfterClear: boolean;
typing: TypingController;
}): Promise<void> {
const {
blockReplyPipeline,
clearRestartRecoveryDeliveryClaim,
providedReplyOperation,
queueKey,
replyOperation,
runFollowupTurn,
sessionKey,
shouldDrainQueuedFollowupsAfterClear,
typing,
} = context;
try {
await clearRestartRecoveryDeliveryClaim();
} catch (error) {
logVerbose(
`failed to clear restart recovery delivery context for ${sessionKey ?? "unknown"}: ${String(
error,
)}`,
);
}
if (shouldDrainQueuedFollowupsAfterClear) {
scheduleFollowupDrainAfterReplyOperationClear({
operation: replyOperation,
queueKey,
runFollowup: runFollowupTurn,
});
if (!providedReplyOperation) {
replyOperation.complete();
}
} else if (!providedReplyOperation) {
replyOperation.complete();
}
blockReplyPipeline?.stop();
typing.markRunComplete();
// Safety net: the dispatcher's onIdle callback normally fires
// markDispatchIdle(), but if the dispatcher exits early, errors,
// or the reply path doesn't go through it cleanly, the second
// signal never fires and the typing keepalive loop runs forever.
// Calling this twice is harmless — cleanup() is guarded by the
// `active` flag. Same pattern as the followup runner fix (#26881).
typing.markDispatchIdle();
}
export type RunReplyAgentParams = {
commandBody: string;
transcriptCommandBody?: string;
followupRun: FollowupRun;
queueKey: string;
resolvedQueue: QueueSettings;
shouldSteer: boolean;
shouldFollowup: boolean;
isActive: boolean;
isRunActive?: () => boolean;
isStreaming: boolean;
opts?: InternalGetReplyOptions;
typing: TypingController;
sessionEntry?: SessionEntry;
sessionStore?: Record<string, SessionEntry>;
sessionKey?: string;
runtimePolicySessionKey?: string;
storePath?: string;
defaultModel: string;
agentCfgContextTokens?: number;
resolvedVerboseLevel: VerboseLevel;
toolProgressDetail?: "explain" | "raw";
isNewSession: boolean;
blockStreamingEnabled: boolean;
blockReplyChunking?: {
minChars: number;
maxChars: number;
breakPreference: "paragraph" | "newline" | "sentence";
flushOnParagraph?: boolean;
};
resolvedBlockStreamingBreak: "text_end" | "message_end";
sessionCtx: TemplateContext;
shouldInjectGroupIntro: boolean;
typingMode: TypingMode;
resetTriggered?: boolean;
replyThreadingOverride?: TemplateContext["ReplyThreading"];
replyOperation?: ReplyOperation;
};
@@ -0,0 +1,552 @@
import crypto from "node:crypto";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { isLikelyContextOverflowError } from "../../agents/embedded-agent-helpers/errors.js";
import type { OpenClawConfig } from "../../config/config.js";
import type { SessionEntry } from "../../config/sessions.js";
import { logVerbose } from "../../globals.js";
import { withBeforeAgentReplyObserver } from "../../plugins/before-agent-reply.js";
import { setReplyPayloadMetadata } from "../reply-payload.js";
import type { OriginatingChannelType } from "../templating.js";
import { SILENT_REPLY_TOKEN } from "../tokens.js";
import type { ReplyPayload } from "../types.js";
import {
resolveReplyRunDeliveryContext,
resolveSourceReplyPolicy,
type RunReplyAgentParams,
} from "./agent-runner-core.js";
import { runAgentTurnWithFallback } from "./agent-runner-execution.js";
import { runMemoryFlushIfNeeded, runPreflightCompactionIfNeeded } from "./agent-runner-memory.js";
import { finalizeReplyAgentRun } from "./agent-runner-result.js";
import { buildThreadingToolContext } from "./agent-runner-utils.js";
import type { BlockReplyPipeline } from "./block-reply-pipeline.js";
import type { CompactionNoticePhase } from "./compaction-notice.js";
import { createFollowupRunner } from "./followup-runner.js";
import {
buildRecoverablePendingFinalDeliveryText,
normalizePendingFinalDeliveryPayloads,
} from "./pending-final-delivery.js";
import type { FollowupRun } from "./queue.js";
import type { ReplyMediaContext } from "./reply-media-paths.js";
import type { ReplyOperation } from "./reply-run-registry.js";
import { resolveReplyToMode } from "./reply-threading.js";
import { createReplyRestartRecoveryClaimController } from "./restart-recovery-claim.js";
import { resolveRoutedDeliveryThreadId } from "./routed-delivery-thread.js";
import type { TypingSignaler } from "./typing-mode.js";
type SessionResetOptions = {
failureLabel: string;
buildLogMessage: (nextSessionId: string) => string;
cleanupTranscripts?: boolean;
};
type ExecutePreparedReplyAgentRunInput = Pick<
RunReplyAgentParams,
| "agentCfgContextTokens"
| "blockReplyChunking"
| "blockStreamingEnabled"
| "commandBody"
| "defaultModel"
| "followupRun"
| "opts"
| "queueKey"
| "replyThreadingOverride"
| "resolvedBlockStreamingBreak"
| "resolvedQueue"
| "resolvedVerboseLevel"
| "runtimePolicySessionKey"
| "sessionCtx"
| "sessionKey"
| "shouldInjectGroupIntro"
| "storePath"
| "toolProgressDetail"
| "transcriptCommandBody"
| "typing"
| "typingMode"
> & {
activeSessionStore: Record<string, SessionEntry> | undefined;
admitUserTurn: ReturnType<typeof createReplyRestartRecoveryClaimController>["admitUserTurn"];
applyReplyToMode: (payload: ReplyPayload) => ReplyPayload;
beforeAgentReplyDispatchedForSteer: boolean;
beginBeforeAgentReply: ReturnType<
typeof createReplyRestartRecoveryClaimController
>["beginBeforeAgentReply"];
blockReplyPipeline: BlockReplyPipeline | null;
cfg: OpenClawConfig;
checkpointBeforeAgentReply: ReturnType<
typeof createReplyRestartRecoveryClaimController
>["checkpointBeforeAgentReply"];
getActiveIsNewSession: () => boolean;
getActiveSessionEntry: () => SessionEntry | undefined;
isHeartbeat: boolean;
isRestartRecoveryArmed: () => boolean;
pendingToolTasks: Set<Promise<void>>;
performSessionReset: (options: SessionResetOptions) => Promise<boolean>;
replyMediaContext: ReplyMediaContext;
replyOperation: ReplyOperation;
replyRouteThreadId: ReturnType<typeof resolveRoutedDeliveryThreadId>;
replyToChannel: OriginatingChannelType | undefined;
replyToMode: ReturnType<typeof resolveReplyToMode>;
resetSessionAfterRoleOrderingConflict: (reason: string) => Promise<boolean>;
returnWithQueuedFollowupDrain: <T>(value: T) => T;
runFollowupTurn: (queued: FollowupRun) => Promise<void>;
sendDirectCompactionNotice: ((phase: CompactionNoticePhase) => Promise<void>) | undefined;
setRunFollowupTurn: (runner: (queued: FollowupRun) => Promise<void>) => void;
setActiveSessionEntry: (entry: SessionEntry | undefined) => void;
shouldEmitToolOutput: () => boolean;
shouldEmitToolResult: () => boolean;
traceAgentPhase: <T>(name: string, run: () => Promise<T> | T) => Promise<T>;
turnAdoptionLifecycle: NonNullable<RunReplyAgentParams["opts"]>["turnAdoptionLifecycle"];
typingSignals: TypingSignaler;
};
export async function executePreparedReplyAgentRun(
context: ExecutePreparedReplyAgentRunInput,
): Promise<ReplyPayload | ReplyPayload[] | undefined> {
const {
activeSessionStore,
admitUserTurn: admitUserTurnWithRecovery,
agentCfgContextTokens,
applyReplyToMode,
beforeAgentReplyDispatchedForSteer,
beginBeforeAgentReply: beginBeforeAgentReplyWithRecovery,
blockReplyChunking,
blockReplyPipeline,
blockStreamingEnabled,
cfg,
checkpointBeforeAgentReply: checkpointBeforeAgentReplyWithRecovery,
commandBody,
defaultModel,
followupRun,
getActiveIsNewSession,
getActiveSessionEntry,
isHeartbeat,
isRestartRecoveryArmed,
opts,
pendingToolTasks,
performSessionReset,
queueKey,
replyMediaContext,
replyOperation,
replyRouteThreadId,
replyThreadingOverride,
replyToChannel,
replyToMode,
resetSessionAfterRoleOrderingConflict,
resolvedBlockStreamingBreak,
resolvedQueue,
resolvedVerboseLevel,
returnWithQueuedFollowupDrain,
runtimePolicySessionKey,
sendDirectCompactionNotice,
sessionCtx,
sessionKey,
setActiveSessionEntry,
setRunFollowupTurn,
shouldEmitToolOutput,
shouldEmitToolResult,
shouldInjectGroupIntro,
storePath,
toolProgressDetail,
traceAgentPhase,
transcriptCommandBody,
turnAdoptionLifecycle,
typing,
typingMode,
typingSignals,
} = context;
let activeSessionEntry = getActiveSessionEntry();
let activeIsNewSession: boolean;
let preflightCompactionApplied: boolean | undefined;
const resetSession = async (options: SessionResetOptions): Promise<boolean> => {
const reset = await performSessionReset(options);
activeSessionEntry = getActiveSessionEntry();
activeIsNewSession = getActiveIsNewSession();
return reset;
};
const admitUserTurn = async (
...args: Parameters<typeof admitUserTurnWithRecovery>
): ReturnType<typeof admitUserTurnWithRecovery> => {
const result = await admitUserTurnWithRecovery(...args);
activeSessionEntry = getActiveSessionEntry();
return result;
};
const beginBeforeAgentReply = async (
...args: Parameters<typeof beginBeforeAgentReplyWithRecovery>
): ReturnType<typeof beginBeforeAgentReplyWithRecovery> => {
const result = await beginBeforeAgentReplyWithRecovery(...args);
activeSessionEntry = getActiveSessionEntry();
return result;
};
const checkpointBeforeAgentReply = async (
...args: Parameters<typeof checkpointBeforeAgentReplyWithRecovery>
): ReturnType<typeof checkpointBeforeAgentReplyWithRecovery> => {
const result = await checkpointBeforeAgentReplyWithRecovery(...args);
activeSessionEntry = getActiveSessionEntry();
return result;
};
await typingSignals.signalRunStart();
// Preserve the one-flush-per-compaction-cycle gate: an earlier same-cycle
// flush is the checkpoint for this upcoming compaction, not a reason to rerun maintenance.
const memoryFlushResult = await traceAgentPhase("reply.memory_flush", () =>
runMemoryFlushIfNeeded({
cfg,
followupRun,
promptForEstimate: followupRun.prompt,
sessionCtx,
opts,
defaultModel,
agentCfgContextTokens,
resolvedVerboseLevel,
sessionEntry: activeSessionEntry,
sessionStore: activeSessionStore,
sessionKey,
runtimePolicySessionKey,
storePath,
isHeartbeat,
replyOperation,
onVisibleErrorPayloads: (payloads) => {
logVerbose(
`memory flush produced ${payloads.length} visible maintenance error payload(s); continuing user reply`,
);
},
}),
);
activeSessionEntry = memoryFlushResult.sessionEntry;
setActiveSessionEntry(activeSessionEntry);
if (replyOperation.result?.kind === "aborted") {
throw replyOperation.abortSignal.reason ?? new Error("reply operation aborted");
}
const prePreflightCompactionCount = activeSessionEntry?.compactionCount ?? 0;
try {
activeSessionEntry = await traceAgentPhase("reply.preflight_compaction", () =>
runPreflightCompactionIfNeeded({
cfg,
followupRun,
promptForEstimate: followupRun.prompt,
defaultModel,
agentCfgContextTokens,
sessionEntry: activeSessionEntry,
sessionStore: activeSessionStore,
sessionKey,
runtimePolicySessionKey,
storePath,
isHeartbeat,
replyOperation,
onCompactionNotice: sendDirectCompactionNotice,
}),
);
setActiveSessionEntry(activeSessionEntry);
preflightCompactionApplied =
(activeSessionEntry?.compactionCount ?? 0) > prePreflightCompactionCount;
} catch (err) {
const canRotateAfterPreflightFailure =
memoryFlushResult.outcome === "exhausted" &&
!replyOperation.abortSignal.aborted &&
isLikelyContextOverflowError(String(err));
if (!canRotateAfterPreflightFailure) {
throw err;
}
logVerbose(`Preflight compaction could not recover exhausted memory flush: ${String(err)}`);
}
if (memoryFlushResult.outcome === "exhausted" && !preflightCompactionApplied) {
await resetSession({
failureLabel: "memory flush exhaustion",
buildLogMessage: (nextSessionId) =>
`Memory flush exhausted. Rotating bloated session ${sessionKey} -> ${nextSessionId}.`,
// Rotate only when compaction could not recover the bloated context.
cleanupTranscripts: false,
});
if (activeSessionEntry?.sessionId) {
replyOperation.updateSessionId(activeSessionEntry.sessionId);
}
}
// Exhausted background maintenance is non-terminal: optionally notify, then reply normally.
if (memoryFlushResult.outcome === "exhausted") {
await sendDirectCompactionNotice?.("memory_flush_degraded");
}
const runFollowupTurn = createFollowupRunner({
opts,
typing,
typingMode,
sessionEntry: activeSessionEntry,
sessionStore: activeSessionStore,
sessionKey,
storePath,
defaultModel,
agentCfgContextTokens,
toolProgressDetail,
});
setRunFollowupTurn(runFollowupTurn);
replyOperation.setPhase("running");
const runStartedAt = Date.now();
const userTurnAdmission = await admitUserTurn(followupRun.userTurnTranscriptRecorder);
if (userTurnAdmission === "duplicate-source") {
return returnWithQueuedFollowupDrain(undefined);
}
// Adoption marks run start and must never be spool-replayed (would re-run tools).
// Suppressed delivery persists only the user transcript; crashed suppressed runs die
// silently. Deliverable turns atomically persist transcript plus recovery ownership.
await turnAdoptionLifecycle?.onAdopted();
const runOutcome = await withBeforeAgentReplyObserver(
{
beforeDispatch: async () => {
const shouldDispatch = await beginBeforeAgentReply();
if (!shouldDispatch || !beforeAgentReplyDispatchedForSteer) {
return shouldDispatch;
}
// The same source fell through from steering. Advance recovery while
// preserving the hook decision made before the attempted injection.
await checkpointBeforeAgentReply({ state: "continue" });
return false;
},
afterDispatch: async (hookResult) => {
if (!hookResult?.handled) {
await checkpointBeforeAgentReply({ state: "continue" });
return hookResult;
}
const hookReply = hookResult.reply ?? { text: SILENT_REPLY_TOKEN };
const hookFinalDeliveryText = buildRecoverablePendingFinalDeliveryText([hookReply]);
const normalizedHookReplies = normalizePendingFinalDeliveryPayloads([hookReply]);
let hookCheckpoint: Parameters<typeof checkpointBeforeAgentReply>[0] = {
state: normalizedHookReplies.length === 0 ? "handled-silent" : "handled-unrecoverable",
};
if (sessionKey && storePath && normalizedHookReplies.length > 0) {
const sourceReplyPolicy = resolveSourceReplyPolicy({
cfg,
sessionCtx,
sessionEntry: activeSessionEntry,
sessionKey,
runtimePolicySessionKey,
opts,
});
if (!sourceReplyPolicy.suppressDelivery) {
const pendingFinalDeliveryIntentId = crypto.randomUUID();
setReplyPayloadMetadata(hookReply, {
pendingFinalDeliveryIntentId,
pendingFinalDeliveryRetryText: hookFinalDeliveryText,
});
hookCheckpoint = {
state: hookFinalDeliveryText ? "handled-reply" : "handled-unrecoverable",
pendingFinalDelivery: {
text: hookFinalDeliveryText ?? "",
intentId: pendingFinalDeliveryIntentId,
context: resolveReplyRunDeliveryContext({
cfg,
sessionCtx,
sessionEntry: activeSessionEntry,
sessionKey,
runtimePolicySessionKey,
opts,
}),
},
};
} else {
// dispatch-from-config owns source visibility for every returned payload.
// This checkpoint records that recovery owes no delivery; the outer gate drops the reply.
hookCheckpoint = { state: "handled-silent" };
}
}
await checkpointBeforeAgentReply(hookCheckpoint);
return { ...hookResult, reply: hookReply };
},
},
() =>
traceAgentPhase("reply.run_agent_turn", () =>
runAgentTurnWithFallback({
commandBody,
transcriptCommandBody,
followupRun,
sessionCtx,
replyThreading: replyThreadingOverride ?? sessionCtx.ReplyThreading,
replyOperation,
opts,
typingSignals,
blockReplyPipeline,
blockStreamingEnabled,
blockReplyChunking,
resolvedBlockStreamingBreak,
applyReplyToMode,
shouldEmitToolResult,
shouldEmitToolOutput,
pendingToolTasks,
resetSessionAfterRoleOrderingConflict,
isHeartbeat,
sessionKey,
runtimePolicySessionKey,
getActiveSessionEntry,
activeSessionStore,
storePath,
resolvedVerboseLevel,
toolProgressDetail,
replyMediaContext,
isRestartRecoveryArmed,
}),
),
);
activeSessionEntry = getActiveSessionEntry();
activeIsNewSession = getActiveIsNewSession();
if (runOutcome.kind === "final") {
if (!replyOperation.result) {
replyOperation.fail("run_failed", new Error("reply operation exited with final payload"));
}
return returnWithQueuedFollowupDrain(runOutcome.payload);
}
return await finalizeReplyAgentRun({
activeIsNewSession,
activeSessionEntry,
activeSessionStore,
agentCfgContextTokens,
blockReplyPipeline,
blockStreamingEnabled,
cfg,
commandBody,
defaultModel,
followupRun,
isHeartbeat,
opts,
pendingToolTasks,
preflightCompactionApplied,
queueKey,
replyMediaContext,
replyOperation,
replyRouteThreadId,
replyThreadingOverride,
replyToChannel,
replyToMode,
resolvedBlockStreamingBreak,
resolvedQueue,
resolvedVerboseLevel,
returnWithQueuedFollowupDrain,
runFollowupTurn,
runOutcome,
runStartedAt,
runtimePolicySessionKey,
sessionCtx,
sessionKey,
shouldInjectGroupIntro,
storePath,
typingSignals,
});
}
export function createReplyAgentRestartRecoveryController(
context: Pick<
RunReplyAgentParams,
"followupRun" | "opts" | "runtimePolicySessionKey" | "sessionCtx" | "sessionKey" | "storePath"
> & {
activeSessionStore: Record<string, SessionEntry> | undefined;
cfg: OpenClawConfig;
getActiveSessionEntry: () => SessionEntry | undefined;
replyOperation: ReplyOperation;
restartRecoverySourceTurnId: string | undefined;
setActiveSessionEntry: (entry: SessionEntry) => void;
},
) {
const {
activeSessionStore,
cfg,
followupRun,
getActiveSessionEntry,
opts,
replyOperation,
restartRecoverySourceTurnId,
runtimePolicySessionKey,
sessionCtx,
sessionKey,
setActiveSessionEntry,
storePath,
} = context;
const restartRecoverySameChannelThreadRequired = restartRecoverySourceTurnId
? buildThreadingToolContext({
sessionCtx,
config: cfg,
hasRepliedRef: undefined,
}).sameChannelThreadRequired
: undefined;
const {
admitUserTurn,
beginBeforeAgentReply,
checkpointBeforeAgentReply,
clear: clearRestartRecoveryDeliveryClaim,
isArmed: isRestartRecoveryArmed,
} = createReplyRestartRecoveryClaimController({
admissionRunId:
normalizeOptionalString(sessionCtx.MessageSid) ??
normalizeOptionalString(sessionCtx.MessageSidFull),
getEntry: () =>
sessionKey
? (activeSessionStore?.[sessionKey] ?? getActiveSessionEntry())
: getActiveSessionEntry(),
getSessionId: () => replyOperation.sessionId,
beforeAgentReplyState: "admitted",
isRestartAbort: () =>
replyOperation.result?.kind === "aborted" &&
replyOperation.result.code === "aborted_for_restart",
resolveDeliveryContext: (entry) =>
sessionKey
? resolveReplyRunDeliveryContext({
cfg,
sessionCtx,
sessionEntry: entry,
sessionKey,
runtimePolicySessionKey,
opts,
})
: undefined,
requesterAccountId:
followupRun.originatingAccountId ?? sessionCtx.AccountId ?? followupRun.run.agentAccountId,
requesterSenderId: sessionCtx.SenderId,
resolveUserTurnTarget: ({
entry,
sessionId,
sessionKey: targetSessionKey,
storePath: targetStorePath,
}) => ({
sessionId,
sessionKey: targetSessionKey,
sessionEntry: entry,
...(activeSessionStore ? { sessionStore: activeSessionStore } : {}),
storePath: targetStorePath,
agentId: followupRun.run.agentId,
cwd: followupRun.run.workspaceDir,
config: cfg,
}),
...(sessionKey ? { sessionKey } : {}),
setEntry: (entry) => {
setActiveSessionEntry(entry);
if (activeSessionStore && sessionKey) {
activeSessionStore[sessionKey] = entry;
}
},
sameChannelThreadRequired: restartRecoverySameChannelThreadRequired,
sourceTurnId: restartRecoverySourceTurnId,
sourceReplyDeliveryMode: sessionKey
? resolveSourceReplyPolicy({
cfg,
sessionCtx,
sessionEntry: getActiveSessionEntry(),
sessionKey,
runtimePolicySessionKey,
opts,
}).sourceReplyDeliveryMode
: opts?.sourceReplyDeliveryMode,
...(storePath ? { storePath } : {}),
});
return {
admitUserTurn,
beginBeforeAgentReply,
checkpointBeforeAgentReply,
clear: clearRestartRecoveryDeliveryClaim,
isArmed: isRestartRecoveryArmed,
};
}
@@ -0,0 +1,281 @@
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { resolveContextTokensForModel } from "../../agents/context.js";
import { DEFAULT_CONTEXT_TOKENS } from "../../agents/defaults.js";
import { resolveFastModeState } from "../../agents/fast-mode.js";
import { consolidateLiveModelSwitchAfterRun } from "../../agents/live-model-switch.js";
import { isCliProvider } from "../../agents/model-selection.js";
import { updateSessionEntry } from "../../config/sessions/session-accessor.js";
import { logVerbose } from "../../globals.js";
import { shouldPreserveUserFacingSessionStateForInputProvenance } from "../../sessions/input-provenance.js";
import { resolveFallbackTransition } from "../fallback-state.js";
import { resolveConfiguredFallbackModel } from "./agent-runner-core.js";
import type { FinalizeReplyAgentRunInput } from "./agent-runner-result.types.js";
import { drainPendingToolTasks } from "./pending-tool-task-drain.js";
import { buildReplyUsageState, recordReplyUsageState } from "./reply-usage-state.js";
import { persistRunSessionUsage } from "./session-run-accounting.js";
export async function accountReplyAgentRun(context: FinalizeReplyAgentRunInput) {
const {
activeSessionStore,
agentCfgContextTokens,
blockReplyPipeline,
cfg,
defaultModel,
followupRun,
isHeartbeat,
pendingToolTasks,
preflightCompactionApplied,
resolvedVerboseLevel,
runOutcome,
runStartedAt,
sessionKey,
sessionCtx,
shouldInjectGroupIntro,
storePath,
} = context;
let { activeSessionEntry } = context;
const {
runId,
runResult,
fallbackProvider,
fallbackModel,
fallbackExhausted,
fallbackAttempts,
directlySentBlockKeys,
directlySentBlockPayloads,
terminalFailurePayload,
} = runOutcome;
const { autoCompactionCount } = runOutcome;
const { didLogHeartbeatStrip } = runOutcome;
if (
shouldInjectGroupIntro &&
activeSessionEntry &&
activeSessionStore &&
sessionKey &&
activeSessionEntry.groupActivationNeedsSystemIntro
) {
const updatedAt = Date.now();
activeSessionEntry.groupActivationNeedsSystemIntro = false;
activeSessionEntry.updatedAt = updatedAt;
activeSessionStore[sessionKey] = activeSessionEntry;
if (storePath) {
await updateSessionEntry(
{ storePath, sessionKey },
() => ({
groupActivationNeedsSystemIntro: false,
updatedAt,
}),
{
skipMaintenance: true,
takeCacheOwnership: true,
},
);
}
}
const payloadArray = runResult.payloads ?? [];
if (blockReplyPipeline) {
await blockReplyPipeline.flush({ force: true });
blockReplyPipeline.stop();
}
if (pendingToolTasks.size > 0) {
await drainPendingToolTasks({
tasks: pendingToolTasks,
onTimeout: logVerbose,
});
}
const usage = runResult.meta?.agentMeta?.usage;
const hasBillableUsageBuckets =
usage &&
(usage.input !== undefined ||
usage.output !== undefined ||
usage.cacheRead !== undefined ||
usage.cacheWrite !== undefined);
const promptTokens = runResult.meta?.agentMeta?.promptTokens;
const modelUsed = runResult.meta?.agentMeta?.model ?? fallbackModel ?? defaultModel;
const providerUsed =
runResult.meta?.agentMeta?.provider ?? fallbackProvider ?? followupRun.run.provider;
const winnerProvider = fallbackExhausted
? undefined
: (runResult.meta?.executionTrace?.winnerProvider ?? providerUsed);
const winnerModel = fallbackExhausted
? undefined
: (runResult.meta?.executionTrace?.winnerModel ?? modelUsed);
const ctxTokens = runResult.meta?.agentMeta?.contextTokens;
const compactions = runResult.meta?.agentMeta?.compactionCount;
const lastCallUsage = runResult.meta?.agentMeta?.lastCallUsage;
const replyUsageState = buildReplyUsageState({
config: cfg,
provider: providerUsed,
model: modelUsed,
fallbackExhausted,
winnerProvider,
winnerModel,
reasoningEffort:
typeof followupRun.run.thinkLevel === "string" ? followupRun.run.thinkLevel : undefined,
fastMode: resolveFastModeState({
cfg,
provider: providerUsed ?? "",
model: modelUsed ?? "",
agentId: followupRun.run.agentId,
sessionEntry: activeSessionEntry,
}).enabled,
fallbackUsed: runResult.meta?.executionTrace?.fallbackUsed === true,
agentId: followupRun.run.agentId,
sessionId: followupRun.run.sessionId,
chatType: typeof sessionCtx.ChatType === "string" ? sessionCtx.ChatType : undefined,
authMode: runResult.meta?.requestShaping?.authMode ?? undefined,
overrideSource: activeSessionEntry?.modelOverrideSource ?? undefined,
requestedProvider: followupRun.run.provider,
requestedModel: followupRun.run.model,
durationMs: Date.now() - runStartedAt,
compactionCount: typeof compactions === "number" ? compactions : undefined,
contextTokenBudget:
typeof ctxTokens === "number" && Number.isFinite(ctxTokens) ? ctxTokens : undefined,
contextUsedTokens:
typeof promptTokens === "number" && Number.isFinite(promptTokens) ? promptTokens : undefined,
promptTokens,
usage,
lastCallUsage,
});
recordReplyUsageState(runId, replyUsageState);
const verboseEnabled = resolvedVerboseLevel !== "off";
const preserveUserFacingSessionState = shouldPreserveUserFacingSessionStateForInputProvenance(
followupRun.run.inputProvenance,
);
const fallbackStateEntry =
activeSessionEntry ?? (sessionKey ? activeSessionStore?.[sessionKey] : undefined);
const configuredFallbackModel = resolveConfiguredFallbackModel({
run: followupRun.run,
fallbackStateEntry,
});
const selectedProvider = configuredFallbackModel.provider;
const selectedModel = configuredFallbackModel.model;
const fallbackTransition = resolveFallbackTransition({
selectedProvider,
selectedModel,
activeProvider: providerUsed,
activeModel: modelUsed,
attempts: fallbackAttempts,
state: fallbackStateEntry,
cfg,
});
if (fallbackTransition.stateChanged && !fallbackExhausted && !preserveUserFacingSessionState) {
if (fallbackStateEntry) {
fallbackStateEntry.fallbackNoticeSelectedModel = fallbackTransition.nextState.selectedModel;
fallbackStateEntry.fallbackNoticeActiveModel = fallbackTransition.nextState.activeModel;
fallbackStateEntry.fallbackNoticeReason = fallbackTransition.nextState.reason;
fallbackStateEntry.updatedAt = Date.now();
activeSessionEntry = fallbackStateEntry;
}
if (sessionKey && fallbackStateEntry && activeSessionStore) {
activeSessionStore[sessionKey] = fallbackStateEntry;
}
if (sessionKey && storePath) {
await updateSessionEntry(
{ storePath, sessionKey },
() => ({
fallbackNoticeSelectedModel: fallbackTransition.nextState.selectedModel,
fallbackNoticeActiveModel: fallbackTransition.nextState.activeModel,
fallbackNoticeReason: fallbackTransition.nextState.reason,
}),
{
skipMaintenance: true,
takeCacheOwnership: true,
},
);
}
}
const usedCliProvider = isCliProvider(providerUsed, cfg);
const cliSessionId = usedCliProvider
? normalizeOptionalString(runResult.meta?.agentMeta?.sessionId)
: undefined;
const cliSessionBinding = usedCliProvider
? runResult.meta?.agentMeta?.cliSessionBinding
: undefined;
const clearCliSessionBinding =
usedCliProvider && runResult.meta?.agentMeta?.clearCliSessionBinding === true;
const runtimeContextTokens =
typeof runResult.meta?.agentMeta?.contextTokens === "number" &&
Number.isFinite(runResult.meta.agentMeta.contextTokens) &&
runResult.meta.agentMeta.contextTokens > 0
? Math.floor(runResult.meta.agentMeta.contextTokens)
: undefined;
const contextTokensUsed =
runtimeContextTokens ??
resolveContextTokensForModel({
cfg,
provider: providerUsed,
model: modelUsed,
contextTokensOverride: agentCfgContextTokens,
fallbackContextTokens: activeSessionEntry?.contextTokens ?? DEFAULT_CONTEXT_TOKENS,
allowAsyncLoad: false,
}) ??
DEFAULT_CONTEXT_TOKENS;
await persistRunSessionUsage({
storePath,
sessionKey,
cfg,
usage,
lastCallUsage: runResult.meta?.agentMeta?.lastCallUsage,
compactionTokensAfter: runResult.meta?.agentMeta?.compactionTokensAfter,
promptTokens,
usageIsContextSnapshot: usedCliProvider ? true : undefined,
isHeartbeat,
preserveRuntimeModel: fallbackExhausted,
preserveUserFacingSessionModelState: preserveUserFacingSessionState,
modelUsed,
providerUsed,
contextTokensUsed,
systemPromptReport: runResult.meta?.systemPromptReport,
cliSessionId,
cliSessionBinding,
clearCliSessionBinding,
preserveFreshTotalTokensOnStaleUsage: preflightCompactionApplied,
});
if (!isHeartbeat && !preserveUserFacingSessionState && !fallbackExhausted) {
// A completed run that executed the persisted selection consumes the
// pending live-switch flag; CLI harness runs never hit the embedded
// attempt-recovery clear, so /status would report the switch forever.
await consolidateLiveModelSwitchAfterRun({
cfg,
sessionKey,
agentId: followupRun.run.agentId,
providerUsed,
modelUsed,
});
}
return {
activeSessionEntry,
autoCompactionCount,
configuredFallbackModel,
contextTokensUsed,
didLogHeartbeatStrip,
directlySentBlockKeys,
directlySentBlockPayloads,
fallbackAttempts,
fallbackExhausted,
fallbackTransition,
hasBillableUsageBuckets,
modelUsed,
payloadArray,
preserveUserFacingSessionState,
promptTokens,
providerUsed,
replyUsageState,
runId,
runResult,
selectedModel,
selectedProvider,
terminalFailurePayload,
usage,
verboseEnabled,
};
}
@@ -0,0 +1,423 @@
import crypto from "node:crypto";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { hasConfiguredModelFallbacks, resolveSessionAgentId } from "../../agents/agent-scope.js";
import { resolveModelAuthMode } from "../../agents/model-auth.js";
import { isCliProvider } from "../../agents/model-selection.js";
import type { SessionEntry } from "../../config/sessions.js";
import { updateSessionEntry } from "../../config/sessions/session-accessor.js";
import { enqueueSystemEvent } from "../../infra/system-events.js";
import { DEFAULT_HEARTBEAT_ACK_MAX_CHARS, stripHeartbeatToken } from "../heartbeat.js";
import { setReplyPayloadMetadata } from "../reply-payload.js";
import type { ReplyPayload } from "../types.js";
import {
buildInlinePluginStatusPayload,
markBeforeAgentRunBlockedPayloads,
resolvePendingFinalDeliveryRetryText,
resolveReplyRunDeliveryContext,
resolveSourceReplyPolicy,
} from "./agent-runner-core.js";
import { normalizeAssistantFinalDeliveryText } from "./agent-runner-core.js";
import type { accountReplyAgentRun } from "./agent-runner-result-accounting.js";
import type { FinalizeReplyAgentRunInput } from "./agent-runner-result.types.js";
import {
accumulateSessionUsageFromTranscript,
buildInlineRawTracePayload,
derivePromptSegments,
} from "./agent-runner-trace.js";
import {
type TraceCompletionView,
type TraceContextManagementView,
type TraceExecutionView,
type TracePromptSegmentView,
type TraceToolSummaryView,
mergeExecutionTrace,
} from "./agent-runner-trace.js";
import { appendUsageLine } from "./agent-runner-usage-line.js";
import { buildPendingFinalDeliveryText } from "./pending-final-delivery.js";
import { readPostCompactionContext } from "./post-compaction-context.js";
import {
shouldWarnAboutPrivateMessageToolFinal,
warnPrivateMessageToolFinal,
} from "./private-message-tool-final.js";
import { enqueueFollowupRun, refreshQueuedFollowupSession } from "./queue.js";
import { incrementRunCompactionCount } from "./session-run-accounting.js";
import {
buildStrandedReplyDeliveryFailurePayload,
buildStrandedReplyRetryFollowupRun,
} from "./stranded-reply-recovery.js";
type ReplyAgentAccounting = Awaited<ReturnType<typeof accountReplyAgentRun>>;
type PreparedReplyAgentPayloads = {
kind: "continue";
activeSessionEntry: SessionEntry | undefined;
completedSourceReplyDelivery: boolean;
guardedReplyPayloads: ReplyPayload[];
responseUsageLine: string | undefined;
};
export async function completeReplyAgentRun(input: {
context: FinalizeReplyAgentRunInput;
accounting: ReplyAgentAccounting;
prepared: PreparedReplyAgentPayloads;
}) {
const { context, accounting, prepared } = input;
const {
activeIsNewSession,
activeSessionStore,
cfg,
followupRun,
isHeartbeat,
opts,
preflightCompactionApplied,
queueKey,
resolvedBlockStreamingBreak,
resolvedQueue,
resolvedVerboseLevel,
returnWithQueuedFollowupDrain,
runFollowupTurn,
runtimePolicySessionKey,
sessionCtx,
sessionKey,
storePath,
} = context;
const {
autoCompactionCount,
contextTokensUsed,
fallbackAttempts,
fallbackExhausted,
modelUsed,
promptTokens,
providerUsed,
runResult,
verboseEnabled,
} = accounting;
const { completedSourceReplyDelivery, guardedReplyPayloads, responseUsageLine } = prepared;
let { activeSessionEntry } = prepared;
// Prepend verbose operational notices. Model fallback notices are prepared
// earlier so they pass through normal reply threading and stream-dedupe.
let finalPayloads = guardedReplyPayloads;
const prefixNotices: ReplyPayload[] = [];
if (verboseEnabled && activeIsNewSession) {
prefixNotices.push({ text: `🧭 New session: ${followupRun.run.sessionId}` });
}
if (autoCompactionCount > 0) {
const previousSessionId = activeSessionEntry?.sessionId ?? followupRun.run.sessionId;
const count = await incrementRunCompactionCount({
cfg,
sessionEntry: activeSessionEntry,
sessionStore: activeSessionStore,
sessionKey,
storePath,
amount: autoCompactionCount,
compactionTokensAfter: runResult.meta?.agentMeta?.compactionTokensAfter,
lastCallUsage: runResult.meta?.agentMeta?.lastCallUsage,
contextTokensUsed,
newSessionId: runResult.meta?.agentMeta?.sessionId,
newSessionFile: runResult.meta?.agentMeta?.sessionFile,
});
const refreshedSessionEntry =
sessionKey && activeSessionStore ? activeSessionStore[sessionKey] : undefined;
if (refreshedSessionEntry) {
activeSessionEntry = refreshedSessionEntry;
refreshQueuedFollowupSession({
key: queueKey,
previousSessionId,
nextSessionId: refreshedSessionEntry.sessionId,
nextSessionFile: refreshedSessionEntry.sessionFile,
});
}
// Inject post-compaction workspace context for the next agent turn
if (sessionKey) {
readPostCompactionContext(followupRun.run.workspaceDir, {
cfg,
agentId: resolveSessionAgentId({ sessionKey, config: cfg }),
})
.then((contextContent) => {
if (contextContent) {
enqueueSystemEvent(contextContent, { sessionKey });
}
})
.catch(() => {
// Silent failure — post-compaction context is best-effort
});
}
if (verboseEnabled) {
const suffix = typeof count === "number" ? ` (count ${count})` : "";
prefixNotices.push({ text: `🧹 Auto-compaction complete${suffix}.` });
}
}
const prefixPayloads = [...prefixNotices];
const isHookBlockedRun = runResult.meta?.error?.kind === "hook_block";
const rawUserText = isHookBlockedRun
? runResult.meta?.finalPromptText
: (runResult.meta?.finalPromptText ??
sessionCtx.CommandBody ??
sessionCtx.RawBody ??
sessionCtx.BodyForAgent ??
sessionCtx.Body);
const rawAssistantText = isHookBlockedRun
? undefined
: (runResult.meta?.finalAssistantRawText ?? runResult.meta?.finalAssistantVisibleText);
const traceAuthorized = followupRun.run.traceAuthorized === true;
const executionTrace = mergeExecutionTrace({
fallbackAttempts,
executionTrace: runResult.meta?.executionTrace as TraceExecutionView | undefined,
provider: providerUsed,
model: modelUsed,
runner: isCliProvider(providerUsed, cfg) ? "cli" : "embedded",
exhausted: fallbackExhausted,
});
const requestShaping = {
authMode:
runResult.meta?.requestShaping?.authMode ??
(cfg?.models?.providers && providerUsed in cfg.models.providers
? (resolveModelAuthMode(providerUsed, cfg, undefined, {
workspaceDir: followupRun.run.workspaceDir,
}) ?? undefined)
: undefined),
thinking:
runResult.meta?.requestShaping?.thinking ??
normalizeOptionalString(followupRun.run.thinkLevel),
reasoning:
runResult.meta?.requestShaping?.reasoning ??
normalizeOptionalString(followupRun.run.reasoningLevel),
verbose:
runResult.meta?.requestShaping?.verbose ?? normalizeOptionalString(resolvedVerboseLevel),
trace:
runResult.meta?.requestShaping?.trace ??
normalizeOptionalString(activeSessionEntry?.traceLevel),
fallbackEligible:
runResult.meta?.requestShaping?.fallbackEligible ??
hasConfiguredModelFallbacks({
cfg,
agentId: followupRun.run.agentId,
sessionKey: followupRun.run.sessionKey,
}),
blockStreaming:
runResult.meta?.requestShaping?.blockStreaming ??
normalizeOptionalString(resolvedBlockStreamingBreak),
};
const promptSegments =
(runResult.meta?.promptSegments as TracePromptSegmentView[] | undefined) ??
derivePromptSegments(rawUserText);
const toolSummary = runResult.meta?.toolSummary as TraceToolSummaryView | undefined;
const completion =
(runResult.meta?.completion as TraceCompletionView | undefined) ??
(runResult.meta?.stopReason
? {
stopReason: runResult.meta.stopReason,
finishReason: runResult.meta.stopReason,
...(runResult.meta.stopReason.toLowerCase().includes("refusal") ? { refusal: true } : {}),
}
: undefined);
const contextManagement = {
...(typeof activeSessionEntry?.compactionCount === "number"
? { sessionCompactions: activeSessionEntry.compactionCount }
: {}),
...(typeof runResult.meta?.contextManagement?.lastTurnCompactions === "number"
? { lastTurnCompactions: runResult.meta.contextManagement.lastTurnCompactions }
: typeof runResult.meta?.agentMeta?.compactionCount === "number"
? { lastTurnCompactions: runResult.meta.agentMeta.compactionCount }
: {}),
...(runResult.meta?.contextManagement &&
typeof runResult.meta.contextManagement.preflightCompactionApplied === "boolean"
? {
preflightCompactionApplied: runResult.meta.contextManagement.preflightCompactionApplied,
}
: preflightCompactionApplied
? { preflightCompactionApplied }
: {}),
...(runResult.meta?.contextManagement &&
typeof runResult.meta.contextManagement.postCompactionContextInjected === "boolean"
? {
postCompactionContextInjected:
runResult.meta.contextManagement.postCompactionContextInjected,
}
: {}),
} satisfies TraceContextManagementView;
const sessionUsage =
traceAuthorized && activeSessionEntry?.traceLevel === "raw"
? await accumulateSessionUsageFromTranscript({
sessionId: runResult.meta?.agentMeta?.sessionId ?? followupRun.run.sessionId,
storePath,
sessionFile: followupRun.run.sessionFile,
})
: undefined;
const traceEnabledForSender =
traceAuthorized &&
(activeSessionEntry?.traceLevel === "on" || activeSessionEntry?.traceLevel === "raw");
const shouldAppendTracePayload = verboseEnabled || traceEnabledForSender;
let trailingPluginStatusPayload: ReplyPayload | undefined;
if (shouldAppendTracePayload) {
const pluginStatusPayload = buildInlinePluginStatusPayload({
entry: activeSessionEntry,
includeTraceLines: traceEnabledForSender,
});
const rawTracePayload =
traceAuthorized && activeSessionEntry?.traceLevel === "raw"
? buildInlineRawTracePayload({
entry: activeSessionEntry,
rawUserText,
rawAssistantText,
sessionUsage,
usage: runResult.meta?.agentMeta?.usage,
lastCallUsage: runResult.meta?.agentMeta?.lastCallUsage,
provider: providerUsed,
model: modelUsed,
contextLimit: contextTokensUsed,
promptTokens,
executionTrace,
requestShaping,
promptSegments,
toolSummary,
completion,
contextManagement,
})
: undefined;
trailingPluginStatusPayload =
pluginStatusPayload && rawTracePayload
? { text: `${pluginStatusPayload.text}\n\n${rawTracePayload.text}` }
: (pluginStatusPayload ?? rawTracePayload);
}
if (prefixPayloads.length > 0) {
finalPayloads = [...prefixPayloads, ...finalPayloads];
}
if (trailingPluginStatusPayload) {
finalPayloads = [...finalPayloads, trailingPluginStatusPayload];
}
if (responseUsageLine) {
finalPayloads = appendUsageLine(finalPayloads, responseUsageLine);
}
if (isHookBlockedRun) {
finalPayloads = markBeforeAgentRunBlockedPayloads(finalPayloads);
}
// Capture only policy-visible final payloads in session store to support
// durable delivery retries. Hidden reasoning, message-tool-only replies,
// and sendPolicy-denied replies must not become heartbeat-replayable text.
const isStrandedReplyRetryRun = followupRun.strandedReplyRetry === true;
if (sessionKey && storePath && (finalPayloads.length > 0 || isStrandedReplyRetryRun)) {
const sourceReplyPolicy = resolveSourceReplyPolicy({
cfg,
sessionCtx,
sessionEntry: activeSessionEntry,
sessionKey,
runtimePolicySessionKey,
opts,
});
const finalDeliveryText = buildPendingFinalDeliveryText(finalPayloads);
// #85714: warn only for unusually substantive private final text. In
// message_tool_only, no tool call can be intentional silence, and
// finalDeliveryText also includes verbose/status/usage metadata.
const assistantFinalText = normalizeAssistantFinalDeliveryText(
typeof runResult.meta?.finalAssistantVisibleText === "string"
? runResult.meta.finalAssistantVisibleText
: (rawAssistantText ?? ""),
);
const isRoomEvent = sessionCtx.InboundEventKind === "room_event";
// Heartbeats already deliver fallback finals via sendDurableMessageBatch;
// recovering here would duplicate that message.
const isStrandedReply =
!isHeartbeat &&
!isRoomEvent &&
shouldWarnAboutPrivateMessageToolFinal({
sourceReplyDeliveryMode: sourceReplyPolicy.sourceReplyDeliveryMode,
sendPolicyDenied: sourceReplyPolicy.sendPolicyDenied,
successfulSourceReplyDelivery: completedSourceReplyDelivery,
finalText: assistantFinalText,
});
const retryMissingSourceDelivery =
isStrandedReplyRetryRun &&
!isHeartbeat &&
!isRoomEvent &&
sourceReplyPolicy.sourceReplyDeliveryMode === "message_tool_only" &&
!sourceReplyPolicy.sendPolicyDenied &&
!completedSourceReplyDelivery;
if (isStrandedReply) {
warnPrivateMessageToolFinal({
sessionKey,
channel:
sessionCtx.OriginatingChannel ??
sessionCtx.Surface ??
sessionCtx.Provider ??
activeSessionEntry?.channel,
finalTextLength: assistantFinalText.trim().length,
});
}
if (isStrandedReply || retryMissingSourceDelivery) {
if (isStrandedReplyRetryRun) {
finalPayloads = [...finalPayloads, buildStrandedReplyDeliveryFailurePayload()];
} else {
const retryEnqueued = enqueueFollowupRun(
queueKey,
buildStrandedReplyRetryFollowupRun(followupRun, {
finalText: assistantFinalText,
sourceReplyDeliveryMode: sourceReplyPolicy.sourceReplyDeliveryMode,
}),
resolvedQueue,
"none",
runFollowupTurn,
false,
{ position: "front" },
);
if (!retryEnqueued) {
finalPayloads = [...finalPayloads, buildStrandedReplyDeliveryFailurePayload()];
}
}
}
const pendingText = sourceReplyPolicy.suppressDelivery ? "" : finalDeliveryText;
const heartbeatAckMaxChars = DEFAULT_HEARTBEAT_ACK_MAX_CHARS;
const resolvedPendingText = isHeartbeat
? (() => {
const stripped = stripHeartbeatToken(pendingText, {
mode: "heartbeat",
maxAckChars: heartbeatAckMaxChars,
});
return stripped.shouldSkip ? "" : stripped.text || pendingText;
})()
: pendingText;
if (resolvedPendingText) {
const pendingFinalDeliveryIntentId = crypto.randomUUID();
for (const payload of finalPayloads) {
setReplyPayloadMetadata(payload, {
pendingFinalDeliveryIntentId,
pendingFinalDeliveryRetryText: resolvePendingFinalDeliveryRetryText({
isHeartbeat,
payload,
}),
});
}
const pendingFinalDeliveryContext = resolveReplyRunDeliveryContext({
cfg,
sessionCtx,
sessionEntry: activeSessionEntry,
sessionKey,
runtimePolicySessionKey,
opts,
});
await updateSessionEntry(
{ storePath, sessionKey },
() => ({
pendingFinalDelivery: true,
pendingFinalDeliveryText: resolvedPendingText,
pendingFinalDeliveryIntentId,
pendingFinalDeliveryContext,
pendingFinalDeliveryCreatedAt: Date.now(),
updatedAt: Date.now(),
}),
{
skipMaintenance: true,
takeCacheOwnership: true,
},
);
}
}
const result = returnWithQueuedFollowupDrain(
finalPayloads.length === 1 ? finalPayloads[0] : finalPayloads,
);
return result;
}
@@ -0,0 +1,537 @@
import {
hasCommittedSourceReplyDeliveryEvidence,
hasCompletedSourceReplyDeliveryEvidence,
hasCompletedTerminalDeliveryEvidence,
hasVisibleOutboundDeliveryEvidence,
} from "../../agents/embedded-agent-runner/delivery-evidence.js";
import { hasDeliberateSilentTerminalReply } from "../../agents/embedded-agent-runner/result-fallback-classifier.js";
import { deriveContextPromptTokens, hasNonzeroUsage } from "../../agents/usage.js";
import { emitAgentEvent } from "../../infra/agent-events.js";
import { emitTrustedDiagnosticEvent, isDiagnosticsEnabled } from "../../infra/diagnostic-events.js";
import {
createChildDiagnosticTraceContext,
freezeDiagnosticTraceContext,
} from "../../infra/diagnostic-trace-context.js";
import { estimateUsageCost, resolveModelCostConfig } from "../../utils/usage-format.js";
import { buildFallbackClearedNotice, buildFallbackNotice } from "../fallback-state.js";
import {
isReplyPayloadStatusNotice,
markReplyPayloadForSourceSuppressionDelivery,
} from "../reply-payload.js";
import type { ReplyPayload } from "../types.js";
import {
buildSilentFallbackFailurePayload,
enqueueCommitmentExtractionForTurn,
hasSuccessfulSourceReplyDelivery,
hasSuccessfulTerminalSourceReplyDelivery,
refreshSessionEntryFromStore,
resolveSourceReplyPolicy,
} from "./agent-runner-core.js";
import { buildEmptyInteractiveReplyPayload } from "./agent-runner-failure-reply.js";
import { signalTypingIfNeeded } from "./agent-runner-helpers.js";
import { buildReplyPayloads } from "./agent-runner-payloads.js";
import {
appendUnscheduledReminderNote,
hasSessionRelatedCronJobs,
hasUnbackedReminderCommitment,
} from "./agent-runner-reminder-guard.js";
import type { accountReplyAgentRun } from "./agent-runner-result-accounting.js";
import type { FinalizeReplyAgentRunInput } from "./agent-runner-result.types.js";
import { resolveResponseUsageLine } from "./agent-runner-usage-line.js";
import { attachMcpAppChannelAction } from "./mcp-app-channel-action.js";
import { normalizeReplyPayload } from "./normalize-reply.js";
import { resolveOriginMessageTo } from "./origin-routing.js";
import { createReplyToModeFilterForChannel } from "./reply-threading.js";
import { buildStrandedReplyDeliveryFailurePayload } from "./stranded-reply-recovery.js";
type ReplyAgentAccounting = Awaited<ReturnType<typeof accountReplyAgentRun>>;
export async function prepareReplyAgentPayloads(state: {
context: FinalizeReplyAgentRunInput;
accounting: ReplyAgentAccounting;
}) {
const { context, accounting } = state;
const {
activeSessionStore,
blockReplyPipeline,
blockStreamingEnabled,
cfg,
commandBody,
followupRun,
isHeartbeat,
opts,
replyMediaContext,
replyOperation,
replyRouteThreadId,
replyThreadingOverride,
replyToChannel,
replyToMode,
returnWithQueuedFollowupDrain,
runStartedAt,
runtimePolicySessionKey,
sessionCtx,
sessionKey,
storePath,
typingSignals,
} = context;
const {
configuredFallbackModel,
contextTokensUsed,
directlySentBlockKeys,
directlySentBlockPayloads,
fallbackAttempts,
fallbackExhausted,
fallbackTransition,
hasBillableUsageBuckets,
modelUsed,
payloadArray,
preserveUserFacingSessionState,
promptTokens,
providerUsed,
replyUsageState,
runId,
runResult,
selectedModel,
selectedProvider,
terminalFailurePayload,
usage,
verboseEnabled,
} = accounting;
let { activeSessionEntry, didLogHeartbeatStrip } = accounting;
const successfulSourceReplyDelivery = hasSuccessfulSourceReplyDelivery({
blockReplyPipeline,
directlySentBlockKeys,
messagingToolSentTexts: runResult.messagingToolSentTexts,
messagingToolSentMediaUrls: runResult.messagingToolSentMediaUrls,
messagingToolSentTargets: runResult.messagingToolSentTargets,
});
const committedMessagingToolSourceReplyDelivery =
hasCommittedSourceReplyDeliveryEvidence(runResult);
const completedSourceReplyDelivery = hasCompletedSourceReplyDeliveryEvidence(runResult);
const visibleOutboundDelivery = hasVisibleOutboundDeliveryEvidence(runResult);
const successfulSideEffectDelivery =
successfulSourceReplyDelivery ||
committedMessagingToolSourceReplyDelivery ||
visibleOutboundDelivery ||
runResult.didSendDeterministicApprovalPrompt === true;
const successfulTerminalDelivery =
hasSuccessfulTerminalSourceReplyDelivery({
blockReplyPipeline,
directlySentBlockPayloads,
}) || hasCompletedTerminalDeliveryEvidence(runResult);
// Compaction notices are progress, not a terminal reply. Dispatcher-backed
// delivery settles after this run returns, so it cannot prove turn completion here.
const shouldDeliverTerminalFailure = Boolean(
terminalFailurePayload && !successfulTerminalDelivery,
);
const fallbackFailureKnown =
fallbackAttempts.length > 0 || configuredFallbackModel.persistedAutoFallback;
const hasSpecificFallbackFailure = fallbackTransition.fallbackActive && fallbackFailureKnown;
const emptyInteractiveReplyPayload = terminalFailurePayload
? undefined
: buildEmptyInteractiveReplyPayload({
isInteractive:
followupRun.currentInboundEventKind !== "room_event" &&
(followupRun.run.inputProvenance?.kind === undefined ||
followupRun.run.inputProvenance.kind === "external_user"),
isHeartbeat,
silentExpected: followupRun.run.silentExpected,
allowEmptyAssistantReplyAsSilent: followupRun.run.allowEmptyAssistantReplyAsSilent,
isMessageToolOnly:
(opts?.sourceReplyDeliveryMode ?? followupRun.run.sourceReplyDeliveryMode) ===
"message_tool_only",
hasPendingContinuation:
runResult.meta?.yielded === true || (runResult.meta?.pendingToolCalls?.length ?? 0) > 0,
hasExplicitSilentReply: hasDeliberateSilentTerminalReply(runResult),
hasCommittedDelivery: successfulTerminalDelivery,
sessionCtx,
cfg,
});
const buildStrandedRetryMissingDeliveryDiagnostic = (): ReplyPayload | undefined => {
if (!sessionKey || !storePath || followupRun.strandedReplyRetry !== true) {
return undefined;
}
if (sessionCtx.InboundEventKind === "room_event" || completedSourceReplyDelivery) {
return undefined;
}
const sourceReplyPolicy = resolveSourceReplyPolicy({
cfg,
sessionCtx,
sessionEntry: activeSessionEntry,
sessionKey,
runtimePolicySessionKey,
opts,
});
if (
sourceReplyPolicy.sourceReplyDeliveryMode !== "message_tool_only" ||
sourceReplyPolicy.sendPolicyDenied
) {
return undefined;
}
return buildStrandedReplyDeliveryFailurePayload();
};
if (opts?.sourceReplyDeliveryMode === "message_tool_only" && completedSourceReplyDelivery) {
await opts.onObservedReplyDelivery?.();
}
const currentMessageId = sessionCtx.MessageSidFull ?? sessionCtx.MessageSid;
// A terminal fallback is built separately after normal payload filtering.
// Share this state across deliverable lanes so replyToMode=first still threads
// at most one visible payload without hidden reasoning/commentary consuming it.
const applyDeliveredReplyToMode = createReplyToModeFilterForChannel(replyToMode, replyToChannel);
const applyFinalReplyToMode = (payload: ReplyPayload) => {
const isDisabledReasoningLane =
payload.isReasoning === true && opts?.reasoningPayloadsEnabled !== true;
const isDisabledCommentaryLane =
payload.isCommentary === true && opts?.commentaryPayloadsEnabled !== true;
const isFilteredPayload =
normalizeReplyPayload(payload, { applyChannelTransforms: false }) === null;
return isDisabledReasoningLane || isDisabledCommentaryLane || isFilteredPayload
? payload
: applyDeliveredReplyToMode(payload);
};
const buildFinalPayloads = (payloads: ReplyPayload[]) =>
buildReplyPayloads({
config: cfg,
payloads,
isHeartbeat,
didLogHeartbeatStrip,
silentExpected: followupRun.run.silentExpected,
blockStreamingEnabled,
blockReplyPipeline,
directlySentBlockKeys,
directlySentBlockPayloads,
replyToMode,
replyToChannel,
currentMessageId,
replyThreading: replyThreadingOverride ?? sessionCtx.ReplyThreading,
applyReplyToMode: applyFinalReplyToMode,
messageProvider: followupRun.run.messageProvider,
messagingToolSentTexts: runResult.messagingToolSentTexts,
messagingToolSentMediaUrls: runResult.messagingToolSentMediaUrls,
messagingToolSentTargets: runResult.messagingToolSentTargets,
originatingChannel: sessionCtx.OriginatingChannel,
originatingChatType: sessionCtx.ChatType,
originatingTo: resolveOriginMessageTo({
originatingTo: sessionCtx.OriginatingTo,
to: sessionCtx.To,
}),
originatingThreadId: replyRouteThreadId,
accountId: sessionCtx.AccountId,
normalizeMediaPaths: replyMediaContext.normalizePayload,
});
const returnPreparedFallbackPayload = async (
payload: ReplyPayload,
): Promise<ReplyPayload | undefined> => {
const result = await buildFinalPayloads([payload]);
didLogHeartbeatStrip = result.didLogHeartbeatStrip;
const preparedPayload = result.replyPayloads[0];
if (!preparedPayload) {
return undefined;
}
await signalTypingIfNeeded([preparedPayload], typingSignals);
return returnWithQueuedFollowupDrain(preparedPayload);
};
const returnSilentFallbackFailureIfNeeded = async (): Promise<ReplyPayload | undefined> => {
const silentFallbackFailurePayload = buildSilentFallbackFailurePayload({
fallbackTransition,
fallbackFailureKnown,
isHeartbeat,
hasSuccessfulTerminalDelivery: successfulTerminalDelivery,
allowEmptyAssistantReplyAsSilent: followupRun.run.allowEmptyAssistantReplyAsSilent,
silentExpected: followupRun.run.silentExpected,
});
if (!silentFallbackFailurePayload) {
return undefined;
}
replyOperation.fail(
"run_failed",
new Error(
`configured model backend ${fallbackTransition.selectedModelRef} failed and fallback ${fallbackTransition.activeModelRef} produced no visible reply`,
),
);
return returnPreparedFallbackPayload(silentFallbackFailurePayload);
};
const fallbackNoticePayloads: ReplyPayload[] = [];
if (
!fallbackExhausted &&
!preserveUserFacingSessionState &&
fallbackTransition.fallbackTransitioned
) {
emitAgentEvent({
runId,
sessionKey,
stream: "lifecycle",
data: {
phase: "fallback",
selectedProvider,
selectedModel,
activeProvider: providerUsed,
activeModel: modelUsed,
reasonSummary: fallbackTransition.reasonSummary,
attemptSummaries: fallbackTransition.attemptSummaries,
attempts: fallbackAttempts,
},
});
const fallbackNotice = buildFallbackNotice({
selectedProvider,
selectedModel,
activeProvider: providerUsed,
activeModel: modelUsed,
attempts: fallbackAttempts,
cfg,
});
if (fallbackNotice) {
fallbackNoticePayloads.push(
markReplyPayloadForSourceSuppressionDelivery({
text: fallbackNotice,
isFallbackNotice: true,
}),
);
}
}
if (!fallbackExhausted && !preserveUserFacingSessionState && fallbackTransition.fallbackCleared) {
emitAgentEvent({
runId,
sessionKey,
stream: "lifecycle",
data: {
phase: "fallback_cleared",
selectedProvider,
selectedModel,
activeProvider: providerUsed,
activeModel: modelUsed,
previousActiveModel: fallbackTransition.previousState.activeModel,
},
});
fallbackNoticePayloads.push(
markReplyPayloadForSourceSuppressionDelivery({
text: buildFallbackClearedNotice({
selectedProvider,
selectedModel,
previousActiveModel: fallbackTransition.previousState.activeModel,
}),
isFallbackNotice: true,
}),
);
}
// Drain any late tool/block deliveries before deciding there's "nothing to send".
// Otherwise, a late typing trigger (e.g. from a tool callback) can outlive the run and
// keep the typing indicator stuck.
if (
payloadArray.length === 0 &&
fallbackNoticePayloads.length === 0 &&
!shouldDeliverTerminalFailure &&
(!emptyInteractiveReplyPayload || hasSpecificFallbackFailure)
) {
const silentFallbackFailurePayload = await returnSilentFallbackFailureIfNeeded();
if (silentFallbackFailurePayload) {
return { kind: "return" as const, value: silentFallbackFailurePayload };
}
const strandedRetryDiagnostic = buildStrandedRetryMissingDeliveryDiagnostic();
if (strandedRetryDiagnostic) {
return {
kind: "return" as const,
value: returnWithQueuedFollowupDrain(strandedRetryDiagnostic),
};
}
return { kind: "return" as const, value: returnWithQueuedFollowupDrain(undefined) };
}
const payloadCandidates = (
fallbackNoticePayloads.length > 0 ? [...fallbackNoticePayloads, ...payloadArray] : payloadArray
).filter(
(payload) =>
(payload.isReasoning !== true || opts?.reasoningPayloadsEnabled === true) &&
(payload.isCommentary !== true || opts?.commentaryPayloadsEnabled === true),
);
const payloadResult = await buildFinalPayloads(payloadCandidates);
let { replyPayloads } = payloadResult;
didLogHeartbeatStrip = payloadResult.didLogHeartbeatStrip;
const hasTerminalReplyPayload = replyPayloads.some(
(payload) =>
!payload.isReasoning &&
!payload.isCommentary &&
!isReplyPayloadStatusNotice(payload) &&
normalizeReplyPayload(payload, { applyChannelTransforms: false }) !== null,
);
if (shouldDeliverTerminalFailure && !hasTerminalReplyPayload && terminalFailurePayload) {
const terminalPayloadResult = await buildFinalPayloads([terminalFailurePayload]);
replyPayloads = [...replyPayloads, ...terminalPayloadResult.replyPayloads];
didLogHeartbeatStrip = terminalPayloadResult.didLogHeartbeatStrip;
} else if (hasSpecificFallbackFailure && !hasTerminalReplyPayload) {
const silentFallbackFailurePayload = await returnSilentFallbackFailureIfNeeded();
if (silentFallbackFailurePayload) {
return { kind: "return" as const, value: silentFallbackFailurePayload };
}
} else if (emptyInteractiveReplyPayload && !hasTerminalReplyPayload) {
const emptyPayloadResult = await buildFinalPayloads([emptyInteractiveReplyPayload]);
replyPayloads = [...replyPayloads, ...emptyPayloadResult.replyPayloads];
didLogHeartbeatStrip = emptyPayloadResult.didLogHeartbeatStrip;
if (emptyPayloadResult.replyPayloads.length > 0) {
replyOperation.retainFailureUntilComplete();
replyOperation.fail(
"run_failed",
new Error("interactive agent run completed without a visible reply"),
);
}
}
replyPayloads = attachMcpAppChannelAction({
payloads: replyPayloads,
channel: replyToChannel,
sessionKey,
view: runResult.latestMcpAppChannelView,
});
const hasVisibleReplyPayload = replyPayloads.some(
(payload) =>
!isReplyPayloadStatusNotice(payload) &&
(payload.isReasoning !== true || opts?.reasoningPayloadsEnabled === true) &&
(payload.isCommentary !== true || opts?.commentaryPayloadsEnabled === true) &&
normalizeReplyPayload(payload, { applyChannelTransforms: false }) !== null,
);
const hasDeliveredBlockStream = Boolean(
blockReplyPipeline?.didStream() && !blockReplyPipeline.isAborted(),
);
const canDeliverStandaloneFallbackNotice =
hasDeliveredBlockStream || successfulSideEffectDelivery;
if (
replyPayloads.length === 0 ||
(!hasVisibleReplyPayload && !canDeliverStandaloneFallbackNotice)
) {
const silentFallbackFailurePayload = await returnSilentFallbackFailureIfNeeded();
if (silentFallbackFailurePayload) {
return { kind: "return" as const, value: silentFallbackFailurePayload };
}
const strandedRetryDiagnostic = buildStrandedRetryMissingDeliveryDiagnostic();
if (strandedRetryDiagnostic) {
return {
kind: "return" as const,
value: returnWithQueuedFollowupDrain(strandedRetryDiagnostic),
};
}
return { kind: "return" as const, value: returnWithQueuedFollowupDrain(undefined) };
}
const successfulCronAdds = runResult.successfulCronAdds ?? 0;
const hasReminderCommitment = replyPayloads.some(
(payload) =>
!payload.isError &&
!isReplyPayloadStatusNotice(payload) &&
typeof payload.text === "string" &&
hasUnbackedReminderCommitment(payload.text),
);
// Suppress the guard note when an existing cron job (created in a prior
// turn) already covers the commitment — avoids false positives (#32228).
const coveredByExistingCron =
hasReminderCommitment && successfulCronAdds === 0
? await hasSessionRelatedCronJobs({
cronStorePath: undefined,
sessionKey,
})
: false;
const guardedReplyPayloads =
hasReminderCommitment && successfulCronAdds === 0 && !coveredByExistingCron
? appendUnscheduledReminderNote(replyPayloads)
: replyPayloads;
enqueueCommitmentExtractionForTurn({
cfg,
commandBody,
isHeartbeat,
followupRun,
sessionCtx,
sessionKey,
replyToChannel,
payloads: replyPayloads,
runId,
});
await signalTypingIfNeeded(guardedReplyPayloads, typingSignals);
if (isDiagnosticsEnabled(cfg) && hasNonzeroUsage(usage)) {
const input = usage.input ?? 0;
const output = usage.output ?? 0;
const cacheRead = usage.cacheRead ?? 0;
const cacheWrite = usage.cacheWrite ?? 0;
const usagePromptTokens = input + cacheRead + cacheWrite;
const totalTokens = usage.total ?? usagePromptTokens + output;
const contextUsedTokens = deriveContextPromptTokens({
lastCallUsage: runResult.meta?.agentMeta?.lastCallUsage,
promptTokens,
usage,
});
const costConfig = resolveModelCostConfig({
provider: providerUsed,
model: modelUsed,
config: cfg,
});
const costUsd = hasBillableUsageBuckets
? estimateUsageCost({ usage, cost: costConfig })
: undefined;
emitTrustedDiagnosticEvent({
type: "model.usage",
...(runResult.diagnosticTrace
? {
trace: freezeDiagnosticTraceContext(
createChildDiagnosticTraceContext(runResult.diagnosticTrace),
),
}
: {}),
sessionKey,
sessionId: followupRun.run.sessionId,
channel: replyToChannel,
agentId: followupRun.run.agentId,
provider: providerUsed,
model: modelUsed,
usage: {
input,
output,
cacheRead,
cacheWrite,
promptTokens: usagePromptTokens,
total: totalTokens,
},
lastCallUsage: runResult.meta?.agentMeta?.lastCallUsage,
context: {
limit: contextTokensUsed,
...(contextUsedTokens !== undefined ? { used: contextUsedTokens } : {}),
},
costUsd,
durationMs: Date.now() - runStartedAt,
});
}
const responseUsageSessionRaw =
activeSessionEntry?.responseUsage ??
(sessionKey ? activeSessionStore?.[sessionKey]?.responseUsage : undefined);
const responseUsageLine = resolveResponseUsageLine({
config: cfg,
sessionRaw: responseUsageSessionRaw,
channel: replyToChannel,
usage,
provider: providerUsed,
model: modelUsed,
preserveUserFacingSessionState,
replyUsageState,
});
if (verboseEnabled) {
activeSessionEntry = refreshSessionEntryFromStore({
storePath,
sessionKey,
fallbackEntry: activeSessionEntry,
activeSessionStore,
});
}
return {
kind: "continue" as const,
activeSessionEntry,
completedSourceReplyDelivery,
didLogHeartbeatStrip,
guardedReplyPayloads,
responseUsageLine,
};
}
@@ -0,0 +1,16 @@
import type { ReplyPayload } from "../types.js";
import { accountReplyAgentRun } from "./agent-runner-result-accounting.js";
import { completeReplyAgentRun } from "./agent-runner-result-complete.js";
import { prepareReplyAgentPayloads } from "./agent-runner-result-payloads.js";
import type { FinalizeReplyAgentRunInput } from "./agent-runner-result.types.js";
export async function finalizeReplyAgentRun(
context: FinalizeReplyAgentRunInput,
): Promise<ReplyPayload | ReplyPayload[] | undefined> {
const accounting = await accountReplyAgentRun(context);
const prepared = await prepareReplyAgentPayloads({ context, accounting });
if (prepared.kind === "return") {
return prepared.value;
}
return await completeReplyAgentRun({ context, accounting, prepared });
}
@@ -0,0 +1,53 @@
import type { OpenClawConfig } from "../../config/config.js";
import type { SessionEntry } from "../../config/sessions.js";
import type { OriginatingChannelType } from "../templating.js";
import type { RunReplyAgentParams } from "./agent-runner-core.js";
import type { AgentRunLoopResult } from "./agent-runner-execution.types.js";
import type { BlockReplyPipeline } from "./block-reply-pipeline.js";
import type { FollowupRun } from "./queue.js";
import type { ReplyMediaContext } from "./reply-media-paths.js";
import type { ReplyOperation } from "./reply-run-registry.js";
import type { resolveReplyToMode } from "./reply-threading.js";
import type { resolveRoutedDeliveryThreadId } from "./routed-delivery-thread.js";
import type { TypingSignaler } from "./typing-mode.js";
type SuccessfulAgentRun = Extract<AgentRunLoopResult, { kind: "success" }>;
export type FinalizeReplyAgentRunInput = Pick<
RunReplyAgentParams,
| "agentCfgContextTokens"
| "blockStreamingEnabled"
| "commandBody"
| "defaultModel"
| "followupRun"
| "opts"
| "queueKey"
| "replyThreadingOverride"
| "resolvedBlockStreamingBreak"
| "resolvedQueue"
| "resolvedVerboseLevel"
| "runtimePolicySessionKey"
| "sessionCtx"
| "sessionKey"
| "shouldInjectGroupIntro"
| "storePath"
> & {
activeIsNewSession: boolean;
activeSessionEntry: SessionEntry | undefined;
activeSessionStore: Record<string, SessionEntry> | undefined;
blockReplyPipeline: BlockReplyPipeline | null;
cfg: OpenClawConfig;
isHeartbeat: boolean;
pendingToolTasks: Set<Promise<void>>;
preflightCompactionApplied: boolean | undefined;
replyMediaContext: ReplyMediaContext;
replyOperation: ReplyOperation;
replyRouteThreadId: ReturnType<typeof resolveRoutedDeliveryThreadId>;
replyToChannel: OriginatingChannelType | undefined;
replyToMode: ReturnType<typeof resolveReplyToMode>;
returnWithQueuedFollowupDrain: <T>(value: T) => T;
runFollowupTurn: (queued: FollowupRun) => Promise<void>;
runOutcome: SuccessfulAgentRun;
runStartedAt: number;
typingSignals: TypingSignaler;
};
+692
View File
@@ -0,0 +1,692 @@
import { expectDefined } from "@openclaw/normalization-core";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import {
formatEmbeddedAgentQueueFailureSummary,
queueEmbeddedAgentMessageWithOutcomeAsync,
} from "../../agents/embedded-agent-runner/runs.js";
import { isIngressAdoptionLostError } from "../../channels/message/ingress-drain.js";
import { hasRestartRecoverySourceClaim } from "../../config/sessions/restart-recovery-state.js";
import { loadSessionEntry, updateSessionEntry } from "../../config/sessions/session-accessor.js";
import { logVerbose } from "../../globals.js";
import { measureDiagnosticsTimelineSpan } from "../../infra/diagnostics-timeline.js";
import {
buildHandledBeforeAgentReplyPayloads,
runBeforeAgentReplyForTurn,
withBeforeAgentReplyObserver,
} from "../../plugins/before-agent-reply.js";
import {
buildAgentHookContextChannelFields,
buildAgentHookContextIdentityFields,
} from "../../plugins/hook-agent-context.js";
import { markReplyPayloadForSourceSuppressionDelivery } from "../reply-payload.js";
import type { OriginatingChannelType } from "../templating.js";
import type { ReplyPayload } from "../types.js";
import {
BLOCK_REPLY_SEND_TIMEOUT_MS,
cleanupReplyAgentRun,
handleReplyAgentRunError,
refreshSessionEntryFromStore,
resolveAdmittedRunSessionFile,
type RunReplyAgentParams,
scheduleFollowupDrainAfterReplyOperationClear,
} from "./agent-runner-core.js";
import {
createReplyAgentRestartRecoveryController,
executePreparedReplyAgentRun,
} from "./agent-runner-execute.js";
import {
createShouldEmitToolOutput,
createShouldEmitToolResult,
isAudioPayload,
} from "./agent-runner-helpers.js";
import { resetReplyRunSession } from "./agent-runner-session-reset.js";
import { resolveQueuedReplyExecutionConfig } from "./agent-runner-utils.js";
import { createAudioAsVoiceBuffer, createBlockReplyPipeline } from "./block-reply-pipeline.js";
import { resolveEffectiveBlockStreamingConfig } from "./block-streaming.js";
import {
type CompactionNoticePhase,
createCompactionNoticePayload,
shouldNotifyUserAboutCompaction,
} from "./compaction-notice.js";
import { createFollowupRunner } from "./followup-runner.js";
import { REPLY_RUN_STILL_SHUTTING_DOWN_TEXT } from "./get-reply-run-queue.js";
import { resolveOriginMessageProvider } from "./origin-routing.js";
import { resolveActiveRunQueueAction } from "./queue-policy.js";
import { enqueueFollowupRun, type FollowupRun, scheduleFollowupDrain } from "./queue.js";
import { createReplyMediaContext } from "./reply-media-paths.js";
import { resolveReplyOperationRunState } from "./reply-operation-run-state.js";
import { type ReplyOperation, replyRunRegistry } from "./reply-run-registry.js";
import { createReplyToModeFilterForChannel, resolveReplyToMode } from "./reply-threading.js";
import { admitReplyTurn, resolveReplyTurnKind } from "./reply-turn-admission.js";
import {
isDuplicateRestartRecoverySource,
retireTerminalRestartRecoverySourceClaim,
} from "./restart-recovery-claim.js";
import { resolveRoutedDeliveryThreadId } from "./routed-delivery-thread.js";
import { buildChannelSourceTurnId, readChannelSourceTurnId } from "./source-turn-id.js";
import { createTypingSignaler } from "./typing-mode.js";
export async function runReplyAgent(
params: RunReplyAgentParams,
): Promise<ReplyPayload | ReplyPayload[] | undefined> {
const {
commandBody,
transcriptCommandBody,
followupRun,
queueKey,
resolvedQueue,
shouldSteer,
shouldFollowup,
isActive,
isRunActive,
opts,
typing,
sessionEntry,
sessionStore,
sessionKey,
runtimePolicySessionKey,
storePath,
defaultModel,
agentCfgContextTokens,
resolvedVerboseLevel,
toolProgressDetail,
isNewSession,
blockStreamingEnabled,
blockReplyChunking,
resolvedBlockStreamingBreak,
sessionCtx,
shouldInjectGroupIntro,
typingMode,
resetTriggered,
replyThreadingOverride,
replyOperation: providedReplyOperation,
} = params;
// One lifecycle for all adoption sites in this run.
const turnAdoptionLifecycle = opts?.turnAdoptionLifecycle;
let activeSessionEntry = sessionEntry;
const activeSessionStore = sessionStore;
let activeIsNewSession = isNewSession;
const effectiveResetTriggered = resetTriggered === true;
const activeRunQueueMode = effectiveResetTriggered ? "interrupt" : resolvedQueue.mode;
const isHeartbeat = opts?.isHeartbeat === true;
const replyOperationRunState = resolveReplyOperationRunState(opts);
const traceAttributes = {
provider: followupRun.run.provider,
hasSessionKey: Boolean(sessionKey ?? followupRun.run.sessionKey),
isHeartbeat,
queueMode: resolvedQueue.mode,
isActive,
blockStreamingEnabled,
};
const traceAgentPhase = <T>(name: string, run: () => Promise<T> | T): Promise<T> =>
measureDiagnosticsTimelineSpan(name, run, {
phase: "agent-turn",
config: followupRun.run.config,
attributes: traceAttributes,
});
const effectiveShouldSteer = !isHeartbeat && !effectiveResetTriggered && shouldSteer;
const effectiveShouldFollowup = !effectiveResetTriggered && shouldFollowup;
const typingSignals = createTypingSignaler({
typing,
mode: typingMode,
isHeartbeat,
});
const restartRecoverySourceTurnId = readChannelSourceTurnId(sessionCtx);
const restartRecoveryEntry =
sessionKey && storePath
? (loadSessionEntry({
storePath,
sessionKey,
clone: false,
hydrateSkillPromptRefs: false,
}) ?? activeSessionEntry)
: activeSessionEntry;
if (
restartRecoverySourceTurnId &&
isDuplicateRestartRecoverySource(restartRecoveryEntry, restartRecoverySourceTurnId)
) {
// Durable source ownership identifies provider redelivery even if the run
// became terminal before its claim cleanup committed.
if (
restartRecoveryEntry?.status !== "running" &&
sessionKey &&
storePath &&
hasRestartRecoverySourceClaim(restartRecoveryEntry, restartRecoverySourceTurnId)
) {
const retired = await retireTerminalRestartRecoverySourceClaim({
sessionId: restartRecoveryEntry.sessionId,
sessionKey,
sourceTurnId: restartRecoverySourceTurnId,
storePath,
});
if (retired) {
activeSessionEntry = retired;
if (activeSessionStore) {
activeSessionStore[sessionKey] = retired;
}
}
}
typing.cleanup();
return undefined;
}
const baseShouldEmitToolResult = createShouldEmitToolResult({
sessionKey,
storePath,
resolvedVerboseLevel,
});
const channelProgressCanConsumeToolResults =
Boolean(opts?.forceToolResultProgress) && Boolean(opts?.onToolResult);
const shouldEmitToolResult = () =>
channelProgressCanConsumeToolResults || baseShouldEmitToolResult();
const shouldEmitToolOutput = createShouldEmitToolOutput({
sessionKey,
storePath,
resolvedVerboseLevel,
});
const pendingToolTasks = new Set<Promise<void>>();
const blockReplyTimeoutMs = opts?.blockReplyTimeoutMs ?? BLOCK_REPLY_SEND_TIMEOUT_MS;
const touchActiveSessionEntry = async () => {
if (!activeSessionEntry || !activeSessionStore || !sessionKey) {
return;
}
const updatedAt = Date.now();
activeSessionEntry.updatedAt = updatedAt;
activeSessionStore[sessionKey] = activeSessionEntry;
if (storePath) {
await updateSessionEntry({ storePath, sessionKey }, () => ({ updatedAt }), {
skipMaintenance: true,
takeCacheOwnership: true,
});
}
};
let shouldQueueAfterSteerRejection = false;
let beforeAgentReplyDispatchedForSteer = false;
if (effectiveShouldSteer && isActive) {
// Steer against the operation that owns THIS session's run slot. A native
// command continuation whose slot adoption was skipped (#104844) still
// carries a source-keyed reservation; steering by its stale sessionId
// would miss the live target run.
const registeredReplyOperation = sessionKey ? replyRunRegistry.get(sessionKey) : undefined;
const activeReplyOperation =
providedReplyOperation?.key === sessionKey
? providedReplyOperation
: (registeredReplyOperation ?? providedReplyOperation);
const steerSessionId = activeReplyOperation?.sessionId ?? followupRun.run.sessionId;
// Channel dispatch normally stamps the route-scoped source id. Internal
// callers can derive the same per-message identity from the prepared turn.
const steerRunId = expectDefined(
restartRecoverySourceTurnId ??
buildChannelSourceTurnId({
provider:
followupRun.originatingChannel ??
followupRun.run.messageProvider ??
sessionCtx.Provider,
accountId:
followupRun.originatingAccountId ??
followupRun.run.agentAccountId ??
sessionCtx.AccountId,
conversationId:
followupRun.originatingTo ??
followupRun.originatingChatId ??
sessionKey ??
followupRun.run.sessionKey,
messageId: followupRun.messageId ?? sessionCtx.MessageSidFull ?? sessionCtx.MessageSid,
}) ??
normalizeOptionalString(opts?.runId),
"steered turn id",
);
const trigger = "user";
const hookResult = await runBeforeAgentReplyForTurn({
runId: steerRunId,
trigger,
event: { cleanedBody: followupRun.prompt },
context: {
runId: steerRunId,
agentId: followupRun.run.agentId,
sessionKey: sessionKey ?? followupRun.run.sessionKey,
sessionId: steerSessionId,
workspaceDir: followupRun.run.workspaceDir,
modelProviderId: followupRun.run.provider,
modelId: followupRun.run.model,
trigger,
...buildAgentHookContextChannelFields({
sessionKey: sessionKey ?? followupRun.run.sessionKey,
messageChannel: followupRun.originatingChannel,
messageProvider: followupRun.run.messageProvider,
currentChannelId: followupRun.originatingChatId,
messageTo: followupRun.originatingTo,
senderId: followupRun.run.senderId,
}),
...buildAgentHookContextIdentityFields({
trigger,
senderId: followupRun.run.senderId,
chatId: followupRun.originatingChatId,
channelContext: followupRun.run.channelContext,
}),
},
});
beforeAgentReplyDispatchedForSteer = true;
if (hookResult?.handled) {
typing.cleanup();
return buildHandledBeforeAgentReplyPayloads(hookResult.reply);
}
const steerOutcome = await queueEmbeddedAgentMessageWithOutcomeAsync(
steerSessionId,
followupRun.prompt,
{
steeringMode: "all",
isInboundUserMessage: true,
...(followupRun.images?.length ? { images: followupRun.images } : {}),
...(followupRun.imageOrder?.length ? { imageOrder: followupRun.imageOrder } : {}),
...(followupRun.media?.length ? { media: followupRun.media } : {}),
...(turnAdoptionLifecycle ? { waitForTranscriptCommit: true } : {}),
...(resolvedQueue.debounceMs !== undefined ? { debounceMs: resolvedQueue.debounceMs } : {}),
...(followupRun.run.sourceReplyDeliveryMode
? { sourceReplyDeliveryMode: followupRun.run.sourceReplyDeliveryMode }
: {}),
taskSuggestionDeliveryMode: followupRun.run.taskSuggestionDeliveryMode,
...(followupRun.userTurnTranscriptRecorder
? { userTurnTranscriptRecorder: followupRun.userTurnTranscriptRecorder }
: {}),
},
);
if (steerOutcome.queued) {
activeReplyOperation?.recordActivity();
try {
await turnAdoptionLifecycle?.onAdopted();
} catch (error) {
if (isIngressAdoptionLostError(error)) {
// Claim was tombstoned/superseded/guillotined after transcript commit.
// Cancel the active run so steered tools do not keep executing; do not
// rethrow — replaying ingress would duplicate the injected user turn.
const abortKey = sessionKey ?? queueKey;
if (abortKey) {
replyRunRegistry.abort(abortKey);
}
logVerbose(
`queue: active session ${steerSessionId} adoption lost after transcript commit (${error.code}); aborting steered turn without ingress replay`,
);
typing.cleanup();
return undefined;
}
// Ordinary callback failures: transcript-backed steering is irrevocable.
logVerbose(
`queue: active session ${steerSessionId} adoption finalizer failed after transcript commit: ${String(
error,
)}`,
);
}
if (followupRun.currentInboundAudio === true) {
activeReplyOperation?.markAcceptedSteeredInboundAudio();
}
await touchActiveSessionEntry();
typing.cleanup();
return undefined;
}
// The active runtime still owns the turn but cannot prove transcript adoption.
// Keep the inbound message queued so ingress can finalize after a later run.
shouldQueueAfterSteerRejection = steerOutcome.reason === "transcript_commit_wait_unsupported";
const summary = formatEmbeddedAgentQueueFailureSummary(steerOutcome);
logVerbose(`queue: active session ${steerSessionId} rejected steering injection: ${summary}`);
}
const activeRunQueueAction = resolveActiveRunQueueAction({
isActive,
isHeartbeat,
shouldFollowup: effectiveShouldFollowup || shouldQueueAfterSteerRejection,
queueMode: activeRunQueueMode,
resetTriggered: effectiveResetTriggered,
});
const baseQueuedRunFollowupTurn = createFollowupRunner({
opts,
typing,
typingMode,
sessionEntry: activeSessionEntry,
sessionStore: activeSessionStore,
sessionKey,
storePath,
defaultModel,
agentCfgContextTokens,
toolProgressDetail,
});
// A transcript-rejected steer can become this exact queued turn. Preserve its
// earlier hook decision without suppressing hooks for other queued messages.
const queuedRunFollowupTurn = (queued: FollowupRun) =>
beforeAgentReplyDispatchedForSteer && queued === followupRun
? withBeforeAgentReplyObserver(
{
beforeDispatch: async () => false,
afterDispatch: async (result) => result,
},
() => baseQueuedRunFollowupTurn(queued),
)
: baseQueuedRunFollowupTurn(queued);
if (activeRunQueueAction === "drop") {
if (replyOperationRunState) {
replyOperationRunState.admission = { status: "skipped", reason: "active-run" };
}
typing.cleanup();
return undefined;
}
if (activeRunQueueAction === "enqueue-followup") {
const enqueued = enqueueFollowupRun(
queueKey,
followupRun,
resolvedQueue,
"message-id",
queuedRunFollowupTurn,
false,
);
if (!enqueued) {
typing.cleanup();
return undefined;
}
// The queue must stay dormant while the active owner can still collect
// messages. Registering after enqueue closes the owner-clear race.
const activeReplyOperation = replyRunRegistry.get(queueKey);
if (activeReplyOperation) {
scheduleFollowupDrainAfterReplyOperationClear({
operation: activeReplyOperation,
queueKey,
runFollowup: queuedRunFollowupTurn,
});
} else {
scheduleFollowupDrain(queueKey, queuedRunFollowupTurn);
}
const queuedBehindActiveRun = isRunActive?.() === true;
await touchActiveSessionEntry();
if (queuedBehindActiveRun) {
await typingSignals.signalToolStart();
} else {
typing.cleanup();
}
return undefined;
}
followupRun.run.config = await resolveQueuedReplyExecutionConfig(followupRun.run.config, {
originatingChannel: sessionCtx.OriginatingChannel,
messageProvider: followupRun.run.messageProvider,
originatingAccountId: followupRun.originatingAccountId,
agentAccountId: followupRun.run.agentAccountId,
});
const replyToChannel = resolveOriginMessageProvider({
originatingChannel: sessionCtx.OriginatingChannel,
provider: sessionCtx.Surface ?? sessionCtx.Provider,
}) as OriginatingChannelType | undefined;
const replyToMode = resolveReplyToMode(
followupRun.run.config,
replyToChannel,
sessionCtx.AccountId,
sessionCtx.ChatType,
);
const applyReplyToMode = createReplyToModeFilterForChannel(replyToMode, replyToChannel);
const cfg = followupRun.run.config;
const replyMediaContext = createReplyMediaContext({
cfg,
sessionKey,
workspaceDir: followupRun.run.workspaceDir,
messageProvider: followupRun.run.messageProvider,
accountId: followupRun.originatingAccountId ?? followupRun.run.agentAccountId,
groupId: followupRun.run.groupId,
groupChannel: followupRun.run.groupChannel,
groupSpace: followupRun.run.groupSpace,
requesterSenderId: followupRun.run.senderId,
requesterSenderName: followupRun.run.senderName,
requesterSenderUsername: followupRun.run.senderUsername,
requesterSenderE164: followupRun.run.senderE164,
});
const compactionNoticeMessageId = sessionCtx.MessageSidFull ?? sessionCtx.MessageSid;
const sendDirectCompactionNotice = shouldNotifyUserAboutCompaction(cfg)
? async (phase: CompactionNoticePhase) => {
if (!opts?.onBlockReply) {
return;
}
const noticePayload = createCompactionNoticePayload({
phase,
currentMessageId: compactionNoticeMessageId,
applyReplyToMode,
});
try {
await opts.onBlockReply(noticePayload);
} catch (err) {
logVerbose(`context maintenance notice delivery failed: ${String(err)}`);
}
}
: undefined;
const blockReplyCoalescing =
blockStreamingEnabled && opts?.onBlockReply
? resolveEffectiveBlockStreamingConfig({
cfg,
provider: sessionCtx.Provider,
accountId: sessionCtx.AccountId,
chunking: blockReplyChunking,
}).coalescing
: undefined;
const blockReplyPipeline =
blockStreamingEnabled && opts?.onBlockReply
? createBlockReplyPipeline({
onBlockReply: opts.onBlockReply,
timeoutMs: blockReplyTimeoutMs,
coalescing: blockReplyCoalescing,
buffer: createAudioAsVoiceBuffer({ isAudioPayload }),
})
: null;
const replySessionKey = sessionKey ?? followupRun.run.sessionKey;
const replyRouteThreadId = resolveRoutedDeliveryThreadId({
ctx: sessionCtx,
sessionKey: replySessionKey,
});
let replyOperation: ReplyOperation;
if (providedReplyOperation) {
replyOperation = providedReplyOperation;
if (replyOperationRunState) {
replyOperationRunState.admission = { status: "owned" };
}
} else {
const replyTurnKind = resolveReplyTurnKind(opts);
const admission = await admitReplyTurn({
sessionId: followupRun.run.sessionId,
sessionKey: replySessionKey ?? "",
expectedSessionId: activeSessionEntry?.sessionId,
storePath,
kind: replyTurnKind,
resetTriggered: effectiveResetTriggered,
routeThreadId: replyRouteThreadId,
upstreamAbortSignal: opts?.abortSignal,
onReplyAdmissionWaitChange: opts?.onReplyAdmissionWaitChange,
});
if (replyOperationRunState) {
replyOperationRunState.admission =
admission.status === "owned"
? { status: "owned" }
: { status: "skipped", reason: admission.reason };
}
if (admission.status === "skipped") {
typing.cleanup();
if (admission.reason !== "active-run" || replyTurnKind !== "visible") {
return undefined;
}
return markReplyPayloadForSourceSuppressionDelivery({
text: REPLY_RUN_STILL_SHUTTING_DOWN_TEXT,
});
}
replyOperation = admission.operation;
const previousRunSessionId = followupRun.run.sessionId;
followupRun.run.sessionId = replyOperation.sessionId;
if (replyOperation.sessionId !== previousRunSessionId) {
const admittedSessionEntry = refreshSessionEntryFromStore({
storePath,
sessionKey: replySessionKey,
fallbackEntry: replySessionKey
? (activeSessionStore?.[replySessionKey] ?? activeSessionEntry)
: activeSessionEntry,
activeSessionStore,
});
if (admittedSessionEntry?.sessionId === replyOperation.sessionId) {
activeSessionEntry = admittedSessionEntry;
const admittedSessionFile = resolveAdmittedRunSessionFile({
agentId: followupRun.run.agentId,
sessionId: replyOperation.sessionId,
sessionFile: admittedSessionEntry.sessionFile,
storePath,
});
if (admittedSessionFile) {
followupRun.run.sessionFile = admittedSessionFile;
}
}
}
}
let runFollowupTurn = queuedRunFollowupTurn;
let shouldDrainQueuedFollowupsAfterClear = false;
const returnWithQueuedFollowupDrain = <T>(value: T): T => {
shouldDrainQueuedFollowupsAfterClear = true;
return value;
};
const {
admitUserTurn,
beginBeforeAgentReply,
checkpointBeforeAgentReply,
clear: clearRestartRecoveryDeliveryClaim,
isArmed: isRestartRecoveryArmed,
} = createReplyAgentRestartRecoveryController({
activeSessionStore,
cfg,
followupRun,
getActiveSessionEntry: () => activeSessionEntry,
opts,
replyOperation,
restartRecoverySourceTurnId,
runtimePolicySessionKey,
sessionCtx,
sessionKey,
setActiveSessionEntry: (entry) => {
activeSessionEntry = entry;
},
storePath,
});
type SessionResetOptions = {
failureLabel: string;
buildLogMessage: (nextSessionId: string) => string;
cleanupTranscripts?: boolean;
};
const resetSession = async ({
failureLabel,
buildLogMessage,
cleanupTranscripts,
}: SessionResetOptions): Promise<boolean> =>
await resetReplyRunSession({
options: {
failureLabel,
buildLogMessage,
cleanupTranscripts,
},
sessionKey,
queueKey,
activeSessionEntry,
activeSessionStore,
storePath,
messageThreadId:
typeof sessionCtx.MessageThreadId === "string" ? sessionCtx.MessageThreadId : undefined,
followupRun,
onActiveSessionEntry: (nextEntry) => {
activeSessionEntry = nextEntry;
},
onNewSession: () => {
activeIsNewSession = true;
},
});
const resetSessionAfterRoleOrderingConflict = async (reason: string): Promise<boolean> =>
resetSession({
failureLabel: "role ordering conflict",
buildLogMessage: (nextSessionId) =>
`Role ordering conflict (${reason}). Restarting session ${sessionKey} -> ${nextSessionId}.`,
cleanupTranscripts: true,
});
try {
return await executePreparedReplyAgentRun({
activeSessionStore,
admitUserTurn,
agentCfgContextTokens,
applyReplyToMode,
beforeAgentReplyDispatchedForSteer,
beginBeforeAgentReply,
blockReplyChunking,
blockReplyPipeline,
blockStreamingEnabled,
cfg,
checkpointBeforeAgentReply,
commandBody,
defaultModel,
followupRun,
getActiveIsNewSession: () => activeIsNewSession,
getActiveSessionEntry: () => activeSessionEntry,
isHeartbeat,
isRestartRecoveryArmed,
opts,
pendingToolTasks,
performSessionReset: resetSession,
queueKey,
replyMediaContext,
replyOperation,
replyRouteThreadId,
replyThreadingOverride,
replyToChannel,
replyToMode,
resetSessionAfterRoleOrderingConflict,
resolvedBlockStreamingBreak,
resolvedQueue,
resolvedVerboseLevel,
returnWithQueuedFollowupDrain,
runFollowupTurn,
runtimePolicySessionKey,
sendDirectCompactionNotice,
sessionCtx,
sessionKey,
setActiveSessionEntry: (entry) => {
activeSessionEntry = entry;
},
setRunFollowupTurn: (runner) => {
runFollowupTurn = runner;
},
shouldEmitToolOutput,
shouldEmitToolResult,
shouldInjectGroupIntro,
storePath,
toolProgressDetail,
traceAgentPhase,
transcriptCommandBody,
turnAdoptionLifecycle,
typing,
typingMode,
typingSignals,
});
} catch (error) {
return await handleReplyAgentRunError(error, {
cfg,
isRestartRecoveryArmed,
replyOperation,
resolvedVerboseLevel,
returnWithQueuedFollowupDrain,
sessionCtx,
});
} finally {
await cleanupReplyAgentRun({
blockReplyPipeline,
clearRestartRecoveryDeliveryClaim,
providedReplyOperation,
queueKey,
replyOperation,
runFollowupTurn,
sessionKey,
shouldDrainQueuedFollowupsAfterClear,
typing,
});
}
}
+668
View File
@@ -0,0 +1,668 @@
import { expectDefined } from "@openclaw/normalization-core";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { deriveContextPromptTokens } from "../../agents/usage.js";
import type { SessionEntry } from "../../config/sessions.js";
import { readLatestSessionUsageFromTranscriptAsync } from "../../gateway/session-transcript-readers.js";
import { formatTokenCount } from "../../utils/usage-format.js";
import type { ReplyPayload } from "../types.js";
function formatRawTraceBlock(title: string, value: string | undefined): string {
const body = value?.trim() ? escapeTraceFence(value) : "<empty>";
return `🔎 ${title}:\n~~~text\n${body}\n~~~`;
}
function escapeTraceFence(value: string): string {
return value.replace(/^~~~/gm, "\\~~~");
}
function hasTraceUsageFields(
usage:
| {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
}
| undefined,
): boolean {
if (!usage) {
return false;
}
return ["input", "output", "cacheRead", "cacheWrite", "total"].some((key) => {
const value = usage[key as keyof typeof usage];
return typeof value === "number" && Number.isFinite(value);
});
}
function formatTraceUsageLine(label: string, value: number | undefined): string {
return `${label}=${typeof value === "number" && Number.isFinite(value) ? `${value.toLocaleString()} tok (${formatTokenCount(value)})` : "n/a"}`;
}
function formatUsageTraceBlock(
title: string,
usage:
| {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
}
| undefined,
): string | undefined {
if (!hasTraceUsageFields(usage)) {
return undefined;
}
return `🔎 ${title}:\n~~~text\n${[
formatTraceUsageLine("input", usage?.input),
formatTraceUsageLine("output", usage?.output),
formatTraceUsageLine("cacheRead", usage?.cacheRead),
formatTraceUsageLine("cacheWrite", usage?.cacheWrite),
formatTraceUsageLine("total", usage?.total),
].join("\n")}\n~~~`;
}
type TraceAttemptView = {
provider: string;
model: string;
result: string;
reason?: string;
stage?: string;
elapsedMs?: number;
status?: number;
};
export type TraceExecutionView = {
winnerProvider?: string;
winnerModel?: string;
attempts?: TraceAttemptView[];
fallbackUsed?: boolean;
runner?: "embedded" | "cli";
};
export type TracePromptSegmentView = {
key: string;
chars: number;
};
export type TraceToolSummaryView = {
calls: number;
tools: string[];
failures?: number;
totalToolTimeMs?: number;
};
export type TraceCompletionView = {
finishReason?: string;
stopReason?: string;
refusal?: boolean;
};
export type TraceContextManagementView = {
sessionCompactions?: number;
lastTurnCompactions?: number;
preflightCompactionApplied?: boolean;
postCompactionContextInjected?: boolean;
};
function formatTraceScalar(value: string | number | boolean | undefined): string | undefined {
if (typeof value === "boolean") {
return value ? "yes" : "no";
}
if (typeof value === "number") {
return Number.isFinite(value) ? value.toLocaleString() : undefined;
}
const trimmed = normalizeOptionalString(value);
return trimmed ?? undefined;
}
function formatKeyValueTraceBlock(
title: string,
fields: Array<[string, string | number | boolean | undefined]>,
): string | undefined {
const lines = fields.flatMap(([key, rawValue]) => {
const value = formatTraceScalar(rawValue);
return value ? [`${key}=${value}`] : [];
});
if (lines.length === 0) {
return undefined;
}
return `🔎 ${title}:\n~~~text\n${lines.join("\n")}\n~~~`;
}
function inferFallbackAttemptResult(attempt: { reason?: string; status?: number }): string {
if (attempt.reason === "timeout") {
return "timeout";
}
return "candidate_failed";
}
export function mergeExecutionTrace(params: {
fallbackAttempts?: Array<{
provider: string;
model: string;
reason?: string;
status?: number;
}>;
executionTrace?: {
winnerProvider?: string;
winnerModel?: string;
attempts?: TraceAttemptView[];
fallbackUsed?: boolean;
runner?: "embedded" | "cli";
};
provider?: string;
model?: string;
runner: "embedded" | "cli";
exhausted?: boolean;
}): TraceExecutionView | undefined {
const executionAttempts = params.exhausted
? (params.executionTrace?.attempts ?? []).filter((attempt) => attempt.result !== "success")
: (params.executionTrace?.attempts ?? []);
const attempts: TraceAttemptView[] = [
...(params.fallbackAttempts ?? []).map((attempt) =>
Object.assign(
{
provider: attempt.provider,
model: attempt.model,
result: inferFallbackAttemptResult(attempt),
},
attempt.reason ? { reason: attempt.reason } : {},
typeof attempt.status === `number` ? { status: attempt.status } : {},
),
),
...executionAttempts,
];
const winnerProvider = params.exhausted
? undefined
: (params.executionTrace?.winnerProvider ?? normalizeOptionalString(params.provider));
const winnerModel = params.exhausted
? undefined
: (params.executionTrace?.winnerModel ?? normalizeOptionalString(params.model));
if (
winnerProvider &&
winnerModel &&
!attempts.some(
(attempt) =>
attempt.provider === winnerProvider &&
attempt.model === winnerModel &&
attempt.result === "success",
)
) {
attempts.push({
provider: winnerProvider,
model: winnerModel,
result: "success",
});
}
if (!winnerProvider && !winnerModel && attempts.length === 0) {
return undefined;
}
const fallbackAttemptCount = params.fallbackAttempts?.length ?? 0;
const traceFallbackUsed = params.executionTrace?.fallbackUsed;
return {
winnerProvider,
winnerModel,
attempts: attempts.length > 0 ? attempts : undefined,
fallbackUsed:
traceFallbackUsed === true ||
fallbackAttemptCount > 0 ||
(traceFallbackUsed === undefined && attempts.length > 1),
runner: params.executionTrace?.runner ?? params.runner,
};
}
function formatExecutionResultTraceBlock(
executionTrace: TraceExecutionView | undefined,
): string | undefined {
if (!executionTrace?.winnerProvider && !executionTrace?.winnerModel) {
return undefined;
}
return formatKeyValueTraceBlock("Execution Result", [
[
"winner",
executionTrace.winnerProvider && executionTrace.winnerModel
? `${executionTrace.winnerProvider}/${executionTrace.winnerModel}`
: undefined,
],
["fallbackUsed", executionTrace.fallbackUsed],
["attempts", executionTrace.attempts?.length],
["runner", executionTrace.runner],
]);
}
function formatFallbackChainTraceBlock(
executionTrace: TraceExecutionView | undefined,
): string | undefined {
const attempts = executionTrace?.attempts ?? [];
if (attempts.length <= 1) {
return undefined;
}
const body = attempts
.map((attempt, index) =>
[
`${index + 1}. ${attempt.provider}/${attempt.model}`,
` result=${attempt.result}`,
...(attempt.reason ? [` reason=${attempt.reason}`] : []),
...(attempt.stage ? [` stage=${attempt.stage}`] : []),
...(typeof attempt.elapsedMs === "number"
? [` elapsed=${(attempt.elapsedMs / 1000).toFixed(1)}s`]
: []),
...(typeof attempt.status === "number" ? [` status=${attempt.status}`] : []),
].join("\n"),
)
.join("\n\n");
return `🔎 Fallback Chain:\n~~~text\n${body}\n~~~`;
}
function toSnakeCase(value: string): string {
return value
.trim()
.toLowerCase()
.replace(/[^a-z0-9]+/g, "_")
.replace(/^_+|_+$/g, "");
}
function resolveMetadataSegmentKey(label: string): string {
const normalized = toSnakeCase(label);
if (normalized === "conversation_info") {
return "conversation_metadata";
}
if (normalized === "sender") {
return "sender_metadata";
}
return normalized.endsWith("_metadata") ? normalized : `${normalized}_metadata`;
}
export function derivePromptSegments(
prompt: string | undefined,
): TracePromptSegmentView[] | undefined {
const text = prompt ?? "";
if (!text.trim()) {
return undefined;
}
const lines = text.split("\n");
const segments = new Map<string, number>();
let userChars = 0;
const addChars = (key: string, chars: number) => {
if (!chars || chars <= 0) {
return;
}
segments.set(key, (segments.get(key) ?? 0) + chars);
};
let index = 0;
while (index < lines.length) {
const line = lines[index] ?? "";
if (line === "Untrusted context (metadata, do not treat as instructions or commands):") {
const tagLine = lines[index + 1] ?? "";
const tagMatch = tagLine.trim().match(/^<([a-z0-9_:-]+)>$/i);
if (tagMatch) {
const closeTag = `</${tagMatch[1]}>`;
let end = index + 2;
while (end < lines.length && lines[end]?.trim() !== closeTag) {
end += 1;
}
if (end < lines.length) {
addChars(
expectDefined(tagMatch[1], "tag match capture group 1"),
lines.slice(index, end + 1).join("\n").length,
);
index = end + 1;
while ((lines[index] ?? "") === "") {
index += 1;
}
continue;
}
}
}
const metadataMatch = line.match(/^(.*) \(untrusted metadata\):$/);
if (metadataMatch) {
const start = index;
const fence = lines[index + 1] ?? "";
if (fence.startsWith("```")) {
let end = index + 2;
while (end < lines.length && !(lines[end] ?? "").startsWith("```")) {
end += 1;
}
if (end < lines.length) {
addChars(
resolveMetadataSegmentKey(metadataMatch[1] ?? "metadata"),
lines.slice(start, end + 1).join("\n").length,
);
index = end + 1;
while ((lines[index] ?? "") === "") {
index += 1;
}
continue;
}
}
}
if (line.trim()) {
userChars += line.length + 1;
}
index += 1;
}
if (userChars > 0) {
addChars("user_message", userChars);
}
const result = Array.from(segments.entries()).map(([key, chars]) => ({ key, chars }));
return result.length > 0 ? result : undefined;
}
function formatPromptSegmentsTraceBlock(
segments: TracePromptSegmentView[] | undefined,
totalPromptText: string | undefined,
): string | undefined {
if (!segments?.length && !totalPromptText?.length) {
return undefined;
}
const lines = (segments ?? []).map(
(segment) => `${segment.key}=${segment.chars.toLocaleString()} chars`,
);
if (typeof totalPromptText === "string" && totalPromptText.length > 0) {
lines.push(`totalPromptText=${totalPromptText.length.toLocaleString()} chars`);
}
return lines.length > 0 ? `🔎 Prompt Segments:\n~~~text\n${lines.join("\n")}\n~~~` : undefined;
}
function formatToolSummaryTraceBlock(
toolSummary: TraceToolSummaryView | undefined,
): string | undefined {
if (!toolSummary || toolSummary.calls <= 0) {
return undefined;
}
return formatKeyValueTraceBlock("Tool Summary", [
["calls", toolSummary.calls],
["tools", toolSummary.tools.length > 0 ? toolSummary.tools.join(", ") : undefined],
["failures", toolSummary.failures],
["totalToolTimeMs", toolSummary.totalToolTimeMs],
]);
}
function formatCompletionTraceBlock(
completion: TraceCompletionView | undefined,
): string | undefined {
if (!completion) {
return undefined;
}
return formatKeyValueTraceBlock("Completion", [
["finishReason", completion.finishReason],
["stopReason", completion.stopReason],
["refusal", completion.refusal],
]);
}
function formatContextManagementTraceBlock(
contextManagement: TraceContextManagementView | undefined,
): string | undefined {
if (!contextManagement) {
return undefined;
}
return formatKeyValueTraceBlock("Context Management", [
["sessionCompactions", contextManagement.sessionCompactions],
["lastTurnCompactions", contextManagement.lastTurnCompactions],
["preflightCompactionApplied", contextManagement.preflightCompactionApplied],
["postCompactionContextInjected", contextManagement.postCompactionContextInjected],
]);
}
export async function accumulateSessionUsageFromTranscript(params: {
sessionId?: string;
storePath?: string;
sessionFile?: string;
}): Promise<
| {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
}
| undefined
> {
const sessionId = normalizeOptionalString(params.sessionId);
if (!sessionId) {
return undefined;
}
try {
const usage = await readLatestSessionUsageFromTranscriptAsync({
sessionId,
storePath: params.storePath,
sessionFile: params.sessionFile,
});
if (!usage) {
return undefined;
}
return {
input: usage.inputTokens,
output: usage.outputTokens,
cacheRead: usage.cacheRead,
cacheWrite: usage.cacheWrite,
total: usage.totalTokens,
};
} catch {
return undefined;
}
}
function formatRequestContextTraceBlock(params: {
provider?: string;
model?: string;
contextLimit?: number;
promptTokens?: number;
}): string | undefined {
const limit = params.contextLimit;
const used = params.promptTokens;
if (
(typeof limit !== "number" || !Number.isFinite(limit) || limit <= 0) &&
(typeof used !== "number" || !Number.isFinite(used) || used <= 0) &&
!params.provider &&
!params.model
) {
return undefined;
}
const headroom =
typeof limit === "number" &&
Number.isFinite(limit) &&
typeof used === "number" &&
Number.isFinite(used)
? Math.max(0, limit - used)
: undefined;
const percent =
typeof limit === "number" &&
Number.isFinite(limit) &&
limit > 0 &&
typeof used === "number" &&
Number.isFinite(used)
? Math.round((used / limit) * 100)
: undefined;
return `🔎 Context Window (Last Model Request):\n~~~text\n${[
`provider=${params.provider ?? "n/a"}`,
`model=${params.model ?? "n/a"}`,
`used=${typeof used === "number" && Number.isFinite(used) ? `${used.toLocaleString()} tok (${formatTokenCount(used)})` : "n/a"}`,
`limit=${typeof limit === "number" && Number.isFinite(limit) ? `${limit.toLocaleString()} tok (${formatTokenCount(limit)})` : "n/a"}`,
`headroom=${typeof headroom === "number" ? `${headroom.toLocaleString()} tok (${formatTokenCount(headroom)})` : "n/a"}`,
`usage=${typeof percent === "number" ? `${percent}%` : "n/a"}`,
].join("\n")}\n~~~`;
}
function formatSummaryPromptValue(params: {
contextLimit?: number;
promptTokens?: number;
}): string | undefined {
const used = params.promptTokens;
const limit = params.contextLimit;
if (
typeof used !== "number" ||
!Number.isFinite(used) ||
used <= 0 ||
typeof limit !== "number" ||
!Number.isFinite(limit) ||
limit <= 0
) {
return undefined;
}
return `${formatTokenCount(used)}/${formatTokenCount(limit)}`;
}
function formatRawTraceSummaryLine(params: {
executionTrace?: TraceExecutionView;
completion?: TraceCompletionView;
contextLimit?: number;
promptTokens?: number;
usage?: {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
};
toolSummary?: TraceToolSummaryView;
contextManagement?: TraceContextManagementView;
requestShaping?: {
thinking?: string;
};
}): string | undefined {
const thinking = normalizeOptionalString(params.requestShaping?.thinking);
const fields = [
params.executionTrace?.winnerModel
? `winner=${params.executionTrace.winnerModel}${thinking ? ` 🧠 ${thinking}` : ""}`
: undefined,
typeof params.executionTrace?.fallbackUsed === "boolean"
? `fallback=${params.executionTrace.fallbackUsed ? "yes" : "no"}`
: undefined,
typeof params.executionTrace?.attempts?.length === "number"
? `attempts=${params.executionTrace.attempts.length.toLocaleString()}`
: undefined,
params.completion?.stopReason ? `stop=${params.completion.stopReason}` : undefined,
(() => {
const prompt = formatSummaryPromptValue({
contextLimit: params.contextLimit,
promptTokens: params.promptTokens,
});
return prompt ? `prompt=${prompt}` : undefined;
})(),
typeof params.usage?.input === "number" && params.usage.input > 0
? `⬇️ ${formatTokenCount(params.usage.input)}`
: undefined,
typeof params.usage?.output === "number" && params.usage.output > 0
? `⬆️ ${formatTokenCount(params.usage.output)}`
: undefined,
typeof params.usage?.cacheRead === "number" && params.usage.cacheRead > 0
? `♻️ ${formatTokenCount(params.usage.cacheRead)}`
: undefined,
typeof params.usage?.cacheWrite === "number" && params.usage.cacheWrite > 0
? `🆕 ${formatTokenCount(params.usage.cacheWrite)}`
: undefined,
typeof params.usage?.total === "number" && params.usage.total > 0
? `🔢 ${formatTokenCount(params.usage.total)}`
: undefined,
typeof params.toolSummary?.calls === "number" && params.toolSummary.calls > 0
? `tools=${params.toolSummary.calls.toLocaleString()}`
: undefined,
typeof params.contextManagement?.lastTurnCompactions === "number" &&
params.contextManagement.lastTurnCompactions > 0
? `compactions=${params.contextManagement.lastTurnCompactions.toLocaleString()}`
: undefined,
].filter((value): value is string => Boolean(value));
return fields.length > 0 ? `Summary: ${fields.join(" ")}` : undefined;
}
export function buildInlineRawTracePayload(params: {
entry: SessionEntry | undefined;
rawUserText?: string;
rawAssistantText?: string;
sessionUsage?: {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
};
usage?: {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
};
lastCallUsage?: {
input?: number;
output?: number;
cacheRead?: number;
cacheWrite?: number;
total?: number;
};
provider?: string;
model?: string;
contextLimit?: number;
promptTokens?: number;
executionTrace?: TraceExecutionView;
requestShaping?: {
authMode?: string;
thinking?: string;
reasoning?: string;
verbose?: string;
trace?: string;
fallbackEligible?: boolean;
blockStreaming?: string;
};
promptSegments?: TracePromptSegmentView[];
toolSummary?: TraceToolSummaryView;
completion?: TraceCompletionView;
contextManagement?: TraceContextManagementView;
}): ReplyPayload | undefined {
if (params.entry?.traceLevel !== "raw") {
return undefined;
}
const resolvedPromptTokens = deriveContextPromptTokens({
lastCallUsage: params.lastCallUsage,
promptTokens: params.promptTokens,
usage: params.usage,
});
const requestContextBlock = formatRequestContextTraceBlock({
provider: params.provider,
model: params.model,
contextLimit: params.contextLimit,
promptTokens: resolvedPromptTokens,
});
const usageBlocks = [
formatUsageTraceBlock("Usage (Session Total)", params.sessionUsage),
formatUsageTraceBlock("Usage (Last Turn Total)", params.usage),
requestContextBlock,
formatExecutionResultTraceBlock(params.executionTrace),
formatFallbackChainTraceBlock(params.executionTrace),
formatKeyValueTraceBlock("Request Shaping", [
["provider", params.provider],
["model", params.model],
["auth", params.requestShaping?.authMode],
["thinking", params.requestShaping?.thinking],
["reasoning", params.requestShaping?.reasoning],
["verbose", params.requestShaping?.verbose],
["trace", params.requestShaping?.trace],
["fallbackEligible", params.requestShaping?.fallbackEligible],
["blockStreaming", params.requestShaping?.blockStreaming],
]),
formatPromptSegmentsTraceBlock(params.promptSegments, params.rawUserText),
formatToolSummaryTraceBlock(params.toolSummary),
formatCompletionTraceBlock(params.completion),
formatContextManagementTraceBlock(params.contextManagement),
].filter((value): value is string => Boolean(value));
return {
text: [
...usageBlocks,
formatRawTraceBlock("Model Input (User Role)", params.rawUserText),
formatRawTraceBlock("Model Output (Assistant Role)", params.rawAssistantText),
formatRawTraceSummaryLine({
executionTrace: params.executionTrace,
completion: params.completion,
contextLimit: params.contextLimit,
promptTokens: resolvedPromptTokens,
usage: params.usage,
toolSummary: params.toolSummary,
contextManagement: params.contextManagement,
requestShaping: params.requestShaping,
}),
].join("\n\n\n"),
};
}
File diff suppressed because it is too large Load Diff