diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 556a70ca7d93..d37e494f170a 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -385,11 +385,8 @@ src/agents/embedded-agent-runner/tool-result-truncation.test.ts src/agents/embedded-agent-runner/tool-result-truncation.ts src/agents/embedded-agent-runner/transcript-file-state.test.ts src/agents/embedded-agent-subscribe.handlers.lifecycle.test.ts -src/agents/embedded-agent-subscribe.handlers.messages.test.ts -src/agents/embedded-agent-subscribe.handlers.messages.ts src/agents/embedded-agent-subscribe.handlers.tools.test.ts src/agents/embedded-agent-subscribe.subscribe-embedded-agent-session.subscribeembeddedagentsession.test.ts -src/agents/embedded-agent-subscribe.tools.ts src/agents/embedded-agent-subscribe.ts src/agents/failover-error.test.ts src/agents/failover-error.ts diff --git a/src/agents/cli-runner/execute-events.ts b/src/agents/cli-runner/execute-events.ts index 1dbdbac840c0..ce8346b74863 100644 --- a/src/agents/cli-runner/execute-events.ts +++ b/src/agents/cli-runner/execute-events.ts @@ -7,7 +7,7 @@ import type { CliToolUseStartDelta, } from "../cli-output-contracts.js"; import type { ToolSummaryTrace } from "../embedded-agent-runner/types.js"; -import { sanitizeToolArgs, sanitizeToolResult } from "../embedded-agent-subscribe.tools.js"; +import { sanitizeToolArgs, sanitizeToolResult } from "../embedded-agent-tool-results.js"; import { applyPluginTextReplacements } from "../plugin-text-transforms.js"; import { resolveCliToolTerminalReason } from "../run-termination.js"; import type { CliToolTracking } from "./execute-tool-tracking.js"; diff --git a/src/agents/cli-runner/execute-messaging.ts b/src/agents/cli-runner/execute-messaging.ts index 1919713aa527..8b3bc17c1f9c 100644 --- a/src/agents/cli-runner/execute-messaging.ts +++ b/src/agents/cli-runner/execute-messaging.ts @@ -1,11 +1,11 @@ import crypto from "node:crypto"; +import { extractMessagingToolSend } from "../embedded-agent-messaging-extraction.js"; import { isMessagingToolTargetEvidenceAction } from "../embedded-agent-messaging.js"; import type { MessagingToolSend } from "../embedded-agent-messaging.types.js"; import { collectMessagingMediaUrlsFromRecord, collectMessagingMediaUrlsFromToolResult, - extractMessagingToolSend, -} from "../embedded-agent-subscribe.tools.js"; +} from "../embedded-agent-tool-media.js"; import { stripOpenClawMcpToolPrefix } from "./tool-policy.js"; import type { PreparedCliRunContext } from "./types.js"; diff --git a/src/agents/cli-runner/execute-tool-tracking.ts b/src/agents/cli-runner/execute-tool-tracking.ts index eea2313d7ab5..28a41cdf8a07 100644 --- a/src/agents/cli-runner/execute-tool-tracking.ts +++ b/src/agents/cli-runner/execute-tool-tracking.ts @@ -13,6 +13,10 @@ import { isDeliveredMessagingToolResult, resolveMessageToolSourceReplyFinal, } from "../embedded-agent-message-tool-source-reply.js"; +import { + extractMessagingToolSendResult, + extractMessagingToolSourceReplyPayload, +} from "../embedded-agent-messaging-extraction.js"; import { isMessagingTool, isMessagingToolDeliveryAction, @@ -22,10 +26,6 @@ import type { MessagingToolSend, MessagingToolSourceReplyPayload, } from "../embedded-agent-messaging.types.js"; -import { - extractMessagingToolSendResult, - extractMessagingToolSourceReplyPayload, -} from "../embedded-agent-subscribe.tools.js"; import { closeClaudeSession } from "./claude-live-registry.js"; import { attachCliMessagingDeliveryEvidence } from "./delivery-evidence.js"; import { diff --git a/src/agents/embedded-agent-message-tool-source-reply.ts b/src/agents/embedded-agent-message-tool-source-reply.ts index 83e117ec7d84..c0e18b3df837 100644 --- a/src/agents/embedded-agent-message-tool-source-reply.ts +++ b/src/agents/embedded-agent-message-tool-source-reply.ts @@ -10,8 +10,8 @@ import { isMessageToolSendActionName, isMessagingToolDeliveryAction, } from "./embedded-agent-messaging.js"; -import { isToolResultError } from "./embedded-agent-subscribe.tools.js"; import { normalizeToolPolicyName } from "./tool-policy.js"; +import { isToolResultError } from "./tool-result-error.js"; const MESSAGE_TOOL_NAME = "message"; const SESSIONS_SEND_TOOL_NAME = "sessions_send"; diff --git a/src/agents/embedded-agent-messaging-extraction.ts b/src/agents/embedded-agent-messaging-extraction.ts new file mode 100644 index 000000000000..4498dd2bd488 --- /dev/null +++ b/src/agents/embedded-agent-messaging-extraction.ts @@ -0,0 +1,349 @@ +/** Extracts message delivery evidence from embedded-agent tool calls and results. */ +import { asOptionalRecord as readRecord } from "@openclaw/normalization-core/record-coerce"; +import { + normalizeOptionalLowercaseString, + normalizeOptionalString, + normalizeOptionalStringifiedId, + readStringValue, +} from "@openclaw/normalization-core/string-coerce"; +import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import { getChannelPlugin, normalizeChannelId } from "../channels/plugins/index.js"; +import type { ChannelMessageActionName } from "../channels/plugins/types.public.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { normalizeTargetForProvider } from "../infra/outbound/target-normalization.js"; +import { + normalizeLegacyInteractiveReply, + normalizeMessagePresentation, +} from "../interactive/payload.js"; +import { isMessagingToolTargetEvidenceAction } from "./embedded-agent-messaging.js"; +import type { + MessagingToolSend, + MessagingToolSourceReplyPayload, +} from "./embedded-agent-messaging.types.js"; +import { readToolResultDetails } from "./tool-result-error.js"; + +export function extractMessagingToolSourceReplyPayload( + result: unknown, +): MessagingToolSourceReplyPayload | undefined { + const details = readToolResultDetails(result); + if (!details || details.sourceReplySink !== "internal-ui") { + return undefined; + } + const status = normalizeOptionalLowercaseString(details.deliveryStatus); + if (status && status !== "sent") { + return undefined; + } + const sourceReply = readRecord(details.sourceReply) ?? details; + const payload: MessagingToolSourceReplyPayload = {}; + const text = readStringValue(sourceReply.text) ?? readStringValue(details.message); + if (text) { + payload.text = text; + } + const mediaUrl = readStringValue(sourceReply.mediaUrl) ?? readStringValue(details.mediaUrl); + if (mediaUrl) { + payload.mediaUrl = mediaUrl; + } + const rawMediaUrls = Array.isArray(sourceReply.mediaUrls) + ? sourceReply.mediaUrls + : Array.isArray(details.mediaUrls) + ? details.mediaUrls + : []; + const mediaUrls = uniqueStrings( + rawMediaUrls.filter((value): value is string => typeof value === "string"), + ); + if (mediaUrls.length > 0) { + payload.mediaUrls = mediaUrls; + } + if (sourceReply.audioAsVoice === true || details.audioAsVoice === true) { + payload.audioAsVoice = true; + } + const presentation = normalizeMessagePresentation(sourceReply.presentation); + if (presentation) { + payload.presentation = presentation; + } + const interactive = normalizeLegacyInteractiveReply(sourceReply.interactive); + if (interactive) { + payload.interactive = interactive; + } + const channelData = readRecord(sourceReply.channelData); + if (channelData) { + payload.channelData = { ...channelData }; + } + const idempotencyKey = + readStringValue(sourceReply.idempotencyKey) ?? readStringValue(details.idempotencyKey); + if (idempotencyKey) { + payload.idempotencyKey = idempotencyKey; + } + return Object.keys(payload).length > 0 ? payload : undefined; +} + +// Core tool names that are allowed to emit trusted local media artifacts. +// Plugin tools must be explicitly passed as trusted run-local names by the caller. + +function resolveMessageToolTarget(params: { + action: string; + args: Record; + providerId: string | null; + currentChannelId?: string; + currentMessagingTarget?: string; +}): string | undefined { + const directTarget = + normalizeOptionalString(params.args.target) ?? + normalizeOptionalString(params.args.to) ?? + normalizeOptionalString(params.args.channelId); + if (directTarget) { + return directTarget; + } + const aliases = params.providerId + ? getChannelPlugin(params.providerId)?.actions?.messageActionTargetAliases?.[ + params.action as ChannelMessageActionName + ]?.deliveryTargetAliases + : undefined; + for (const alias of aliases ?? []) { + const aliasTarget = normalizeOptionalStringifiedId(params.args[alias]); + if (aliasTarget) { + return aliasTarget; + } + } + return params.currentMessagingTarget ?? params.currentChannelId; +} + +function resolveMessagingToolThreadEvidence(params: { + providerId: string; + to: string; + accountId?: string; + threadId?: string; + replyToId?: string; + allowImplicitThread: boolean; + threadSuppressed: boolean; + options?: { + config?: OpenClawConfig; + currentChannelId?: string; + currentMessagingTarget?: string; + currentThreadId?: string; + currentMessageId?: string | number; + replyToMode?: "off" | "first" | "all" | "batched"; + hasRepliedRef?: { value: boolean }; + }; +}): Pick { + const threading = getChannelPlugin(params.providerId)?.threading; + const autoThreadResolver = params.allowImplicitThread + ? threading?.resolveAutoThreadId + : undefined; + const replyTransport = params.replyToId + ? threading?.resolveReplyTransport?.({ + cfg: params.options?.config ?? {}, + accountId: params.accountId, + threadId: params.threadId, + replyToId: params.replyToId, + }) + : undefined; + const transportThreadId = normalizeOptionalStringifiedId(replyTransport?.threadId); + const replyToThreadId = + replyTransport?.threadId === null + ? normalizeOptionalString(replyTransport.replyToId) + : undefined; + const explicitThreadId = transportThreadId ?? replyToThreadId ?? params.threadId; + const currentChannelId = normalizeOptionalString(params.options?.currentChannelId); + const currentMessagingTarget = normalizeOptionalString(params.options?.currentMessagingTarget); + const currentThreadId = normalizeOptionalString(params.options?.currentThreadId); + const replyToMode = params.options?.replyToMode ?? (currentThreadId ? "all" : undefined); + const canResolveCurrentThread = Boolean( + (currentChannelId || currentMessagingTarget) && currentThreadId, + ); + const resolvedCurrentThreadId = + !explicitThreadId && !params.threadSuppressed && autoThreadResolver && canResolveCurrentThread + ? autoThreadResolver({ + cfg: params.options?.config ?? {}, + accountId: params.accountId, + to: params.to, + replyToId: params.replyToId, + toolContext: { + currentChannelId, + currentMessagingTarget, + currentThreadTs: currentThreadId, + currentMessageId: params.options?.currentMessageId, + replyToMode, + hasRepliedRef: params.options?.hasRepliedRef, + }, + }) + : undefined; + const threadImplicit = + !explicitThreadId && + !params.threadSuppressed && + Boolean(autoThreadResolver) && + (!canResolveCurrentThread || Boolean(resolvedCurrentThreadId)); + return { + ...((explicitThreadId ?? resolvedCurrentThreadId) + ? { threadId: explicitThreadId ?? resolvedCurrentThreadId } + : {}), + ...(threadImplicit ? { threadImplicit: true } : {}), + ...(params.threadSuppressed ? { threadSuppressed: true } : {}), + }; +} + +export function extractMessagingToolSend( + toolName: string, + args: Record, + options?: { + config?: OpenClawConfig; + currentChannelId?: string; + currentMessagingTarget?: string; + currentThreadId?: string; + currentMessageId?: string | number; + replyToMode?: "off" | "first" | "all" | "batched"; + hasRepliedRef?: { value: boolean }; + }, +): MessagingToolSend | undefined { + // Provider docking: new provider tools must implement plugin.actions.extractToolSend. + const action = normalizeOptionalString(args.action) ?? ""; + const accountId = normalizeOptionalString(args.accountId); + if (toolName === "conversations_send" || toolName === "conversations_turn") { + const conversationRef = normalizeOptionalString(args.conversationRef); + return conversationRef + ? { + tool: toolName, + provider: "conversation", + to: conversationRef, + } + : undefined; + } + if (toolName === "message") { + if (!isMessagingToolTargetEvidenceAction(toolName, args)) { + return undefined; + } + const providerRaw = normalizeOptionalString(args.provider) ?? ""; + const channelRaw = normalizeOptionalString(args.channel) ?? ""; + const providerHint = providerRaw || channelRaw; + const providerId = providerHint ? normalizeChannelId(providerHint) : null; + const toRaw = resolveMessageToolTarget({ + action, + args, + providerId, + currentChannelId: options?.currentChannelId, + currentMessagingTarget: options?.currentMessagingTarget, + }); + if (!toRaw) { + return undefined; + } + const provider = providerId ?? normalizeOptionalLowercaseString(providerHint) ?? "message"; + const to = normalizeTargetForProvider(provider, toRaw); + const pluginExtractionArgs = { ...args, to: toRaw }; + const pluginExtracted = providerId + ? getChannelPlugin(providerId)?.actions?.extractToolSend?.({ args: pluginExtractionArgs }) + : null; + const resolvedAccountId = normalizeOptionalString(pluginExtracted?.accountId) ?? accountId; + const threadId = + normalizeOptionalString(pluginExtracted?.threadId) ?? normalizeOptionalString(args.threadId); + const replyToId = normalizeOptionalString(args.replyTo); + // Normal sends use prepared core delivery, where provider transport owns + // reply/thread precedence. Other send-like actions use plugin dispatch. + const outboundReplyToId = action === "send" ? replyToId : undefined; + const threadSuppressed = + pluginExtracted?.threadSuppressed === true || + args.topLevel === true || + args.threadId === null; + return to + ? { + tool: toolName, + provider, + accountId: resolvedAccountId, + to, + ...(providerId + ? resolveMessagingToolThreadEvidence({ + providerId, + to, + accountId: resolvedAccountId, + threadId, + replyToId: outboundReplyToId, + allowImplicitThread: pluginExtracted + ? pluginExtracted.threadImplicit === true + : true, + threadSuppressed, + options, + }) + : { + ...(threadId ? { threadId } : {}), + ...(threadSuppressed ? { threadSuppressed: true } : {}), + }), + } + : undefined; + } + + const providerId = normalizeChannelId(toolName); + if (!providerId) { + return undefined; + } + const plugin = getChannelPlugin(providerId); + const extracted = plugin?.actions?.extractToolSend?.({ args }); + if (!extracted?.to) { + return undefined; + } + const to = normalizeTargetForProvider(providerId, extracted.to); + const threadId = normalizeOptionalString(extracted.threadId); + const threadSuppressed = extracted.threadSuppressed === true; + const extractedAccountId = normalizeOptionalString(extracted.accountId) ?? accountId; + const nativeReplyToMode = options?.replyToMode; + const nativeSingleUseMode = nativeReplyToMode === "first" || nativeReplyToMode === "batched"; + const canResolveNativeImplicitThread = + extracted.threadImplicit === true && + nativeReplyToMode !== undefined && + (!nativeSingleUseMode || options?.hasRepliedRef !== undefined); + return to + ? { + tool: toolName, + provider: providerId, + accountId: extractedAccountId, + to, + ...resolveMessagingToolThreadEvidence({ + providerId, + to, + accountId: extractedAccountId, + threadId, + allowImplicitThread: canResolveNativeImplicitThread, + threadSuppressed, + options, + }), + } + : undefined; +} + +/** Reconciles pending send evidence with the provider's successful action result. */ +export function extractMessagingToolSendResult( + pending: MessagingToolSend, + result: unknown, +): MessagingToolSend { + const providerId = normalizeChannelId(pending.provider); + const extracted = providerId + ? getChannelPlugin(providerId)?.actions?.extractToolSendResult?.({ + result, + send: { + to: pending.to ?? "", + accountId: pending.accountId, + threadId: pending.threadId, + threadImplicit: pending.threadImplicit, + threadSuppressed: pending.threadSuppressed, + }, + }) + : null; + if (!extracted?.to) { + return pending; + } + const extractedThreadId = normalizeOptionalString(extracted.threadId); + const providerReportedThread = + extractedThreadId != null || + extracted.threadImplicit === true || + extracted.threadSuppressed === true; + // Thread route fields are one state. Mixing provider and pending values can + // create contradictory implicit and suppressed evidence. + const threadEvidence = providerReportedThread ? extracted : pending; + return { + ...pending, + ...extracted, + accountId: normalizeOptionalString(extracted.accountId) ?? pending.accountId, + to: normalizeTargetForProvider(providerId ?? pending.provider, extracted.to), + threadId: normalizeOptionalString(threadEvidence.threadId), + threadImplicit: threadEvidence.threadImplicit === true ? true : undefined, + threadSuppressed: threadEvidence.threadSuppressed === true ? true : undefined, + }; +} diff --git a/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts b/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts index 2e1b1d658584..ecf6f279dea5 100644 --- a/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts +++ b/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts @@ -23,7 +23,7 @@ import { runBestEffortCallback } from "./embedded-agent-subscribe.callback.js"; import { consumePendingToolMediaReply, hasAssistantVisibleReply, -} from "./embedded-agent-subscribe.handlers.messages.js"; +} from "./embedded-agent-subscribe.handlers.messages.replies.js"; import type { EmbeddedAgentSubscribeContext } from "./embedded-agent-subscribe.handlers.types.js"; import { isAssistantMessage } from "./embedded-agent-utils.js"; import type { AgentSessionEvent } from "./sessions/index.js"; diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.lifecycle.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.lifecycle.test.ts new file mode 100644 index 000000000000..6cc65af16da7 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.lifecycle.test.ts @@ -0,0 +1,699 @@ +import { describe, expect, it, vi } from "vitest"; +import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js"; +import { + createMessageEndContext, + createMessageToolEnvelope, + endMessage, + firstMockCall, + firstMockArg, +} from "./embedded-agent-subscribe.handlers.messages.test-helpers.js"; +import { createOpenAiResponsesTextBlock } from "./embedded-agent-subscribe.openai-responses.test-helpers.js"; + +describe("handleMessageEnd", () => { + it.each(["answer part A msg [[E1008]timeout] answer part B", "answer ending ["])( + "keeps malformed directive-looking final text identical across delivery paths: %s", + (text) => { + const onAgentEvent = vi.fn(); + const emitBlockReply = vi.fn(); + const flushBlockReplyBuffer = vi.fn(); + const accumulator = createStreamingDirectiveAccumulator(); + const streamed = accumulator.consume(text)?.text ?? ""; + const ctx = createMessageEndContext({ + onAgentEvent, + emitBlockReply, + flushBlockReplyBuffer, + consumeReplyDirectives: vi.fn((chunk: string, options?: { final?: boolean }) => + accumulator.consume(chunk, options), + ), + blockChunker: { + hasBuffered: () => true, + reset: vi.fn(), + }, + state: { + blockBuffer: streamed, + deltaBuffer: streamed, + }, + }); + + void endMessage(ctx, { + message: { role: "assistant", content: [{ type: "text", text }] }, + }); + + expect(firstMockArg(onAgentEvent, "agent event")).toMatchObject({ + stream: "assistant", + data: { text, delta: text }, + }); + const finalBlockText = (firstMockArg(emitBlockReply, "block reply") as { text?: string }) + .text; + expect(`${streamed}${finalBlockText ?? ""}`).toBe(text); + expect(ctx.finalizeAssistantTexts).toHaveBeenCalledWith(expect.objectContaining({ text })); + }, + ); + + it("keeps exact NO_REPLY silent after a user-facing message send followed by sessions_send (#119383)", () => { + const emitBlockReply = vi.fn(); + const finalizeAssistantTexts = vi.fn(); + const ctx = createMessageEndContext({ + emitBlockReply, + finalizeAssistantTexts, + consumeReplyDirectives: vi.fn((text: string) => ({ text })), + state: { + blockBuffer: "", + deltaBuffer: "", + messagingToolSentTexts: ["", ""], + messagingToolSentTextsNormalized: ["", ""], + messagingToolSentTargets: [ + { + tool: "message", + provider: "whatsapp", + to: "user:123", + text: "", + }, + ], + }, + }); + + void endMessage(ctx, { + message: { role: "assistant", content: [{ type: "text", text: "NO_REPLY" }] }, + }); + + // The exact silent token must never be rewritten to the sessions_send body: + // the final assistant text keeps NO_REPLY and no block reply carries the note. + expect(finalizeAssistantTexts).toHaveBeenCalledWith( + expect.objectContaining({ text: "NO_REPLY" }), + ); + for (const call of emitBlockReply.mock.calls) { + expect(JSON.stringify(call)).not.toContain(""); + } + }); + + it("keeps exact NO_REPLY silent when only sessions_send delivered (#119383)", () => { + const emitBlockReply = vi.fn(); + const finalizeAssistantTexts = vi.fn(); + const ctx = createMessageEndContext({ + emitBlockReply, + finalizeAssistantTexts, + consumeReplyDirectives: vi.fn((text: string) => ({ text })), + state: { + blockBuffer: "", + deltaBuffer: "", + messagingToolSentTexts: [""], + messagingToolSentTextsNormalized: [""], + messagingToolSentTargets: [], + }, + }); + + void endMessage(ctx, { + message: { role: "assistant", content: [{ type: "text", text: "NO_REPLY" }] }, + }); + + expect(finalizeAssistantTexts).toHaveBeenCalledWith( + expect.objectContaining({ text: "NO_REPLY" }), + ); + for (const call of emitBlockReply.mock.calls) { + expect(JSON.stringify(call)).not.toContain(""); + } + }); + + it.each([ + { + name: "counts a completed provider assistant message", + message: { role: "assistant", content: [{ type: "text", text: "Done." }] }, + expected: 1, + }, + { + name: "ignores transcript-only mirrored assistant messages", + message: { + role: "assistant", + provider: "openclaw", + model: "delivery-mirror", + content: [{ type: "text", text: "Done." }], + }, + expected: 0, + }, + { + name: "ignores non-assistant messages", + message: { role: "user", content: [{ type: "text", text: "hi" }] }, + expected: 0, + }, + ])("$name for assistantTurnCount", ({ message, expected }) => { + const ctx = createMessageEndContext({ state: { assistantTurnCount: 0 } }); + + void endMessage(ctx, { message }); + + expect(ctx.state.assistantTurnCount).toBe(expected); + }); + + it("keeps duplicate-reply diagnostics free of lone surrogates", () => { + const text = `${"a".repeat(49)}😀tail`; + const ctx = createMessageEndContext({ + consumeReplyDirectives: vi.fn((value: string) => ({ text: value })), + state: { messagingToolSentTextsNormalized: [`${"a".repeat(49)}tail`] }, + }); + + void endMessage(ctx, { + message: { role: "assistant", content: [{ type: "text", text }] }, + }); + + const diagnostic = (ctx.log.debug as ReturnType).mock.calls + .flat() + .find((value) => String(value).startsWith("Skipping message_end block reply")); + expect(diagnostic).toEqual(expect.any(String)); + expect(Buffer.from(String(diagnostic)).toString()).toBe(diagnostic); + }); + + it("persists streamed usage when the final assistant snapshot is zeroed", () => { + const ctx = createMessageEndContext({ + state: { + pendingAssistantUsage: { input: 7, output: 5, reasoningTokens: 2, total: 12 }, + }, + }); + const message = { + role: "assistant", + api: "openai-completions", + content: [{ type: "text", text: "Done." }], + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + }, + }; + + void endMessage(ctx, { + message, + }); + + expect(firstMockArg(ctx.noteLastAssistant as never, "last assistant")).toMatchObject({ + usage: { + input: 7, + output: 5, + cacheRead: 0, + cacheWrite: 0, + reasoningTokens: 2, + totalTokens: 12, + }, + }); + expect(ctx.recordAssistantUsage).toHaveBeenCalledWith( + expect.objectContaining({ + input: 7, + output: 5, + reasoningTokens: 2, + totalTokens: 12, + }), + ); + }); + + it("keeps authoritative final usage instead of pending stream usage", () => { + const ctx = createMessageEndContext({ + state: { + pendingAssistantUsage: { input: 7, output: 5, total: 12 }, + }, + }); + const message = { + role: "assistant", + content: [{ type: "text", text: "Done." }], + usage: { + input: 11, + output: 3, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 14, + }, + }; + + void endMessage(ctx, { + message, + }); + + expect(firstMockArg(ctx.noteLastAssistant as never, "last assistant")).toBe(message); + expect(ctx.recordAssistantUsage).toHaveBeenCalledWith(message.usage); + }); + + it("warns when assistant text only pretends to call a registered tool", () => { + const warn = vi.fn(); + const ctx = createMessageEndContext({ + warn, + builtinToolNames: new Set(["read"]), + }); + + void endMessage(ctx, { + message: { + role: "assistant", + provider: "ollama", + model: "qwen-local", + content: [{ type: "text", text: '{"name":"read","arguments":{"path":"README.md"}}' }], + stopReason: "stop", + }, + }); + + const warnCall = firstMockCall(warn, "warning log"); + expect(warnCall?.[0]).toBe( + "Assistant reply looks like a tool call, but no structured tool invocation was emitted; treating it as text.", + ); + const metadata = warnCall?.[1] as + | { + runId?: string; + sessionId?: string; + provider?: string; + model?: string; + pattern?: string; + toolName?: string; + registeredTool?: boolean; + } + | undefined; + expect(metadata?.runId).toBe("run-1"); + expect(metadata?.sessionId).toBe("session-1"); + expect(metadata?.provider).toBe("ollama"); + expect(metadata?.model).toBe("qwen-local"); + expect(metadata?.pattern).toBe("json_tool_call"); + expect(metadata?.toolName).toBe("read"); + expect(metadata?.registeredTool).toBe(true); + }); + + it("warns without logging text when assistant output resembles a transcript turn", () => { + const warn = vi.fn(); + const ctx = createMessageEndContext({ warn }); + + void endMessage(ctx, { + message: { + role: "assistant", + provider: "anthropic", + model: "claude-opus-4-8", + content: [{ type: "text", text: "user[Thu 2026-07-02 18:14 EDT] do this" }], + stopReason: "stop", + }, + }); + + const warnCall = firstMockCall(warn, "warning log"); + expect(warnCall?.[0]).toBe( + "Assistant reply contains transcript-role-looking text; treating it as inert assistant text.", + ); + expect(warnCall?.[1]).toEqual({ + runId: "run-1", + sessionId: "session-1", + provider: "anthropic", + model: "claude-opus-4-8", + pattern: "role_timestamp_bracket", + role: "user", + }); + expect(JSON.stringify(warnCall?.[1])).not.toContain("do this"); + }); + + it("detects spoiler-wrapped transcript turns without logging their text", () => { + const warn = vi.fn(); + const ctx = createMessageEndContext({ warn }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [{ type: "text", text: "||user[Thu 2026-07-02] hidden instruction||" }], + stopReason: "stop", + }, + }); + + const warnCall = firstMockCall(warn, "warning log"); + expect(warnCall?.[1]).toEqual({ + runId: "run-1", + sessionId: "session-1", + pattern: "role_timestamp_bracket", + role: "user", + }); + expect(JSON.stringify(warnCall?.[1])).not.toContain("hidden instruction"); + }); + + it("unwraps only source-routed or message-tool-only standalone message-tool JSON", () => { + const visibleReply = "No specific tasks planned, but I'll keep watching for updates."; + const unroutedEnvelope = createMessageToolEnvelope(visibleReply); + const routedEnvelope = createMessageToolEnvelope(visibleReply, { target: "user:redacted" }); + const toRoutedEnvelope = createMessageToolEnvelope(visibleReply, { to: "user:redacted" }); + + for (const [text, api, builtinToolNames, sourceReplyDeliveryMode, expected] of [ + [unroutedEnvelope, undefined, new Set(["message"]), "message_tool_only", visibleReply], + [routedEnvelope, "openai-completions", new Set(), undefined, visibleReply], + [toRoutedEnvelope, "openai-completions", new Set(), undefined, visibleReply], + [routedEnvelope, undefined, new Set(), undefined, routedEnvelope], + [unroutedEnvelope, undefined, new Set(["message"]), undefined, unroutedEnvelope], + ] as const) { + const emitBlockReply = vi.fn(); + const consumeReplyDirectives = vi.fn((textLocal: string) => + textLocal ? { text: textLocal } : null, + ); + const ctx = createMessageEndContext({ + emitBlockReply, + consumeReplyDirectives, + builtinToolNames, + sourceReplyDeliveryMode, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + ...(api ? { api } : {}), + content: [{ type: "text", text }], + }, + }); + + expect(consumeReplyDirectives).toHaveBeenCalledWith(expected, { final: true }); + expect(firstMockArg(emitBlockReply, "block reply")).toMatchObject({ text: expected }); + } + }); + + it("does not warn when the assistant emitted a structured tool call", () => { + const warn = vi.fn(); + const ctx = createMessageEndContext({ + warn, + builtinToolNames: new Set(["read"]), + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "read", arguments: {} }], + stopReason: "toolUse", + }, + }); + + expect(warn).not.toHaveBeenCalled(); + }); + + it("suppresses commentary-phase replies from user-visible output", () => { + const onAgentEvent = vi.fn(); + const emitBlockReply = vi.fn(); + const finalizeAssistantTexts = vi.fn(); + const ctx = createMessageEndContext({ + onAgentEvent, + finalizeAssistantTexts, + emitBlockReply, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + phase: "commentary", + content: [{ type: "text", text: "Need send." }], + usage: { input: 1, output: 1, total: 2 }, + }, + }); + + // Archive-always: commentary reaches the bus/archive but not the visible reply. + expect(onAgentEvent).toHaveBeenCalled(); + expect(emitBlockReply).not.toHaveBeenCalled(); + expect(finalizeAssistantTexts).not.toHaveBeenCalled(); + }); + + it("suppresses commentary message_end when phase exists only in textSignature metadata", () => { + const onAgentEvent = vi.fn(); + const emitBlockReply = vi.fn(); + const finalizeAssistantTexts = vi.fn(); + const ctx = createMessageEndContext({ + onAgentEvent, + finalizeAssistantTexts, + emitBlockReply, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [ + createOpenAiResponsesTextBlock({ + text: "Need send.", + id: "msg_sig", + phase: "commentary", + }), + ], + usage: { input: 1, output: 1, total: 2 }, + }, + }); + + // Archive-always: commentary (textSignature-only phase) reaches the + // bus/archive but not the visible reply. + expect(onAgentEvent).toHaveBeenCalled(); + expect(emitBlockReply).not.toHaveBeenCalled(); + expect(finalizeAssistantTexts).not.toHaveBeenCalled(); + }); + + it("does not duplicate block reply for text_end channels when text was already delivered", () => { + const onBlockReply = vi.fn(); + const emitBlockReply = vi.fn(); + // In real usage, the directive accumulator returns null for empty/consumed + // input. The non-empty call shouldn't happen for text_end channels (that's + // the safety send we're guarding against). + const consumeReplyDirectives = vi.fn((text: string) => (text ? { text } : null)); + const ctx = createMessageEndContext({ + onBlockReply, + emitBlockReply, + consumeReplyDirectives, + state: { + emittedAssistantUpdate: true, + lastStreamedAssistantCleaned: "Hello world", + blockReplyBreak: "text_end", + // Simulate text_end already delivered this text through emitBlockChunk + lastBlockReplyText: "Hello world", + deltaBuffer: "", + blockBuffer: "", + }, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [{ type: "text", text: "Hello world" }], + usage: { input: 10, output: 5, total: 15 }, + }, + }); + + // The block reply should NOT fire again since text_end already delivered it. + // consumeReplyDirectives is called once with "" (the final flush for + // text_end channels) but returns null, so emitBlockReply is never called. + expect(emitBlockReply).not.toHaveBeenCalled(); + }); + + it("tags message-end safety replies with the current assistant message", () => { + const emitBlockReply = vi.fn(); + const ctx = createMessageEndContext({ + onBlockReply: vi.fn(), + emitBlockReply, + consumeReplyDirectives: vi.fn((text: string) => (text ? { text } : null)), + state: { + assistantMessageIndex: 7, + blockReplyBreak: "text_end", + lastBlockReplyText: null, + }, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [{ type: "text", text: "Final answer" }], + usage: { input: 10, output: 5, total: 15 }, + }, + }); + + expect(emitBlockReply).toHaveBeenCalledWith( + { text: "Final answer" }, + { assistantMessageIndex: 7 }, + ); + }); + + it("does not duplicate block reply for text_end channels even when stripping differs", () => { + const onBlockReply = vi.fn(); + const emitBlockReply = vi.fn(); + // Same pattern: directive accumulator returns null for empty final flush + const consumeReplyDirectives = vi.fn((text: string) => (text ? { text } : null)); + const ctx = createMessageEndContext({ + onBlockReply, + emitBlockReply, + consumeReplyDirectives, + state: { + emittedAssistantUpdate: true, + lastStreamedAssistantCleaned: "Hello world", + blockReplyBreak: "text_end", + // text_end delivered via emitBlockChunk which uses different stripping + lastBlockReplyText: "Hello world.", + deltaBuffer: "", + blockBuffer: "", + }, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + // The raw text differs slightly from lastBlockReplyText due to stripping + content: [{ type: "text", text: "Hello world" }], + usage: { input: 10, output: 5, total: 15 }, + }, + }); + + // Even though text !== lastBlockReplyText (different stripping), the safety + // send should NOT fire for text_end channels. The only consumeReplyDirectives + // call is the final empty flush which returns null. + expect(emitBlockReply).not.toHaveBeenCalled(); + }); + + it("emits final media and malformed pending text after flushing buffered message_end text", () => { + const emitBlockReply = vi.fn(); + const flushBlockReplyBuffer = vi.fn(); + const accumulator = createStreamingDirectiveAccumulator(); + const text = "Caption [[oops\nMEDIA:/tmp/final.png"; + const streamed = accumulator.consume(text)?.text ?? ""; + const consumeReplyDirectives = vi.fn((chunk: string, options?: { final?: boolean }) => + accumulator.consume(chunk, options), + ); + const ctx = createMessageEndContext({ + emitBlockReply, + flushBlockReplyBuffer, + consumeReplyDirectives, + blockChunker: { + hasBuffered: () => true, + reset: vi.fn(), + }, + state: { + emittedAssistantUpdate: true, + lastStreamedAssistantCleaned: "Caption [[oops", + blockReplyBreak: "message_end", + deltaBuffer: streamed, + blockBuffer: streamed, + }, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [{ type: "text", text }], + usage: { input: 10, output: 5, total: 15 }, + }, + }); + + expect(flushBlockReplyBuffer).toHaveBeenCalledWith({ + assistantMessageIndex: undefined, + final: true, + }); + expect(consumeReplyDirectives).toHaveBeenCalledWith("", { final: true }); + const finalReply = firstMockArg(emitBlockReply, "block reply") as { + text?: string; + mediaUrls?: string[]; + }; + expect(finalReply).toMatchObject({ + text: " [[oops", + mediaUrls: ["/tmp/final.png"], + }); + expect(`${streamed}${finalReply.text ?? ""}`).toBe("Caption [[oops"); + }); + + it("preserves literal reasoning-looking tags in unphased final visible text", () => { + const onAgentEvent = vi.fn(); + const stripBlockTags = vi.fn(() => "Before"); + const ctx = createMessageEndContext({ + onAgentEvent, + stripBlockTags, + consumeReplyDirectives: vi.fn((text: string) => ({ text })), + state: { + blockBuffer: "", + deltaBuffer: "", + }, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [ + { + type: "text", + text: "Before literal tag text after", + textSignature: JSON.stringify({ v: 1, id: "item_unphased" }), + }, + ], + usage: { input: 10, output: 5, total: 15 }, + }, + }); + + expect(stripBlockTags).not.toHaveBeenCalled(); + expect(firstMockArg(ctx.emitAssistantStreamData as never, "assistant stream")).toMatchObject({ + text: "Before literal tag text after", + delta: "Before literal tag text after", + }); + expect(ctx.finalizeAssistantTexts).toHaveBeenCalledWith( + expect.objectContaining({ text: "Before literal tag text after" }), + ); + }); + + it("keeps final-tag enforcement in message_end fallback", () => { + const onAgentEvent = vi.fn(); + const stripBlockTags = vi.fn(() => ""); + const ctx = createMessageEndContext({ + enforceFinalTag: true, + onAgentEvent, + stripBlockTags, + consumeReplyDirectives: vi.fn((text: string) => ({ text })), + state: { + blockBuffer: "", + deltaBuffer: "", + }, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: "Hello world", + usage: { input: 10, output: 5, total: 15 }, + }, + }); + + expect(stripBlockTags).toHaveBeenCalledWith( + "Hello world", + { thinking: false, final: false }, + { final: true }, + ); + expect(ctx.emitAssistantStreamData).not.toHaveBeenCalled(); + expect(ctx.finalizeAssistantTexts).toHaveBeenCalledWith(expect.objectContaining({ text: "" })); + }); + + it("emits a replacement final assistant event when final_answer appears only at message_end", () => { + const onAgentEvent = vi.fn(); + const ctx = createMessageEndContext({ + onAgentEvent, + state: { + emittedAssistantUpdate: true, + lastStreamedAssistantCleaned: "Working...", + blockReplyBreak: "text_end", + deltaBuffer: "", + blockBuffer: "", + }, + }); + + void endMessage(ctx, { + message: { + role: "assistant", + content: [ + createOpenAiResponsesTextBlock({ + text: "Working...", + id: "item_commentary", + phase: "commentary", + }), + createOpenAiResponsesTextBlock({ + text: "Done.", + id: "item_final", + phase: "final_answer", + }), + ], + stopReason: "stop", + api: "openai-responses", + provider: "openai", + model: "gpt-5.2", + usage: {}, + timestamp: 0, + }, + }); + + expect(onAgentEvent).toHaveBeenCalledTimes(1); + const event = firstMockArg(onAgentEvent, "agent event") as + | { stream?: string; data?: { text?: string; delta?: string; replace?: boolean } } + | undefined; + expect(event?.stream).toBe("assistant"); + expect(event?.data?.text).toBe("Done."); + expect(event?.data?.delta).toBe(""); + expect(event?.data?.replace).toBe(true); + }); +}); diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.lifecycle.ts b/src/agents/embedded-agent-subscribe.handlers.messages.lifecycle.ts new file mode 100644 index 000000000000..cd115c1203f3 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.lifecycle.ts @@ -0,0 +1,525 @@ +import { isPromiseLike } from "@openclaw/normalization-core/promise-like"; +/** + * Handles assistant message lifecycle boundaries, final reconciliation, and usage. + */ +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload"; +import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; +import { parseReplyDirectives } from "../auto-reply/reply/reply-directives.js"; +import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; +import type { AssistantMessage } from "../llm/types.js"; +import { splitMediaFromOutput } from "../media/parse.js"; +import { coerceChatContentText } from "../shared/chat-content.js"; +import { resolveAssistantMessagePhase } from "../shared/chat-message-content.js"; +import { + isMessagingToolDuplicateNormalized, + normalizeTextForComparison, +} from "./embedded-agent-helpers.js"; +import { hasAssistantVisibleReply } from "./embedded-agent-subscribe.handlers.messages.replies.js"; +import { + buildAssistantStreamData, + emitAssistantMessageStart, + extractStandaloneMessageToolText, + hasMessageToolOnlySourceDelivery, + isOpenAiCompletionsAssistantMessage, + isResponsesApiAssistantMessage, + isSubscribeTranscriptOnlyOpenClawAssistantMessage, + scopeAssistantMessageToStreamBlock, + shouldSuppressAssistantVisibleOutput, + shouldSuppressDeterministicApprovalOutput, +} from "./embedded-agent-subscribe.handlers.messages.stream.js"; +import type { EmbeddedAgentSubscribeContext } from "./embedded-agent-subscribe.handlers.types.js"; +import { appendRawStream } from "./embedded-agent-subscribe.raw-stream.js"; +import { warnIfAssistantEmittedSuspiciousText } from "./embedded-agent-subscribe.tool-text-diagnostics.js"; +import { + createThinkingTagStreamState, + extractAssistantCommentaryText, + extractAssistantThinking, + extractAssistantVisibleText, + extractEmbeddedAssistantText, + extractThinkingFromTaggedText, + promoteThinkingTagsToBlocks, +} from "./embedded-agent-utils.js"; +import type { AgentEvent, AgentMessage } from "./runtime/index.js"; +import { + hasNonzeroUsage, + makeZeroUsageSnapshot, + normalizeUsage, + type NormalizedUsage, + type UsageLike, +} from "./usage.js"; + +export function preservePendingAssistantUsage( + message: AssistantMessage, + pendingUsage: NormalizedUsage | undefined, +): AssistantMessage { + if ( + isSubscribeTranscriptOnlyOpenClawAssistantMessage(message) || + !hasNonzeroUsage(pendingUsage) + ) { + return message; + } + const messageUsage = normalizeUsage((message as { usage?: UsageLike }).usage); + if (hasNonzeroUsage(messageUsage)) { + return message; + } + + // Pending usage resets at each assistant-message boundary, so it belongs to + // this final snapshot. Only replace missing/zero usage; provider totals win. + const input = pendingUsage.input ?? 0; + const output = pendingUsage.output ?? 0; + const cacheRead = pendingUsage.cacheRead ?? 0; + const cacheWrite = pendingUsage.cacheWrite ?? 0; + message.usage = { + ...makeZeroUsageSnapshot(), + input, + output, + cacheRead, + cacheWrite, + ...(pendingUsage.contextUsage ? { contextUsage: { ...pendingUsage.contextUsage } } : {}), + totalTokens: pendingUsage.total ?? input + output + cacheRead + cacheWrite, + ...(pendingUsage.reasoningTokens !== undefined + ? { reasoningTokens: pendingUsage.reasoningTokens } + : {}), + }; + return message; +} + +export function capturePendingAssistantUsage( + ctx: EmbeddedAgentSubscribeContext, + evt: AgentEvent & { message: AgentMessage; assistantMessageEvent?: unknown }, +): void { + const msg = evt.message; + if (msg?.role !== "assistant" || isSubscribeTranscriptOnlyOpenClawAssistantMessage(msg)) { + return; + } + const assistantRecord = + evt.assistantMessageEvent && typeof evt.assistantMessageEvent === "object" + ? (evt.assistantMessageEvent as Record) + : undefined; + const evtType = typeof assistantRecord?.type === "string" ? assistantRecord.type : ""; + if (evtType === "text_end" || evtType === "done" || evtType === "error") { + ctx.recordAssistantUsage(assistantRecord); + } +} + +export function resetPendingAssistantUsage( + ctx: EmbeddedAgentSubscribeContext, + message: AgentMessage, +): void { + if (message?.role !== "assistant" || isSubscribeTranscriptOnlyOpenClawAssistantMessage(message)) { + return; + } + ctx.state.pendingAssistantUsage = undefined; + ctx.state.assistantUsageCommitted = false; +} + +export function handleMessageStart( + ctx: EmbeddedAgentSubscribeContext, + evt: AgentEvent & { message: AgentMessage }, +) { + const msg = evt.message; + if (msg?.role !== "assistant" || isSubscribeTranscriptOnlyOpenClawAssistantMessage(msg)) { + return; + } + + // KNOWN: Resetting at `text_end` is unsafe (late/duplicate end events). + // ASSUME: `message_start` is the only reliable boundary for “new assistant message begins”. + // Start-of-message is a safer reset point than message_end: some providers + // may deliver late text_end updates after message_end, which would otherwise + // re-trigger block replies. + ctx.resetAssistantMessageState(ctx.state.assistantTexts.length); + // Use assistant message_start as the earliest "writing" signal for typing. + emitAssistantMessageStart(ctx); +} + +/** Handles assistant message deltas, reasoning, directives, and block replies. */ + +export function handleMessageEnd( + ctx: EmbeddedAgentSubscribeContext, + evt: AgentEvent & { message: AgentMessage }, +): void | Promise { + const msg = evt.message; + if (msg?.role !== "assistant" || isSubscribeTranscriptOnlyOpenClawAssistantMessage(msg)) { + return; + } + + // Transcript-only messages never reach the provider, so this counts exactly + // the completed model round trips consumers see as `assistantTurns`. + ctx.state.assistantTurnCount += 1; + const assistantMessage = preservePendingAssistantUsage(msg, ctx.state.pendingAssistantUsage); + const assistantPhase = resolveAssistantMessagePhase(assistantMessage); + const suppressVisibleAssistantOutput = shouldSuppressAssistantVisibleOutput(assistantMessage); + const suppressDeterministicApprovalOutput = shouldSuppressDeterministicApprovalOutput(ctx.state); + const suppressMessageToolOnlySourceReplyOutput = hasMessageToolOnlySourceDelivery(ctx); + ctx.noteLastAssistant(assistantMessage); + ctx.noteCompletedAssistant(assistantMessage); + ctx.recordAssistantUsage((assistantMessage as { usage?: unknown }).usage); + ctx.commitAssistantUsage(); + if (suppressVisibleAssistantOutput) { + const isResponsesCommentary = isResponsesApiAssistantMessage(assistantMessage); + const commentaryMessage = isResponsesCommentary + ? scopeAssistantMessageToStreamBlock( + assistantMessage as AssistantMessage, + ctx.state.lastAssistantStreamContentIndex, + ctx.state.lastAssistantStreamItemId, + ) + : assistantMessage; + const commentaryText = coerceChatContentText(extractAssistantCommentaryText(commentaryMessage)); + appendRawStream({ + ts: Date.now(), + event: "assistant_message_end", + runId: ctx.params.runId, + sessionId: (ctx.params.session as { id?: string }).id, + rawText: coerceChatContentText(extractEmbeddedAssistantText(assistantMessage)), + rawThinking: extractAssistantThinking(assistantMessage), + }); + const commentaryAlreadyStreamed = + isResponsesCommentary && + Boolean(ctx.state.deltaBuffer) && + ctx.state.deltaBuffer === commentaryText; + if (commentaryText && !commentaryAlreadyStreamed) { + ctx.emitAssistantStreamData( + buildAssistantStreamData({ + text: commentaryText, + replace: true, + phase: "commentary", + itemId: isResponsesCommentary ? ctx.state.lastAssistantStreamItemId : undefined, + }), + ); + } + // Commentary-tagged tool turns can still carry durable reasoning under /reasoning on. + const suppressedTrimmedReasoning = ctx.state.includeReasoning + ? extractAssistantThinking(assistantMessage).trim() + : ""; + if ( + !ctx.params.silentExpected && + !suppressDeterministicApprovalOutput && + !suppressMessageToolOnlySourceReplyOutput && + ctx.state.includeReasoning && + suppressedTrimmedReasoning && + ctx.params.onBlockReply && + suppressedTrimmedReasoning !== ctx.state.lastReasoningSent + ) { + ctx.state.lastReasoningSent = suppressedTrimmedReasoning; + ctx.emitBlockReply({ text: suppressedTrimmedReasoning, isReasoning: true }); + } + return; + } + promoteThinkingTagsToBlocks(assistantMessage); + + const rawText = coerceChatContentText(extractEmbeddedAssistantText(assistantMessage)); + const rawVisibleText = coerceChatContentText(extractAssistantVisibleText(assistantMessage)); + appendRawStream({ + ts: Date.now(), + event: "assistant_message_end", + runId: ctx.params.runId, + sessionId: (ctx.params.session as { id?: string }).id, + rawText, + rawThinking: extractAssistantThinking(assistantMessage), + }); + warnIfAssistantEmittedSuspiciousText(ctx, assistantMessage); + const visibleText = + extractStandaloneMessageToolText(rawVisibleText, { + allowRoutedReply: isOpenAiCompletionsAssistantMessage(assistantMessage), + allowCurrentSourceReply: + ctx.params.sourceReplyDeliveryMode === "message_tool_only" && + ctx.builtinToolNames?.has("message") === true, + }) ?? rawVisibleText; + const finalVisibleText = ctx.params.enforceFinalTag + ? ctx.stripBlockTags(visibleText, { thinking: false, final: false }, { final: true }) + : visibleText; + + // Exact NO_REPLY stays silent. The legacy rewrite (silentReplyRewrite) was + // removed by contract; global messaging-tool send evidence is not a + // user-route reply and must never be mirrored into the final payload. + const text = finalVisibleText; + const rawThinking = + ctx.state.includeReasoning || ctx.state.streamReasoning + ? extractAssistantThinking(assistantMessage) || extractThinkingFromTaggedText(rawText) + : ""; + const trimmedReasoning = rawThinking ? rawThinking.trim() : ""; + const trimmedText = text.trim(); + const parsedText = trimmedText ? parseReplyDirectives(trimmedText) : null; + const cleanedText = parsedText?.text ?? ""; + const { mediaUrls, hasMedia } = resolveSendableOutboundReplyParts(parsedText ?? {}); + + const finalizeMessageEnd = () => { + ctx.state.deltaBuffer = ""; + ctx.state.thinkingTagStream = createThinkingTagStreamState(); + ctx.state.blockBuffer = ""; + ctx.blockChunker?.reset(); + ctx.state.blockState.thinking = false; + ctx.state.blockState.final = false; + ctx.state.blockState.inlineCode = createInlineCodeState(); + ctx.state.blockState.fence = undefined; + ctx.state.blockState.reasoningInlineCode = undefined; + ctx.state.blockState.reasoningFence = undefined; + ctx.state.blockState.reasoningPendingFenceFragment = undefined; + ctx.state.blockState.finalInlineCode = undefined; + ctx.state.blockState.finalFence = undefined; + ctx.state.blockState.pendingFenceFragment = undefined; + ctx.state.blockState.pendingTagFragment = undefined; + ctx.state.partialBlockState.fence = undefined; + ctx.state.partialBlockState.reasoningInlineCode = undefined; + ctx.state.partialBlockState.reasoningFence = undefined; + ctx.state.partialBlockState.reasoningPendingFenceFragment = undefined; + ctx.state.partialBlockState.finalInlineCode = undefined; + ctx.state.partialBlockState.finalFence = undefined; + ctx.state.partialBlockState.pendingFenceFragment = undefined; + ctx.state.partialBlockState.pendingTagFragment = undefined; + ctx.state.lastStreamedAssistant = undefined; + ctx.state.lastStreamedAssistantCleaned = undefined; + ctx.state.reasoningStreamOpen = false; + }; + + const previousStreamedText = ctx.state.lastStreamedAssistantCleaned ?? ""; + const shouldReplaceFinalStream = Boolean( + previousStreamedText && cleanedText && !cleanedText.startsWith(previousStreamedText), + ); + const didTextChangeWithinCurrentMessage = Boolean( + previousStreamedText && cleanedText !== previousStreamedText, + ); + const finalStreamDelta = shouldReplaceFinalStream + ? "" + : cleanedText.slice(previousStreamedText.length); + + if ( + !ctx.params.silentExpected && + !suppressDeterministicApprovalOutput && + !suppressMessageToolOnlySourceReplyOutput && + (cleanedText || hasMedia) && + (!ctx.state.emittedAssistantUpdate || + shouldReplaceFinalStream || + didTextChangeWithinCurrentMessage || + hasMedia) + ) { + const data = buildAssistantStreamData({ + text: cleanedText, + delta: finalStreamDelta, + replace: shouldReplaceFinalStream, + mediaUrls, + phase: assistantPhase, + }); + ctx.emitAssistantStreamData(data); + ctx.state.emittedAssistantUpdate = true; + ctx.state.lastStreamedAssistantCleaned = cleanedText; + } + + const silentExpectedWithoutSentinel = + ctx.params.silentExpected && !isSilentReplyText(trimmedText, SILENT_REPLY_TOKEN); + const finalAssistantText = silentExpectedWithoutSentinel ? "" : text; + const addedDuringMessage = ctx.state.assistantTexts.length > ctx.state.assistantTextBaseline; + const chunkerHasBuffered = ctx.blockChunker?.hasBuffered() ?? false; + ctx.finalizeAssistantTexts({ + text: finalAssistantText, + addedDuringMessage, + chunkerHasBuffered, + }); + + const onBlockReply = ctx.params.onBlockReply; + const shouldEmitReasoning = Boolean( + !ctx.params.silentExpected && + !suppressDeterministicApprovalOutput && + !suppressMessageToolOnlySourceReplyOutput && + ctx.state.includeReasoning && + trimmedReasoning && + onBlockReply && + trimmedReasoning !== ctx.state.lastReasoningSent, + ); + const shouldEmitReasoningBeforeAnswer = + shouldEmitReasoning && ctx.state.blockReplyBreak === "message_end" && !addedDuringMessage; + const maybeEmitReasoning = () => { + if (!shouldEmitReasoning || !trimmedReasoning) { + return; + } + ctx.state.lastReasoningSent = trimmedReasoning; + // Lane purity: the payload carries raw thinking only. Tool persistence is + // the verbose lane's job; interleaving comes from arrival order. + ctx.emitBlockReply({ text: trimmedReasoning, isReasoning: true }); + }; + + if (shouldEmitReasoningBeforeAnswer) { + maybeEmitReasoning(); + } + + const emitSplitResultAsBlockReply = ( + splitResult: ReturnType | null | undefined, + ) => { + if (!splitResult || !onBlockReply) { + return; + } + const { + text: cleanedTextLocal, + mediaUrls: mediaUrlsLocal, + audioAsVoice, + replyToId, + replyToTag, + replyToCurrent, + } = splitResult; + // Emit if there's content OR audioAsVoice flag (to propagate the flag). + if ( + hasAssistantVisibleReply({ text: cleanedTextLocal, mediaUrls: mediaUrlsLocal, audioAsVoice }) + ) { + ctx.emitBlockReply( + { + text: cleanedTextLocal, + mediaUrls: mediaUrlsLocal?.length ? mediaUrlsLocal : undefined, + audioAsVoice, + replyToId, + replyToTag, + replyToCurrent, + }, + { assistantMessageIndex: ctx.state.assistantMessageIndex }, + ); + } + }; + + const consumeFinalReplyDirectives = () => { + const bufferedResult = ctx.consumeReplyDirectives("", { final: true }); + if (!hasMedia || !parsedText) { + return bufferedResult; + } + const bufferedRawText = bufferedResult?.text ?? ""; + const leadingWhitespace = bufferedRawText.match(/^\s+/u)?.[0] ?? ""; + const strippedBufferedText = bufferedRawText ? splitMediaFromOutput(bufferedRawText).text : ""; + const bufferedText = + leadingWhitespace && + strippedBufferedText && + !strippedBufferedText.startsWith(leadingWhitespace) + ? `${leadingWhitespace}${strippedBufferedText}` + : strippedBufferedText; + return { + ...bufferedResult, + ...parsedText, + text: bufferedText, + }; + }; + + const hasBufferedBlockReply = ctx.blockChunker + ? ctx.blockChunker.hasBuffered() + : ctx.state.blockBuffer.length > 0; + + if ( + !ctx.params.silentExpected && + !suppressDeterministicApprovalOutput && + !suppressMessageToolOnlySourceReplyOutput && + text && + onBlockReply && + (ctx.state.blockReplyBreak === "message_end" || + hasBufferedBlockReply || + text !== ctx.state.lastBlockReplyText || + hasMedia) + ) { + if (hasBufferedBlockReply && ctx.blockChunker?.hasBuffered()) { + const flushBlockReplyBufferResult = ctx.flushBlockReplyBuffer({ + assistantMessageIndex: ctx.state.assistantMessageIndex, + final: true, + }); + if (isPromiseLike(flushBlockReplyBufferResult)) { + void flushBlockReplyBufferResult.catch((err: unknown) => { + ctx.log.debug(`message_end block reply flush failed: ${String(err)}`); + }); + } + // Final-flush the streaming directive accumulator so any partial + // inline reply/audio tag held back by splitTrailingDirective gets + // emitted on the message_end / blockReplyChunking path. + emitSplitResultAsBlockReply(consumeFinalReplyDirectives()); + } else if (text !== ctx.state.lastBlockReplyText || hasMedia) { + // Guard: for text_end channels, if text_end already delivered content + // (lastBlockReplyText is set), skip this safety send. The text comparison + // here uses a different stripping pipeline (stripBlockTags with reset state) + // than emitBlockChunk (stripBlockTags with running blockState + + // stripDowngradedToolCallText), which can false-positive. When text_end + // didn't deliver (e.g. commentary suppressed, provider skipped text_end), + // lastBlockReplyText is still null and message_end must deliver. + if ( + ctx.state.blockReplyBreak === "text_end" && + ctx.state.lastBlockReplyText != null && + !hasMedia + ) { + ctx.log.debug( + `Skipping message_end safety send for text_end channel - content already delivered via text_end`, + ); + } else { + // Check for duplicates before emitting (same logic as emitBlockChunk). + const normalizedText = normalizeTextForComparison(hasMedia ? cleanedText : text); + if ( + isMessagingToolDuplicateNormalized( + normalizedText, + ctx.state.messagingToolSentTextsNormalized, + ) + ) { + ctx.log.debug( + `Skipping message_end block reply - already sent via messaging tool: ${truncateUtf16Safe(text, 50)}...`, + ); + } else { + const alreadyDeliveredFinalText = Boolean( + hasMedia && cleanedText && cleanedText === ctx.state.lastBlockReplyText, + ); + ctx.state.lastBlockReplyText = hasMedia ? cleanedText || text : text; + ctx.state.lastDeliveredBlockReplyText = hasMedia ? cleanedText || text : text; + ctx.state.toolExecutionSinceLastBlockReply = false; + emitSplitResultAsBlockReply( + hasMedia && parsedText + ? { + ...parsedText, + text: alreadyDeliveredFinalText ? "" : cleanedText, + } + : ctx.consumeReplyDirectives(text, { final: true }), + ); + } + } + } + } + + if (!shouldEmitReasoningBeforeAnswer) { + maybeEmitReasoning(); + } + if (!ctx.params.silentExpected && rawThinking) { + // Emit-always: bus/archive get message-end thinking regardless of the + // streamReasoning rendering setting (gated inside emitReasoningStream). + ctx.emitReasoningStream(rawThinking); + } + + if ( + !ctx.params.silentExpected && + !suppressMessageToolOnlySourceReplyOutput && + ctx.state.blockReplyBreak === "text_end" && + onBlockReply + ) { + emitSplitResultAsBlockReply(ctx.consumeReplyDirectives("", { final: true })); + } + + if ( + !ctx.params.silentExpected && + ctx.state.blockReplyBreak === "message_end" && + ctx.params.onBlockReplyFlush + ) { + const flushBlockReplyBufferResult = ctx.flushBlockReplyBuffer(); + if (isPromiseLike(flushBlockReplyBufferResult)) { + return flushBlockReplyBufferResult + .then(() => { + const onBlockReplyFlushResult = ctx.params.onBlockReplyFlush?.({ + reason: "message_end", + }); + if (isPromiseLike(onBlockReplyFlushResult)) { + return onBlockReplyFlushResult; + } + return undefined; + }) + .finally(() => { + finalizeMessageEnd(); + }); + } + const onBlockReplyFlushResult = ctx.params.onBlockReplyFlush({ reason: "message_end" }); + if (isPromiseLike(onBlockReplyFlushResult)) { + return onBlockReplyFlushResult.finally(() => { + finalizeMessageEnd(); + }); + } + } + + finalizeMessageEnd(); + return undefined; +} diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.replies.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.replies.test.ts new file mode 100644 index 000000000000..965e5d6eb83c --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.replies.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from "vitest"; +import { + consumePendingToolMediaIntoReply, + consumePendingToolMediaReply, + readPendingToolMediaReply, +} from "./embedded-agent-subscribe.handlers.messages.replies.js"; + +describe("consumePendingToolMediaIntoReply", () => { + it("attaches queued tool media to the next assistant reply", () => { + const state = { + pendingToolMediaUrls: ["/tmp/a.png", "/tmp/a.png", "/tmp/b.png"], + pendingToolMediaAttachments: [ + { type: "image" as const, path: "/tmp/a.png", width: 640, height: 480 }, + { type: "image" as const, path: "/tmp/a.png", width: 1, height: 1 }, + { type: "image" as const, path: "/tmp/b.png", width: 800, height: 600 }, + ], + pendingToolMediaTrustByUrl: new Map([ + ["/tmp/a.png", true], + ["/tmp/b.png", false], + ]), + pendingToolAudioAsVoice: false, + }; + + expect( + consumePendingToolMediaIntoReply(state, { + text: "done", + }), + ).toEqual({ + text: "done", + mediaUrls: ["/tmp/a.png", "/tmp/b.png"], + attachments: [ + { + type: "image", + path: "/tmp/a.png", + width: 640, + height: 480, + trustedLocalMedia: true, + }, + { type: "image", path: "/tmp/b.png", width: 800, height: 600 }, + ], + audioAsVoice: undefined, + }); + expect(state.pendingToolMediaUrls).toStrictEqual([]); + expect(state.pendingToolMediaAttachments).toStrictEqual([]); + }); + + it("does not append queued image tool media when the reply already names media", () => { + const state = { + pendingToolMediaUrls: ["/tmp/generated.png"], + pendingToolMediaTrustByUrl: new Map([["/tmp/generated.png", true]]), + pendingToolAudioAsVoice: false, + }; + + expect( + consumePendingToolMediaIntoReply(state, { + text: "done", + mediaUrls: ["./selected.png"], + }), + ).toEqual({ + text: "done", + mediaUrls: ["./selected.png"], + }); + expect(state.pendingToolMediaUrls).toStrictEqual([]); + expect(state.pendingToolAudioAsVoice).toBe(false); + expect(state.pendingToolMediaTrustByUrl.size).toBe(0); + }); + + it("retains queued metadata for explicitly selected media", () => { + const state = { + pendingToolMediaUrls: ["/tmp/generated.mp3", "/tmp/generated.mp3", "/tmp/unselected.mp3"], + pendingToolMediaAttachments: [ + { type: "audio" as const, path: "/tmp/generated.mp3", durationMs: 2_000 }, + { type: "audio" as const, path: "/tmp/generated.mp3", durationMs: 9_999 }, + { type: "audio" as const, path: "/tmp/unselected.mp3", durationMs: 3_000 }, + ], + pendingToolMediaTrustByUrl: new Map([ + ["/tmp/generated.mp3", true], + ["/tmp/unselected.mp3", false], + ]), + pendingToolAudioAsVoice: false, + }; + + expect( + consumePendingToolMediaIntoReply(state, { + text: "done", + mediaUrls: [" /tmp/generated.mp3 "], + }), + ).toEqual({ + text: "done", + mediaUrls: [" /tmp/generated.mp3 "], + attachments: [ + { + type: "audio", + path: "/tmp/generated.mp3", + durationMs: 2_000, + trustedLocalMedia: true, + }, + ], + trustedLocalMedia: true, + }); + expect(state.pendingToolMediaAttachments).toStrictEqual([]); + }); + + it("does not trust an explicitly selected untrusted pending URL", () => { + const state = { + pendingToolMediaUrls: ["/tmp/generated.mp3", "/tmp/untrusted.mp3"], + pendingToolMediaAttachments: [ + { type: "audio" as const, path: "/tmp/generated.mp3" }, + { + type: "audio" as const, + path: "/tmp/untrusted.mp3", + trustedLocalMedia: true, + }, + ], + pendingToolMediaTrustByUrl: new Map([ + ["/tmp/generated.mp3", true], + ["/tmp/untrusted.mp3", false], + ]), + pendingToolAudioAsVoice: false, + }; + + expect( + consumePendingToolMediaIntoReply(state, { + text: "done", + mediaUrls: ["/tmp/untrusted.mp3"], + }), + ).toEqual({ + text: "done", + mediaUrls: ["/tmp/untrusted.mp3"], + attachments: [{ type: "audio", path: "/tmp/untrusted.mp3" }], + }); + }); + + it("does not append queued voice media when the reply already names media", () => { + const state = { + pendingToolMediaUrls: ["/tmp/reply.opus"], + pendingToolMediaTrustByUrl: new Map([["/tmp/reply.opus", true]]), + pendingToolAudioAsVoice: true, + }; + + expect( + consumePendingToolMediaIntoReply(state, { + text: "done", + mediaUrls: ["/tmp/assistant-provided.opus"], + }), + ).toEqual({ + text: "done", + mediaUrls: ["/tmp/assistant-provided.opus"], + }); + expect(state.pendingToolMediaUrls).toStrictEqual([]); + expect(state.pendingToolAudioAsVoice).toBe(false); + expect(state.pendingToolMediaTrustByUrl.size).toBe(0); + }); + + it("preserves reasoning replies without consuming queued media", () => { + const state = { + pendingToolMediaUrls: ["/tmp/a.png"], + pendingToolMediaTrustByUrl: new Map([["/tmp/a.png", false]]), + pendingToolAudioAsVoice: true, + }; + + expect( + consumePendingToolMediaIntoReply(state, { + text: "thinking", + isReasoning: true, + }), + ).toEqual({ + text: "thinking", + isReasoning: true, + }); + expect(state.pendingToolMediaUrls).toEqual(["/tmp/a.png"]); + expect(state.pendingToolAudioAsVoice).toBe(true); + }); +}); + +describe("consumePendingToolMediaReply", () => { + it("reads a media-only reply without consuming queued tool media", () => { + const state = { + pendingToolMediaUrls: ["/tmp/reply.opus"], + pendingToolMediaTrustByUrl: new Map([["/tmp/reply.opus", false]]), + pendingToolAudioAsVoice: true, + }; + + expect(readPendingToolMediaReply(state)).toEqual({ + mediaUrls: ["/tmp/reply.opus"], + audioAsVoice: true, + }); + expect(state.pendingToolMediaUrls).toEqual(["/tmp/reply.opus"]); + expect(state.pendingToolAudioAsVoice).toBe(true); + }); + + it("builds a media-only reply for orphaned tool media", () => { + const state = { + pendingToolMediaUrls: ["/tmp/reply.opus"], + pendingToolMediaTrustByUrl: new Map([["/tmp/reply.opus", false]]), + pendingToolAudioAsVoice: true, + }; + + expect(consumePendingToolMediaReply(state)).toEqual({ + mediaUrls: ["/tmp/reply.opus"], + audioAsVoice: true, + }); + expect(state.pendingToolMediaUrls).toStrictEqual([]); + expect(state.pendingToolAudioAsVoice).toBe(false); + }); +}); diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.replies.ts b/src/agents/embedded-agent-subscribe.handlers.messages.replies.ts new file mode 100644 index 000000000000..5e4fb9095d4a --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.replies.ts @@ -0,0 +1,259 @@ +/** + * Owns pending assistant reply directives and tool-media handoff. + */ +import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload"; +import type { ReplyDirectiveParseResult } from "../auto-reply/reply/reply-directives.js"; +import type { BlockReplyPayload } from "./embedded-agent-payloads.js"; +import type { EmbeddedAgentSubscribeState } from "./embedded-agent-subscribe.handlers.types.js"; + +export function hasReplyDirectiveMetadata( + parsed: ReplyDirectiveParseResult | null | undefined, +): boolean { + return Boolean( + parsed && + ((parsed.mediaUrls?.length ?? 0) > 0 || + parsed.audioAsVoice || + parsed.replyToId || + parsed.replyToTag || + parsed.replyToCurrent), + ); +} + +function hasReplyDirectiveMetadataResult( + parsed: ReplyDirectiveParseResult | null | undefined, +): parsed is ReplyDirectiveParseResult { + return hasReplyDirectiveMetadata(parsed); +} + +export function mergeReplyDirectiveResults( + first: ReplyDirectiveParseResult | null | undefined, + second: ReplyDirectiveParseResult | null | undefined, +): ReplyDirectiveParseResult | null { + if (!first) { + return second ?? null; + } + if (!second) { + return first; + } + const mediaUrls = uniqueStrings([...(first.mediaUrls ?? []), ...(second.mediaUrls ?? [])]); + return { + text: `${first.text ?? ""}${second.text ?? ""}`, + mediaUrls: mediaUrls.length ? mediaUrls : undefined, + replyToId: second.replyToId ?? first.replyToId, + replyToCurrent: first.replyToCurrent || second.replyToCurrent, + replyToTag: first.replyToTag || second.replyToTag, + audioAsVoice: first.audioAsVoice || second.audioAsVoice || undefined, + isSilent: first.isSilent || second.isSilent, + }; +} + +function clearPendingToolMedia( + state: Pick< + EmbeddedAgentSubscribeState, + | "pendingToolMediaUrls" + | "pendingToolMediaAttachments" + | "pendingToolMediaTrustByUrl" + | "pendingToolAudioAsVoice" + >, +) { + state.pendingToolMediaUrls = []; + state.pendingToolMediaAttachments = []; + state.pendingToolMediaTrustByUrl.clear(); + state.pendingToolAudioAsVoice = false; +} + +function hasReplyMedia(payload: BlockReplyPayload): boolean { + return (payload.mediaUrls ?? []).some((url) => url.trim().length > 0); +} + +function readAlignedPendingToolMedia( + state: Pick< + EmbeddedAgentSubscribeState, + "pendingToolMediaUrls" | "pendingToolMediaAttachments" | "pendingToolMediaTrustByUrl" + >, +) { + const seen = new Set(); + const mediaUrls: string[] = []; + const attachments: NonNullable = []; + for (const [index, url] of state.pendingToolMediaUrls.entries()) { + if (seen.has(url)) { + continue; + } + seen.add(url); + mediaUrls.push(url); + const { trustedLocalMedia: _untrustedInput, ...attachment } = + state.pendingToolMediaAttachments?.[index] ?? {}; + attachments.push({ + ...attachment, + ...(state.pendingToolMediaTrustByUrl.get(url) === true ? { trustedLocalMedia: true } : {}), + }); + } + return { + mediaUrls, + attachments: attachments.some((entry) => Object.keys(entry).length > 0) + ? attachments + : undefined, + }; +} + +/** Moves queued tool media into a non-reasoning assistant reply payload. */ +export function consumePendingToolMediaIntoReply( + state: Pick< + EmbeddedAgentSubscribeState, + | "pendingToolMediaUrls" + | "pendingToolMediaAttachments" + | "pendingToolMediaTrustByUrl" + | "pendingToolAudioAsVoice" + >, + payload: BlockReplyPayload, +): BlockReplyPayload { + if (payload.isReasoning) { + return payload; + } + if (state.pendingToolMediaUrls.length === 0 && !state.pendingToolAudioAsVoice) { + return payload; + } + if (hasReplyMedia(payload)) { + // Pending tool media is a fallback delivery queue; explicit final media is + // the assistant's user-visible selection, while tool output remains in the transcript. + const alignedPendingMedia = readAlignedPendingToolMedia(state); + const metadataByUrl = new Map( + alignedPendingMedia.mediaUrls.map((url, index) => [ + url, + alignedPendingMedia.attachments?.[index] ?? {}, + ]), + ); + const selectedAttachments = (payload.mediaUrls ?? []).map( + (url) => metadataByUrl.get(url.trim()) ?? {}, + ); + const allSelectedMediaIsPending = + (payload.mediaUrls?.length ?? 0) > 0 && + (payload.mediaUrls ?? []).every((url) => metadataByUrl.has(url.trim())); + const payloadWithMetadata = + payload.attachments?.length || + selectedAttachments.every((entry) => Object.keys(entry).length === 0) + ? payload + : { ...payload, attachments: selectedAttachments }; + const selectedPayload = + allSelectedMediaIsPending && + (payload.mediaUrls ?? []).every( + (url) => state.pendingToolMediaTrustByUrl.get(url.trim()) === true, + ) + ? { ...payloadWithMetadata, trustedLocalMedia: true } + : payloadWithMetadata; + clearPendingToolMedia(state); + return selectedPayload; + } + const pendingMedia = readAlignedPendingToolMedia(state); + const allPendingMediaTrusted = + pendingMedia.mediaUrls.length > 0 && + pendingMedia.mediaUrls.every((url) => state.pendingToolMediaTrustByUrl.get(url) === true); + const mergedPayload: BlockReplyPayload = { + ...payload, + mediaUrls: pendingMedia.mediaUrls.length ? pendingMedia.mediaUrls : undefined, + attachments: pendingMedia.attachments, + audioAsVoice: payload.audioAsVoice || state.pendingToolAudioAsVoice || undefined, + ...(payload.trustedLocalMedia || allPendingMediaTrusted ? { trustedLocalMedia: true } : {}), + }; + clearPendingToolMedia(state); + return mergedPayload; +} + +/** Consumes queued tool media as a standalone reply payload. */ +export function consumePendingToolMediaReply( + state: Pick< + EmbeddedAgentSubscribeState, + | "pendingToolMediaUrls" + | "pendingToolMediaAttachments" + | "pendingToolMediaTrustByUrl" + | "pendingToolAudioAsVoice" + >, +): BlockReplyPayload | null { + const payload = readPendingToolMediaReply(state); + if (!payload) { + return null; + } + clearPendingToolMedia(state); + return payload; +} + +/** Reads queued tool media without clearing it. */ +export function readPendingToolMediaReply( + state: Pick< + EmbeddedAgentSubscribeState, + | "pendingToolMediaUrls" + | "pendingToolMediaAttachments" + | "pendingToolMediaTrustByUrl" + | "pendingToolAudioAsVoice" + >, +): BlockReplyPayload | null { + if (state.pendingToolMediaUrls.length === 0 && !state.pendingToolAudioAsVoice) { + return null; + } + const pendingMedia = readAlignedPendingToolMedia(state); + const allPendingMediaTrusted = + pendingMedia.mediaUrls.length > 0 && + pendingMedia.mediaUrls.every((url) => state.pendingToolMediaTrustByUrl.get(url) === true); + return { + mediaUrls: pendingMedia.mediaUrls.length ? pendingMedia.mediaUrls : undefined, + attachments: pendingMedia.attachments, + audioAsVoice: state.pendingToolAudioAsVoice || undefined, + ...(allPendingMediaTrusted ? { trustedLocalMedia: true } : {}), + }; +} + +export function recordPendingAssistantReplyDirectives( + state: Pick, + parsed: ReplyDirectiveParseResult | null | undefined, +) { + if (!hasReplyDirectiveMetadataResult(parsed)) { + return; + } + const current = state.pendingAssistantReplyDirectives; + const mediaUrls = Array.from( + new Set([...(current?.mediaUrls ?? []), ...(parsed.mediaUrls ?? [])]), + ); + state.pendingAssistantReplyDirectives = { + mediaUrls: mediaUrls.length ? mediaUrls : undefined, + audioAsVoice: current?.audioAsVoice || parsed?.audioAsVoice || undefined, + replyToId: parsed?.replyToId ?? current?.replyToId, + replyToTag: current?.replyToTag || parsed.replyToTag || undefined, + replyToCurrent: current?.replyToCurrent || parsed.replyToCurrent || undefined, + }; +} + +/** Merges pending reply directives into one reply payload and clears them. */ +export function consumePendingAssistantReplyDirectivesIntoReply( + state: Pick, + payload: BlockReplyPayload, +): BlockReplyPayload { + if (payload.isReasoning || !state.pendingAssistantReplyDirectives) { + return payload; + } + const pending = state.pendingAssistantReplyDirectives; + const mediaUrls = Array.from( + new Set([...(payload.mediaUrls ?? []), ...(pending.mediaUrls ?? [])]), + ); + state.pendingAssistantReplyDirectives = undefined; + return { + ...payload, + mediaUrls: mediaUrls.length ? mediaUrls : undefined, + audioAsVoice: payload.audioAsVoice || pending.audioAsVoice || undefined, + replyToId: payload.replyToId ?? pending.replyToId, + replyToTag: Boolean(payload.replyToTag || pending.replyToTag) || undefined, + replyToCurrent: Boolean(payload.replyToCurrent || pending.replyToCurrent) || undefined, + }; +} + +/** True when a reply payload has text, media, or voice content worth sending. */ +export function hasAssistantVisibleReply(params: { + text?: string; + mediaUrls?: string[]; + mediaUrl?: string; + audioAsVoice?: boolean; +}): boolean { + return resolveSendableOutboundReplyParts(params).hasContent || Boolean(params.audioAsVoice); +} + +/** Builds normalized stream payload data for assistant visible output. */ diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.stream-replies.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.stream-replies.test.ts new file mode 100644 index 000000000000..daca55c595d0 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.stream-replies.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { + consumePendingAssistantReplyDirectivesIntoReply, + hasAssistantVisibleReply, +} from "./embedded-agent-subscribe.handlers.messages.replies.js"; +import { + buildAssistantStreamData, + recordPendingAssistantReplyDirectives, +} from "./embedded-agent-subscribe.handlers.messages.test-support.js"; + +describe("hasAssistantVisibleReply", () => { + it("treats audio-only payloads as visible", () => { + expect(hasAssistantVisibleReply({ audioAsVoice: true })).toBe(true); + }); + + it("detects text or media visibility", () => { + expect(hasAssistantVisibleReply({ text: "hello" })).toBe(true); + expect(hasAssistantVisibleReply({ mediaUrls: ["https://example.com/a.png"] })).toBe(true); + expect(hasAssistantVisibleReply({})).toBe(false); + }); +}); + +describe("buildAssistantStreamData", () => { + it("normalizes media payloads for assistant stream events", () => { + expect( + buildAssistantStreamData({ + text: "hello", + delta: "he", + replace: true, + mediaUrl: "https://example.com/a.png", + phase: "final_answer", + }), + ).toEqual({ + text: "hello", + delta: "he", + replace: true, + mediaUrls: ["https://example.com/a.png"], + phase: "final_answer", + }); + }); +}); + +describe("pending assistant reply directives", () => { + it("merges directive metadata into the next non-reasoning block reply", () => { + const state = { pendingAssistantReplyDirectives: undefined }; + + recordPendingAssistantReplyDirectives(state, { + text: "", + mediaUrls: ["/tmp/reply.ogg"], + replyToCurrent: true, + replyToTag: true, + audioAsVoice: true, + isSilent: false, + }); + + expect( + consumePendingAssistantReplyDirectivesIntoReply(state, { + text: "Done.", + }), + ).toEqual({ + text: "Done.", + mediaUrls: ["/tmp/reply.ogg"], + audioAsVoice: true, + replyToId: undefined, + replyToTag: true, + replyToCurrent: true, + }); + expect(state.pendingAssistantReplyDirectives).toBeUndefined(); + }); + + it("does not consume pending directive metadata on reasoning replies", () => { + const state = { + pendingAssistantReplyDirectives: { + mediaUrls: ["/tmp/reply.png"], + }, + }; + + expect( + consumePendingAssistantReplyDirectivesIntoReply(state, { + text: "Thinking...", + isReasoning: true, + }), + ).toEqual({ + text: "Thinking...", + isReasoning: true, + }); + expect(state.pendingAssistantReplyDirectives?.mediaUrls).toEqual(["/tmp/reply.png"]); + }); +}); diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.stream.ts b/src/agents/embedded-agent-subscribe.handlers.messages.stream.ts new file mode 100644 index 000000000000..16464f27f793 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.stream.ts @@ -0,0 +1,441 @@ +/** + * Projects provider assistant messages into ordered visible stream state. + */ +import { asOptionalRecord as asRecord } from "@openclaw/normalization-core/record-coerce"; +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload"; +import { + parseReplyDirectives, + type ReplyDirectiveParseResult, +} from "../auto-reply/reply/reply-directives.js"; +import { splitTrailingDirective } from "../auto-reply/reply/streaming-directives.js"; +import type { AssistantMessage } from "../llm/types.js"; +import { + parseAssistantTextSignature, + resolveAssistantMessagePhase, + type AssistantPhase, +} from "../shared/chat-message-content.js"; +import { normalizeTextForComparison } from "./embedded-agent-helpers.js"; +import { runBestEffortCallback } from "./embedded-agent-subscribe.callback.js"; +import { hasReplyDirectiveMetadata } from "./embedded-agent-subscribe.handlers.messages.replies.js"; +import type { + EmbeddedAgentSubscribeContext, + EmbeddedAgentSubscribeState, +} from "./embedded-agent-subscribe.handlers.types.js"; +import type { AgentMessage } from "./runtime/index.js"; + +export function shouldSuppressAssistantVisibleOutput(message: AgentMessage | undefined): boolean { + return resolveAssistantMessagePhase(message) === "commentary"; +} + +export function isSubscribeTranscriptOnlyOpenClawAssistantMessage( + message: AgentMessage | undefined, +): boolean { + if (!message || message.role !== "assistant") { + return false; + } + const provider = normalizeOptionalString(message.provider) ?? ""; + const model = normalizeOptionalString(message.model) ?? ""; + return provider === "openclaw" && (model === "delivery-mirror" || model === "gateway-injected"); +} + +const RESPONSES_API_IDS = new Set([ + "openai-responses", + "openai-chatgpt-responses", + "azure-openai-responses", + "openclaw-openai-responses-transport", + "openclaw-openai-chatgpt-responses-transport", + "openclaw-azure-openai-responses-transport", +]); + +export function isResponsesApiAssistantMessage(message: AgentMessage | undefined): boolean { + if (!message || message.role !== "assistant") { + return false; + } + const api = normalizeOptionalString((message as { api?: unknown }).api) ?? ""; + return RESPONSES_API_IDS.has(api); +} + +export function isAnthropicAssistantMessage(message: AgentMessage | undefined): boolean { + if (!message || message.role !== "assistant") { + return false; + } + const api = normalizeOptionalString((message as { api?: unknown }).api) ?? ""; + return api === "anthropic-messages"; +} + +export function isOpenAiCompletionsAssistantMessage(message: AgentMessage | undefined): boolean { + if (!message || message.role !== "assistant") { + return false; + } + const api = normalizeOptionalString((message as { api?: unknown }).api) ?? ""; + return api === "openai-completions" || api === "openclaw-openai-completions-transport"; +} + +export function extractStandaloneMessageToolText( + text: string, + params: { allowCurrentSourceReply?: boolean; allowRoutedReply?: boolean } = {}, +): string | undefined { + try { + const record = asRecord(JSON.parse(text.trim()) as unknown); + const args = asRecord(record?.arguments); + const hasRoute = Boolean( + normalizeOptionalString(args?.target) || + normalizeOptionalString(args?.to) || + normalizeOptionalString(args?.channel) || + normalizeOptionalString(args?.accountId) || + Array.isArray(args?.targets), + ); + if ( + normalizeOptionalString(record?.name) !== "message" || + normalizeOptionalString(args?.action) !== "send" || + (hasRoute ? !params.allowRoutedReply : !params.allowCurrentSourceReply) + ) { + return undefined; + } + return normalizeOptionalString(args?.message); + } catch { + return undefined; + } +} + +export function resolveAssistantStreamItemId(params: { + contentIndex?: unknown; + message: AgentMessage | undefined; +}): string | undefined { + const content = (params.message as { content?: unknown } | undefined)?.content; + if (!Array.isArray(content)) { + return undefined; + } + const contentIndex = + typeof params.contentIndex === "number" && + Number.isInteger(params.contentIndex) && + params.contentIndex >= 0 + ? params.contentIndex + : undefined; + const indexedBlock = contentIndex !== undefined ? content[contentIndex] : undefined; + const indexedRecord = + indexedBlock && typeof indexedBlock === "object" + ? (indexedBlock as { type?: unknown }) + : undefined; + const hasIndexedTextBlock = indexedRecord?.type === "text"; + const candidateStart = + hasIndexedTextBlock && contentIndex !== undefined ? contentIndex : content.length - 1; + const candidateEnd = hasIndexedTextBlock ? candidateStart : 0; + for (let index = candidateStart; index >= candidateEnd; index -= 1) { + const block = content[index]; + if (!block || typeof block !== "object") { + continue; + } + const record = block as { type?: unknown; textSignature?: unknown }; + if (record.type !== "text") { + continue; + } + const signature = parseAssistantTextSignature(record); + if (signature?.id) { + return signature.id; + } + } + return undefined; +} + +export function resolveAssistantStreamContentIndex(value: unknown): number | undefined { + return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined; +} + +export function scopeAssistantMessageToStreamBlock( + message: AssistantMessage, + contentIndex: number | undefined, + itemId: string | undefined, +): AssistantMessage { + if (!Array.isArray(message.content)) { + return message; + } + const indexedBlock = contentIndex === undefined ? undefined : message.content[contentIndex]; + let block = + indexedBlock && typeof indexedBlock === "object" && indexedBlock.type === "text" + ? indexedBlock + : undefined; + if (!block && itemId) { + for (let index = message.content.length - 1; index >= 0; index -= 1) { + const candidate = message.content[index]; + if ( + candidate && + typeof candidate === "object" && + candidate.type === "text" && + parseAssistantTextSignature(candidate)?.id === itemId + ) { + block = candidate; + break; + } + } + } + if (!block) { + return message; + } + // Provider partials are cumulative across content blocks. Once a content + // index becomes a logical reply boundary, downstream snapshots must be + // cumulative only within that block or earlier text is replayed. + return { ...message, content: [block] }; +} + +export function emitReasoningEnd(ctx: EmbeddedAgentSubscribeContext) { + if (!ctx.state.reasoningStreamOpen) { + return; + } + ctx.state.reasoningStreamOpen = false; + runBestEffortCallback({ + label: "reasoning end", + log: ctx.log, + callback: () => ctx.params.onReasoningEnd?.(), + }); +} + +export function emitAssistantMessageStart(ctx: EmbeddedAgentSubscribeContext) { + runBestEffortCallback({ + label: "assistant message start", + log: ctx.log, + callback: () => ctx.params.onAssistantMessageStart?.(), + }); +} + +export function openReasoningStream(ctx: EmbeddedAgentSubscribeContext) { + ctx.state.reasoningStreamOpen = true; +} + +export function shouldSuppressDeterministicApprovalOutput( + state: Pick< + EmbeddedAgentSubscribeState, + "deterministicApprovalPromptPending" | "deterministicApprovalPromptSent" + >, +): boolean { + return state.deterministicApprovalPromptPending || state.deterministicApprovalPromptSent; +} + +export function hasMessageToolOnlySourceDelivery(ctx: EmbeddedAgentSubscribeContext): boolean { + return ( + ctx.params.sourceReplyDeliveryMode === "message_tool_only" && + (ctx.state.messageToolOnlySourceReplyDelivered || + ctx.params.hasDeliveredMessageToolOnlySourceReply?.() === true || + (ctx.state.messagingToolSourceReplyPayloads?.length ?? 0) > 0) + ); +} + +export function resolveCurrentSourceMessagingToolPartial( + state: Pick< + EmbeddedAgentSubscribeState, + "currentSourceMessagingToolHeldPartial" | "currentSourceMessagingToolSentTextsNormalized" + >, + params: { + evtType: "text_delta" | "text_start" | "text_end"; + text: string; + visibleDelta: string; + }, +): { hold: boolean; text: string } { + const held = state.currentSourceMessagingToolHeldPartial; + const text = + held && params.evtType === "text_delta" && !params.text.startsWith(held) + ? `${held}${params.visibleDelta || params.text}` + : params.text; + const normalized = normalizeTextForComparison(text); + if (!normalized) { + state.currentSourceMessagingToolHeldPartial = undefined; + return { hold: false, text }; + } + // A confirmed current-source tool send already made this prefix visible. + // Hold it until the assistant either repeats the sent text or diverges with new content. + const hold = state.currentSourceMessagingToolSentTextsNormalized.some( + (sentText) => sentText === normalized || sentText.startsWith(normalized), + ); + state.currentSourceMessagingToolHeldPartial = hold ? text : undefined; + return { hold, text }; +} + +export function appendBlockReplyChunk(ctx: EmbeddedAgentSubscribeContext, chunk: string) { + if (ctx.blockChunker) { + ctx.blockChunker.append(chunk); + return; + } + ctx.state.blockBuffer += chunk; +} + +export function replaceBlockReplyBuffer(ctx: EmbeddedAgentSubscribeContext, text: string) { + if (ctx.blockChunker) { + ctx.blockChunker.reset(); + ctx.blockChunker.append(text); + return; + } + ctx.state.blockBuffer = text; +} + +export function resolveAssistantTextChunk(params: { + evtType: "text_delta" | "text_start" | "text_end"; + delta: string; + content: string; + accumulatedText: string; +}): string { + const { evtType, delta, content, accumulatedText } = params; + if (evtType === "text_delta") { + return delta; + } + if (delta) { + return delta; + } + if (!content) { + return ""; + } + // KNOWN: Some providers resend full content on `text_end`. + // We only append a suffix (or nothing) to keep output monotonic. + if (content.startsWith(accumulatedText)) { + return content.slice(accumulatedText.length); + } + if (accumulatedText.startsWith(content)) { + return ""; + } + if (!accumulatedText.includes(content)) { + return content; + } + return ""; +} + +export function resolveStreamVisibleText(params: { + previousRawText: string; + visibleDelta: string; + finalText?: string; +}): { rawText: string; visibleText: string } { + if (params.finalText !== undefined) { + const rawText = params.finalText; + return { rawText, visibleText: rawText.trim() }; + } + const rawText = `${params.previousRawText}${params.visibleDelta}`; + return { rawText, visibleText: rawText.trim() }; +} + +export function resolveTextAppendDelta(previousText: string, nextText: string): string { + if (!nextText) { + return ""; + } + if (!previousText) { + return nextText; + } + if (nextText.startsWith(previousText)) { + return nextText.slice(previousText.length); + } + if (previousText.startsWith(nextText)) { + return ""; + } + return nextText; +} + +export function copyPartialBlockState( + target: EmbeddedAgentSubscribeState["partialBlockState"], + source: EmbeddedAgentSubscribeState["partialBlockState"], +) { + const copyFenceState = (fence?: typeof source.fence) => + fence + ? { + atLineStart: fence.atLineStart, + ...(fence.open ? { open: { ...fence.open } } : {}), + } + : undefined; + target.thinking = source.thinking; + target.final = source.final; + target.inlineCode = { ...source.inlineCode }; + target.fence = copyFenceState(source.fence); + target.reasoningInlineCode = source.reasoningInlineCode + ? { ...source.reasoningInlineCode } + : undefined; + target.reasoningFence = copyFenceState(source.reasoningFence); + target.reasoningPendingFenceFragment = source.reasoningPendingFenceFragment; + target.finalInlineCode = source.finalInlineCode ? { ...source.finalInlineCode } : undefined; + target.finalFence = copyFenceState(source.finalFence); + target.pendingFenceFragment = source.pendingFenceFragment; + target.pendingTagFragment = source.pendingTagFragment; +} + +function containsCompleteMediaDirectiveLine(text: string): boolean { + return /(?:^|\n)\s*MEDIA:\s*\S[^\n]*(?:\n|$)/i.test(text); +} + +function resolveIncrementalStreamingReplyText(params: { + evtType: "text_delta" | "text_start" | "text_end"; + next: string; + previousRawText: string; + previousCleaned: string; + visibleDelta: string; + parsedStreamDirectives: ReplyDirectiveParseResult | null; + shouldUsePhaseAwareBlockReply: boolean; +}): string | undefined { + if ( + params.evtType === "text_end" || + !params.parsedStreamDirectives || + params.parsedStreamDirectives.isSilent || + hasReplyDirectiveMetadata(params.parsedStreamDirectives) || + containsCompleteMediaDirectiveLine(params.visibleDelta) || + params.parsedStreamDirectives.text !== params.visibleDelta + ) { + return undefined; + } + + if ( + !params.shouldUsePhaseAwareBlockReply && + params.previousCleaned === params.previousRawText.trim() + ) { + return params.next; + } + + const cleanedCandidate = `${params.previousCleaned}${params.parsedStreamDirectives.text}`.trim(); + return cleanedCandidate === params.next ? cleanedCandidate : undefined; +} + +export function resolveStreamingReplyText(params: { + evtType: "text_delta" | "text_start" | "text_end"; + next: string; + previousRawText: string; + previousCleaned: string; + visibleDelta: string; + parsedStreamDirectives: ReplyDirectiveParseResult | null; + shouldUsePhaseAwareBlockReply: boolean; +}): string { + if (!params.parsedStreamDirectives && params.evtType === "text_delta") { + return params.previousCleaned; + } + + return ( + resolveIncrementalStreamingReplyText(params) ?? + parseReplyDirectives( + params.evtType === "text_end" ? params.next : splitTrailingDirective(params.next).text, + ).text + ); +} + +/** Records parsed reply directives until a sendable reply payload is built. */ + +export function buildAssistantStreamData(params: { + text?: string; + delta?: string; + replace?: boolean; + mediaUrls?: string[]; + mediaUrl?: string; + phase?: AssistantPhase; + itemId?: string; +}): { + text: string; + delta: string; + replace?: true; + mediaUrls?: string[]; + phase?: AssistantPhase; + itemId?: string; +} { + const mediaUrls = resolveSendableOutboundReplyParts(params).mediaUrls; + return { + text: params.text ?? "", + delta: params.delta ?? "", + replace: params.replace ? true : undefined, + mediaUrls: mediaUrls.length ? mediaUrls : undefined, + phase: params.phase, + itemId: params.itemId, + }; +} + +/** Handles assistant message-start boundaries for streaming state. */ diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.test-helpers.ts b/src/agents/embedded-agent-subscribe.handlers.messages.test-helpers.ts new file mode 100644 index 000000000000..5be611d025d9 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.test-helpers.ts @@ -0,0 +1,221 @@ +import { vi } from "vitest"; +import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; +import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js"; +import { handleMessageEnd } from "./embedded-agent-subscribe.handlers.messages.lifecycle.js"; +import { handleMessageUpdate } from "./embedded-agent-subscribe.handlers.messages.update.js"; +import type { EmbeddedAgentSubscribeContext } from "./embedded-agent-subscribe.handlers.types.js"; +import { createThinkingTagStreamState } from "./embedded-agent-utils.js"; + +export function updateMessage( + context: EmbeddedAgentSubscribeContext, + event: { message: unknown; assistantMessageEvent?: unknown }, +) { + // Stream fixtures intentionally include incomplete and malformed provider payloads. + return handleMessageUpdate(context, { + type: "message_update", + ...event, + } as Parameters[1]); +} + +export function endMessage(context: EmbeddedAgentSubscribeContext, event: { message: unknown }) { + // Message-end coverage includes malformed content and partial provider usage. + return handleMessageEnd(context, { + type: "message_end", + ...event, + } as Parameters[1]); +} + +export function createMessageUpdateContext( + params: { + onAgentEvent?: ReturnType; + onPartialReply?: ReturnType; + flushBlockReplyBuffer?: ReturnType; + resetAssistantMessageState?: ReturnType; + debug?: ReturnType; + shouldEmitPartialReplies?: boolean; + sourceReplyDeliveryMode?: "automatic" | "message_tool_only"; + consumePartialReplyDirectives?: ReturnType; + stripBlockTags?: ReturnType; + emitReasoningStream?: ReturnType; + state?: Record; + } = {}, +) { + // Update context fixture wires the partial-reply path through the same + // directive accumulator used by streaming runtime events. + const partialReplyDirectiveAccumulator = createStreamingDirectiveAccumulator(); + const onAgentEvent = params.onAgentEvent as ((event: unknown) => void) | undefined; + const onPartialReply = params.onPartialReply as ((event: unknown) => void) | undefined; + return { + params: { + runId: "run-1", + session: { id: "session-1" }, + ...(params.sourceReplyDeliveryMode + ? { sourceReplyDeliveryMode: params.sourceReplyDeliveryMode } + : {}), + ...(params.onAgentEvent ? { onAgentEvent: params.onAgentEvent } : {}), + ...(params.onPartialReply ? { onPartialReply: params.onPartialReply } : {}), + }, + state: { + deterministicApprovalPromptPending: false, + deterministicApprovalPromptSent: false, + currentSourceMessagingToolSentTextsNormalized: [], + currentSourceMessagingToolHeldPartial: undefined, + reasoningStreamOpen: false, + streamReasoning: false, + deltaBuffer: "", + thinkingTagStream: createThinkingTagStreamState(), + blockBuffer: "", + partialBlockState: { + thinking: false, + final: false, + inlineCode: createInlineCodeState(), + }, + lastStreamedAssistant: undefined, + lastStreamedAssistantCleaned: undefined, + emittedAssistantUpdate: false, + shouldEmitPartialReplies: params.shouldEmitPartialReplies ?? true, + blockReplyBreak: "text_end", + assistantMessageIndex: 0, + lastAssistantStreamItemId: undefined, + assistantTexts: [], + pendingAssistantReplyDirectives: undefined, + ...params.state, + }, + log: { debug: params.debug ?? vi.fn() }, + noteLastAssistant: vi.fn(), + noteCompletedAssistant: vi.fn(), + stripBlockTags: params.stripBlockTags ?? vi.fn((text: string) => text), + consumePartialReplyDirectives: + params.consumePartialReplyDirectives ?? + vi.fn((text: string, options?: { final?: boolean }) => + partialReplyDirectiveAccumulator.consume(text, options), + ), + emitReasoningStream: params.emitReasoningStream ?? vi.fn(), + flushBlockReplyBuffer: params.flushBlockReplyBuffer ?? vi.fn(), + resetAssistantMessageState: params.resetAssistantMessageState ?? vi.fn(), + recordAssistantUsage: vi.fn(), + commitAssistantUsage: vi.fn(), + emitAssistantStreamData: vi.fn( + ( + data: Parameters[0], + options?: { emitPartialReply?: boolean }, + ) => { + onAgentEvent?.({ stream: "assistant", data }); + if (options?.emitPartialReply === true && (params.shouldEmitPartialReplies ?? true)) { + onPartialReply?.(data); + } + }, + ), + } as unknown as EmbeddedAgentSubscribeContext; +} + +export function createMessageEndContext( + params: { + onAgentEvent?: ReturnType; + onBlockReply?: ReturnType; + emitBlockReply?: ReturnType; + finalizeAssistantTexts?: ReturnType; + flushBlockReplyBuffer?: ReturnType; + consumeReplyDirectives?: ReturnType; + stripBlockTags?: ReturnType; + warn?: ReturnType; + builtinToolNames?: ReadonlySet; + sourceReplyDeliveryMode?: "automatic" | "message_tool_only"; + enforceFinalTag?: boolean; + blockChunker?: { hasBuffered: () => boolean; reset: () => void }; + state?: Record; + } = {}, +) { + // Message-end context starts with buffered assistant text so tests can assert + // final flushing, directive consumption, and source-reply behavior. + const onAgentEvent = params.onAgentEvent as ((event: unknown) => void) | undefined; + return { + params: { + runId: "run-1", + session: { id: "session-1" }, + ...(params.sourceReplyDeliveryMode + ? { sourceReplyDeliveryMode: params.sourceReplyDeliveryMode } + : {}), + ...(params.enforceFinalTag !== undefined ? { enforceFinalTag: params.enforceFinalTag } : {}), + ...(params.onAgentEvent ? { onAgentEvent: params.onAgentEvent } : {}), + ...(params.onBlockReply ? { onBlockReply: params.onBlockReply } : { onBlockReply: vi.fn() }), + }, + state: { + assistantTexts: [], + assistantTextBaseline: 0, + emittedAssistantUpdate: false, + deterministicApprovalPromptPending: false, + deterministicApprovalPromptSent: false, + messagingToolSentTexts: [], + messagingToolSentTextsNormalized: [], + currentSourceMessagingToolSentTextsNormalized: [], + currentSourceMessagingToolHeldPartial: undefined, + includeReasoning: false, + streamReasoning: false, + blockReplyBreak: "message_end", + deltaBuffer: "Need send.", + blockBuffer: "Need send.", + blockState: { + thinking: false, + final: false, + inlineCode: createInlineCodeState(), + }, + partialBlockState: { + thinking: false, + final: false, + inlineCode: createInlineCodeState(), + }, + lastStreamedAssistant: undefined, + lastStreamedAssistantCleaned: undefined, + lastReasoningSent: undefined, + reasoningStreamOpen: false, + ...params.state, + }, + noteLastAssistant: vi.fn(), + noteCompletedAssistant: vi.fn(), + recordAssistantUsage: vi.fn(), + commitAssistantUsage: vi.fn(), + log: { debug: vi.fn(), info: vi.fn(), warn: params.warn ?? vi.fn() }, + builtinToolNames: params.builtinToolNames, + stripBlockTags: params.stripBlockTags ?? vi.fn((text: string) => text), + finalizeAssistantTexts: params.finalizeAssistantTexts ?? vi.fn(), + emitAssistantStreamData: vi.fn( + (data: Parameters[0]) => { + onAgentEvent?.({ stream: "assistant", data }); + }, + ), + emitBlockReply: params.emitBlockReply ?? vi.fn(), + consumeReplyDirectives: params.consumeReplyDirectives ?? vi.fn(() => ({ text: "Need send." })), + emitReasoningStream: vi.fn(), + flushBlockReplyBuffer: params.flushBlockReplyBuffer ?? vi.fn(), + blockChunker: params.blockChunker ?? null, + } as unknown as EmbeddedAgentSubscribeContext; +} + +export function firstMockCall(mock: { mock: { calls: unknown[][] } }, label: string): unknown[] { + const call = mock.mock.calls[0]; + if (!call) { + throw new Error(`Expected ${label} to be called`); + } + return call; +} + +export function firstMockArg(mock: { mock: { calls: unknown[][] } }, label: string): unknown { + return firstMockCall(mock, label)[0]; +} + +export function createMessageToolEnvelope( + message: string, + args: Record = {}, +): string { + // Messaging tool envelopes mimic provider tool-call JSON used by fallback + // reply extraction when the assistant otherwise says NO_REPLY. + return JSON.stringify({ + name: "message", + arguments: { + action: "send", + message, + ...args, + }, + }); +} diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.test-support.ts b/src/agents/embedded-agent-subscribe.handlers.messages.test-support.ts index f7850b6d8bb7..03f46de2c982 100644 --- a/src/agents/embedded-agent-subscribe.handlers.messages.test-support.ts +++ b/src/agents/embedded-agent-subscribe.handlers.messages.test-support.ts @@ -1,6 +1,6 @@ import type { ReplyDirectiveParseResult } from "../auto-reply/reply/reply-directives.js"; import type { AssistantPhase } from "../shared/chat-message-content.js"; -import "./embedded-agent-subscribe.handlers.messages.js"; +import "./embedded-agent-subscribe.handlers.messages.update.js"; import type { EmbeddedAgentSubscribeState } from "./embedded-agent-subscribe.handlers.types.js"; type AssistantStreamDataParams = { diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.test.ts deleted file mode 100644 index 25516f954a13..000000000000 --- a/src/agents/embedded-agent-subscribe.handlers.messages.test.ts +++ /dev/null @@ -1,2310 +0,0 @@ -// Message handler tests cover assistant stream payloads, partial replies, -// block replies, directives, media, and message-tool reply suppression. -import { describe, expect, it, vi } from "vitest"; -import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; -import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js"; -import { - consumePendingAssistantReplyDirectivesIntoReply, - consumePendingToolMediaIntoReply, - consumePendingToolMediaReply, - handleMessageEnd, - handleMessageUpdate, - hasAssistantVisibleReply, - readPendingToolMediaReply, -} from "./embedded-agent-subscribe.handlers.messages.js"; -import { - buildAssistantStreamData, - recordPendingAssistantReplyDirectives, - resolveCurrentSourceMessagingToolPartial, -} from "./embedded-agent-subscribe.handlers.messages.test-support.js"; -import type { EmbeddedAgentSubscribeContext } from "./embedded-agent-subscribe.handlers.types.js"; -import { - createOpenAiResponsesPartial, - createOpenAiResponsesTextBlock, - createOpenAiResponsesTextEvent as createTextUpdateEvent, -} from "./embedded-agent-subscribe.openai-responses.test-helpers.js"; -import { createThinkingTagStreamState } from "./embedded-agent-utils.js"; - -function updateMessage( - context: EmbeddedAgentSubscribeContext, - event: { message: unknown; assistantMessageEvent?: unknown }, -) { - // Stream fixtures intentionally include incomplete and malformed provider payloads. - return handleMessageUpdate(context, { - type: "message_update", - ...event, - } as Parameters[1]); -} - -function endMessage(context: EmbeddedAgentSubscribeContext, event: { message: unknown }) { - // Message-end coverage includes malformed content and partial provider usage. - return handleMessageEnd(context, { - type: "message_end", - ...event, - } as Parameters[1]); -} - -function createMessageUpdateContext( - params: { - onAgentEvent?: ReturnType; - onPartialReply?: ReturnType; - flushBlockReplyBuffer?: ReturnType; - resetAssistantMessageState?: ReturnType; - debug?: ReturnType; - shouldEmitPartialReplies?: boolean; - sourceReplyDeliveryMode?: "automatic" | "message_tool_only"; - consumePartialReplyDirectives?: ReturnType; - stripBlockTags?: ReturnType; - emitReasoningStream?: ReturnType; - state?: Record; - } = {}, -) { - // Update context fixture wires the partial-reply path through the same - // directive accumulator used by streaming runtime events. - const partialReplyDirectiveAccumulator = createStreamingDirectiveAccumulator(); - const onAgentEvent = params.onAgentEvent as ((event: unknown) => void) | undefined; - const onPartialReply = params.onPartialReply as ((event: unknown) => void) | undefined; - return { - params: { - runId: "run-1", - session: { id: "session-1" }, - ...(params.sourceReplyDeliveryMode - ? { sourceReplyDeliveryMode: params.sourceReplyDeliveryMode } - : {}), - ...(params.onAgentEvent ? { onAgentEvent: params.onAgentEvent } : {}), - ...(params.onPartialReply ? { onPartialReply: params.onPartialReply } : {}), - }, - state: { - deterministicApprovalPromptPending: false, - deterministicApprovalPromptSent: false, - currentSourceMessagingToolSentTextsNormalized: [], - currentSourceMessagingToolHeldPartial: undefined, - reasoningStreamOpen: false, - streamReasoning: false, - deltaBuffer: "", - thinkingTagStream: createThinkingTagStreamState(), - blockBuffer: "", - partialBlockState: { - thinking: false, - final: false, - inlineCode: createInlineCodeState(), - }, - lastStreamedAssistant: undefined, - lastStreamedAssistantCleaned: undefined, - emittedAssistantUpdate: false, - shouldEmitPartialReplies: params.shouldEmitPartialReplies ?? true, - blockReplyBreak: "text_end", - assistantMessageIndex: 0, - lastAssistantStreamItemId: undefined, - assistantTexts: [], - pendingAssistantReplyDirectives: undefined, - ...params.state, - }, - log: { debug: params.debug ?? vi.fn() }, - noteLastAssistant: vi.fn(), - noteCompletedAssistant: vi.fn(), - stripBlockTags: params.stripBlockTags ?? vi.fn((text: string) => text), - consumePartialReplyDirectives: - params.consumePartialReplyDirectives ?? - vi.fn((text: string, options?: { final?: boolean }) => - partialReplyDirectiveAccumulator.consume(text, options), - ), - emitReasoningStream: params.emitReasoningStream ?? vi.fn(), - flushBlockReplyBuffer: params.flushBlockReplyBuffer ?? vi.fn(), - resetAssistantMessageState: params.resetAssistantMessageState ?? vi.fn(), - recordAssistantUsage: vi.fn(), - commitAssistantUsage: vi.fn(), - emitAssistantStreamData: vi.fn( - ( - data: Parameters[0], - options?: { emitPartialReply?: boolean }, - ) => { - onAgentEvent?.({ stream: "assistant", data }); - if (options?.emitPartialReply === true && (params.shouldEmitPartialReplies ?? true)) { - onPartialReply?.(data); - } - }, - ), - } as unknown as EmbeddedAgentSubscribeContext; -} - -function createMessageEndContext( - params: { - onAgentEvent?: ReturnType; - onBlockReply?: ReturnType; - emitBlockReply?: ReturnType; - finalizeAssistantTexts?: ReturnType; - flushBlockReplyBuffer?: ReturnType; - consumeReplyDirectives?: ReturnType; - stripBlockTags?: ReturnType; - warn?: ReturnType; - builtinToolNames?: ReadonlySet; - sourceReplyDeliveryMode?: "automatic" | "message_tool_only"; - enforceFinalTag?: boolean; - blockChunker?: { hasBuffered: () => boolean; reset: () => void }; - state?: Record; - } = {}, -) { - // Message-end context starts with buffered assistant text so tests can assert - // final flushing, directive consumption, and source-reply behavior. - const onAgentEvent = params.onAgentEvent as ((event: unknown) => void) | undefined; - return { - params: { - runId: "run-1", - session: { id: "session-1" }, - ...(params.sourceReplyDeliveryMode - ? { sourceReplyDeliveryMode: params.sourceReplyDeliveryMode } - : {}), - ...(params.enforceFinalTag !== undefined ? { enforceFinalTag: params.enforceFinalTag } : {}), - ...(params.onAgentEvent ? { onAgentEvent: params.onAgentEvent } : {}), - ...(params.onBlockReply ? { onBlockReply: params.onBlockReply } : { onBlockReply: vi.fn() }), - }, - state: { - assistantTexts: [], - assistantTextBaseline: 0, - emittedAssistantUpdate: false, - deterministicApprovalPromptPending: false, - deterministicApprovalPromptSent: false, - messagingToolSentTexts: [], - messagingToolSentTextsNormalized: [], - currentSourceMessagingToolSentTextsNormalized: [], - currentSourceMessagingToolHeldPartial: undefined, - includeReasoning: false, - streamReasoning: false, - blockReplyBreak: "message_end", - deltaBuffer: "Need send.", - blockBuffer: "Need send.", - blockState: { - thinking: false, - final: false, - inlineCode: createInlineCodeState(), - }, - partialBlockState: { - thinking: false, - final: false, - inlineCode: createInlineCodeState(), - }, - lastStreamedAssistant: undefined, - lastStreamedAssistantCleaned: undefined, - lastReasoningSent: undefined, - reasoningStreamOpen: false, - ...params.state, - }, - noteLastAssistant: vi.fn(), - noteCompletedAssistant: vi.fn(), - recordAssistantUsage: vi.fn(), - commitAssistantUsage: vi.fn(), - log: { debug: vi.fn(), info: vi.fn(), warn: params.warn ?? vi.fn() }, - builtinToolNames: params.builtinToolNames, - stripBlockTags: params.stripBlockTags ?? vi.fn((text: string) => text), - finalizeAssistantTexts: params.finalizeAssistantTexts ?? vi.fn(), - emitAssistantStreamData: vi.fn( - (data: Parameters[0]) => { - onAgentEvent?.({ stream: "assistant", data }); - }, - ), - emitBlockReply: params.emitBlockReply ?? vi.fn(), - consumeReplyDirectives: params.consumeReplyDirectives ?? vi.fn(() => ({ text: "Need send." })), - emitReasoningStream: vi.fn(), - flushBlockReplyBuffer: params.flushBlockReplyBuffer ?? vi.fn(), - blockChunker: params.blockChunker ?? null, - } as unknown as EmbeddedAgentSubscribeContext; -} - -function firstMockCall(mock: { mock: { calls: unknown[][] } }, label: string): unknown[] { - const call = mock.mock.calls[0]; - if (!call) { - throw new Error(`Expected ${label} to be called`); - } - return call; -} - -function firstMockArg(mock: { mock: { calls: unknown[][] } }, label: string): unknown { - return firstMockCall(mock, label)[0]; -} - -function createMessageToolEnvelope(message: string, args: Record = {}): string { - // Messaging tool envelopes mimic provider tool-call JSON used by fallback - // reply extraction when the assistant otherwise says NO_REPLY. - return JSON.stringify({ - name: "message", - arguments: { - action: "send", - message, - ...args, - }, - }); -} - -describe("hasAssistantVisibleReply", () => { - it("treats audio-only payloads as visible", () => { - expect(hasAssistantVisibleReply({ audioAsVoice: true })).toBe(true); - }); - - it("detects text or media visibility", () => { - expect(hasAssistantVisibleReply({ text: "hello" })).toBe(true); - expect(hasAssistantVisibleReply({ mediaUrls: ["https://example.com/a.png"] })).toBe(true); - expect(hasAssistantVisibleReply({})).toBe(false); - }); -}); - -describe("buildAssistantStreamData", () => { - it("normalizes media payloads for assistant stream events", () => { - expect( - buildAssistantStreamData({ - text: "hello", - delta: "he", - replace: true, - mediaUrl: "https://example.com/a.png", - phase: "final_answer", - }), - ).toEqual({ - text: "hello", - delta: "he", - replace: true, - mediaUrls: ["https://example.com/a.png"], - phase: "final_answer", - }); - }); -}); - -describe("pending assistant reply directives", () => { - it("merges directive metadata into the next non-reasoning block reply", () => { - const state = { pendingAssistantReplyDirectives: undefined }; - - recordPendingAssistantReplyDirectives(state, { - text: "", - mediaUrls: ["/tmp/reply.ogg"], - replyToCurrent: true, - replyToTag: true, - audioAsVoice: true, - isSilent: false, - }); - - expect( - consumePendingAssistantReplyDirectivesIntoReply(state, { - text: "Done.", - }), - ).toEqual({ - text: "Done.", - mediaUrls: ["/tmp/reply.ogg"], - audioAsVoice: true, - replyToId: undefined, - replyToTag: true, - replyToCurrent: true, - }); - expect(state.pendingAssistantReplyDirectives).toBeUndefined(); - }); - - it("does not consume pending directive metadata on reasoning replies", () => { - const state = { - pendingAssistantReplyDirectives: { - mediaUrls: ["/tmp/reply.png"], - }, - }; - - expect( - consumePendingAssistantReplyDirectivesIntoReply(state, { - text: "Thinking...", - isReasoning: true, - }), - ).toEqual({ - text: "Thinking...", - isReasoning: true, - }); - expect(state.pendingAssistantReplyDirectives?.mediaUrls).toEqual(["/tmp/reply.png"]); - }); -}); - -describe("handleMessageUpdate current-source message-tool previews", () => { - it("holds delta-only continuation fragments and releases one full divergent snapshot", () => { - const state = { - currentSourceMessagingToolHeldPartial: undefined as string | undefined, - currentSourceMessagingToolSentTextsNormalized: ["qa-msteams-dm-ok"], - }; - - expect( - resolveCurrentSourceMessagingToolPartial(state, { - evtType: "text_delta", - text: "QA-MSTEAMS", - visibleDelta: "QA-MSTEAMS", - }), - ).toEqual({ hold: true, text: "QA-MSTEAMS" }); - expect( - resolveCurrentSourceMessagingToolPartial(state, { - evtType: "text_delta", - text: "-DM-OK", - visibleDelta: "-DM-OK", - }), - ).toEqual({ hold: true, text: "QA-MSTEAMS-DM-OK" }); - expect( - resolveCurrentSourceMessagingToolPartial(state, { - evtType: "text_delta", - text: " with more detail", - visibleDelta: " with more detail", - }), - ).toEqual({ hold: false, text: "QA-MSTEAMS-DM-OK with more detail" }); - expect(state.currentSourceMessagingToolHeldPartial).toBeUndefined(); - }); - - it("holds automatic partial prefixes and exact duplicates after source delivery", () => { - const onAgentEvent = vi.fn(); - const onPartialReply = vi.fn(); - const sentText = "QA-MSTEAMS-DM-OK"; - const context = createMessageUpdateContext({ - onAgentEvent, - onPartialReply, - sourceReplyDeliveryMode: "automatic", - state: { - currentSourceMessagingToolSentTextsNormalized: [sentText.toLowerCase()], - }, - }); - - updateMessage( - context, - createTextUpdateEvent({ - type: "text_delta", - text: "QA-MSTEAMS", - id: "msg_source_duplicate", - }), - ); - updateMessage( - context, - createTextUpdateEvent({ - type: "text_end", - text: sentText, - id: "msg_source_duplicate", - }), - ); - - expect(onAgentEvent).toHaveBeenCalledTimes(1); - expect(onPartialReply).not.toHaveBeenCalled(); - }); - - it("releases the full cumulative snapshot when automatic text diverges", () => { - const onPartialReply = vi.fn(); - const sentText = "QA-MSTEAMS-DM-OK"; - const context = createMessageUpdateContext({ - onPartialReply, - sourceReplyDeliveryMode: "automatic", - state: { - currentSourceMessagingToolSentTextsNormalized: [sentText.toLowerCase()], - }, - }); - - updateMessage( - context, - createTextUpdateEvent({ - type: "text_delta", - text: "QA-MSTEAMS", - id: "msg_source_diverges", - }), - ); - updateMessage( - context, - createTextUpdateEvent({ - type: "text_end", - text: `${sentText} with more detail`, - id: "msg_source_diverges", - }), - ); - - expect(onPartialReply).toHaveBeenCalledTimes(1); - expect(onPartialReply).toHaveBeenCalledWith( - expect.objectContaining({ text: `${sentText} with more detail` }), - ); - }); - - it("keeps unrelated automatic partial text visible", () => { - const onPartialReply = vi.fn(); - const context = createMessageUpdateContext({ - onPartialReply, - sourceReplyDeliveryMode: "automatic", - state: { - currentSourceMessagingToolSentTextsNormalized: ["qa-msteams-dm-ok"], - }, - }); - - updateMessage( - context, - createTextUpdateEvent({ - type: "text_end", - text: "A genuinely different answer", - id: "msg_source_different", - }), - ); - - expect(onPartialReply).toHaveBeenCalledWith( - expect.objectContaining({ text: "A genuinely different answer" }), - ); - }); -}); - -describe("handleMessageUpdate text signatures", () => { - it("emits the full incrementally extracted reasoning value on every delta", () => { - const emitReasoningStream = vi.fn(); - const context = createMessageUpdateContext({ emitReasoningStream }); - - for (const chunk of ["reason", "ing"]) { - updateMessage( - context, - createTextUpdateEvent({ type: "text_delta", text: chunk, delta: chunk }), - ); - } - - expect(emitReasoningStream.mock.calls.map(([text]) => text)).toEqual([ - "", - "reason", - "reasoning", - ]); - }); - - it("uses incremental text deltas for unphased OpenAI Responses streams", () => { - const onAgentEvent = vi.fn(); - const stripBlockTags = vi.fn((text: string) => text); - const context = createMessageUpdateContext({ onAgentEvent, stripBlockTags }); - - const createNonPhaseEvent = (text: string, delta: string) => - ({ - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - contentIndex: 0, - delta, - partial: { - role: "assistant", - content: [{ type: "text", text }], - stopReason: "stop", - api: "openai-responses", - provider: "openai", - model: "gpt-5.2", - usage: {}, - timestamp: 0, - }, - }, - }) as never; - - updateMessage(context, createNonPhaseEvent("Hello ", "Hello ")); - updateMessage(context, createNonPhaseEvent("Hello world", "world")); - - expect(stripBlockTags.mock.calls.map(([text]) => text)).toEqual(["Hello ", "world"]); - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - { - stream: "assistant", - data: { text: "Hello", delta: "Hello" }, - }, - { - stream: "assistant", - data: { text: "Hello world", delta: " world" }, - }, - ]); - }); - - it("treats unphased OpenAI Responses content-index changes as message boundaries", () => { - const flushBlockReplyBuffer = vi.fn(); - const onAssistantMessageStart = vi.fn(); - const onPartialReply = vi.fn(); - const context = createMessageUpdateContext({ - flushBlockReplyBuffer, - onPartialReply, - state: { - deltaBuffer: "First block", - lastStreamedAssistant: "First block", - lastStreamedAssistantCleaned: "First block", - lastAssistantStreamContentIndex: 0, - }, - }); - const resetAssistantMessageState = vi.fn(() => { - context.state.deltaBuffer = ""; - context.state.lastStreamedAssistant = undefined; - context.state.lastStreamedAssistantCleaned = undefined; - }); - context.resetAssistantMessageState = resetAssistantMessageState; - context.params.onAssistantMessageStart = onAssistantMessageStart; - - updateMessage(context, { - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_end", - contentIndex: 1, - content: "First block", - partial: { - role: "assistant", - content: [ - { type: "text", text: "First block" }, - { type: "text", text: "First block" }, - ], - api: "openai-responses", - }, - }, - }); - - expect(flushBlockReplyBuffer).toHaveBeenCalledTimes(1); - expect(resetAssistantMessageState).toHaveBeenCalledTimes(1); - expect(onAssistantMessageStart).toHaveBeenCalledTimes(1); - expect(onPartialReply).toHaveBeenCalledWith( - expect.objectContaining({ text: "First block", delta: "First block" }), - ); - expect(context.state.blockBuffer).toBe("First block"); - expect(context.state.lastAssistantStreamContentIndex).toBe(1); - }); - - it("holds incomplete streaming directive tails without emitting them as text", () => { - const onAgentEvent = vi.fn(); - const accumulator = createStreamingDirectiveAccumulator(); - const context = createMessageUpdateContext({ - onAgentEvent, - consumePartialReplyDirectives: vi.fn((text: string, options?: { final?: boolean }) => - accumulator.consume(text, options), - ), - }); - - const createNonPhaseEvent = (delta: string) => - ({ - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - delta, - }, - }) as never; - - updateMessage(context, createNonPhaseEvent("Hello\n")); - updateMessage(context, createNonPhaseEvent("M")); - - expect(onAgentEvent).toHaveBeenCalledTimes(1); - expect(firstMockArg(onAgentEvent, "agent event")).toMatchObject({ - stream: "assistant", - data: { text: "Hello", delta: "Hello" }, - }); - expect(context.state.lastStreamedAssistantCleaned).toBe("Hello"); - }); - - it.each([ - { - name: "the directive accumulator has no parsed result", - text: "answer part A msg [[E1008]timeout] answer part B", - hasParsedDirectives: false, - }, - { - name: "the directive accumulator flushes a buffered tail", - text: "answer part A msg [[E1008]timeout] answer part B", - hasParsedDirectives: true, - }, - { - name: "the final text ends with one bracket", - text: "answer part A [", - hasParsedDirectives: true, - }, - ])("keeps literal final text when $name", ({ text, hasParsedDirectives }) => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ - onAgentEvent, - ...(hasParsedDirectives ? {} : { consumePartialReplyDirectives: vi.fn(() => null) }), - }); - - updateMessage(context, { - message: { role: "assistant", content: [] }, - assistantMessageEvent: { type: "text_end", content: text }, - }); - - expect(context.state.lastStreamedAssistantCleaned).toBe(text); - expect(firstMockArg(onAgentEvent, "final assistant event")).toMatchObject({ - stream: "assistant", - data: { text }, - }); - }); - - it("keeps stripped reply directives out of later plain deltas", () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - - const createNonPhaseEvent = (delta: string) => - ({ - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - delta, - }, - }) as never; - - updateMessage(context, createNonPhaseEvent("[[reply_to_current]]\nHello")); - updateMessage(context, createNonPhaseEvent(" world")); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - { - stream: "assistant", - data: { text: "Hello", delta: "Hello" }, - }, - { - stream: "assistant", - data: { text: "Hello world", delta: " world" }, - }, - ]); - }); - - it("does not expose complete legacy media directives on plain deltas", () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - - updateMessage(context, { - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - delta: "Here it is.\nMEDIA:/tmp/final.png\n", - }, - }); - - expect(firstMockArg(onAgentEvent, "agent event")).toMatchObject({ - stream: "assistant", - data: { text: "Here it is.", delta: "Here it is." }, - }); - }); - - it("uses full partial text for suffix deltas after a suppressed commentary item", () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - - updateMessage( - context, - createTextUpdateEvent({ - type: "text_delta", - text: "Hello", - delta: "Hello", - id: "item-commentary", - signaturePhase: "commentary", - partialPhase: "commentary", - }), - ); - updateMessage( - context, - createTextUpdateEvent({ - type: "text_delta", - text: "Hello world", - delta: " world", - id: "item-final", - signaturePhase: "final_answer", - partialPhase: "final_answer", - }), - ); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - // Emit-always: the commentary delta reaches the bus tagged with its - // phase; reply lanes still exclude it (covered below). - { - stream: "assistant", - data: { delta: "Hello", phase: "commentary", itemId: "item-commentary" }, - }, - { - stream: "assistant", - data: { text: "Hello world", delta: "Hello world", phase: "final_answer" }, - }, - ]); - }); - - it.each([ - "openai-responses", - "openai-chatgpt-responses", - "openclaw-openai-responses-transport", - "openclaw-openai-chatgpt-responses-transport", - "openclaw-azure-openai-responses-transport", - ])("streams %s commentary bytes exactly once across start, deltas, and end", async (api) => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - const createPartial = (text: string) => ({ - ...createOpenAiResponsesPartial({ - text, - id: "item-commentary", - signaturePhase: "commentary", - partialPhase: "commentary", - }), - api, - }); - const startPartial = createPartial("Work"); - const finalPartial = createPartial("Working..."); - - updateMessage(context, { - message: startPartial, - assistantMessageEvent: { - type: "text_start", - contentIndex: 0, - partial: startPartial, - }, - }); - updateMessage(context, { - message: startPartial, - assistantMessageEvent: { - type: "text_delta", - contentIndex: 0, - delta: "Work", - partial: startPartial, - }, - }); - updateMessage(context, { - message: finalPartial, - assistantMessageEvent: { - type: "text_delta", - contentIndex: 0, - delta: "ing...", - partial: finalPartial, - }, - }); - updateMessage(context, { - message: finalPartial, - assistantMessageEvent: { - type: "text_end", - contentIndex: 0, - content: "Working...", - partial: finalPartial, - }, - }); - await endMessage(context, { - message: finalPartial, - }); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - { - stream: "assistant", - data: { delta: "Work", phase: "commentary", itemId: "item-commentary" }, - }, - { - stream: "assistant", - data: { delta: "ing...", phase: "commentary", itemId: "item-commentary" }, - }, - ]); - expect(context.state.deltaBuffer).toBe("Working..."); - expect(context.state.blockBuffer).toBe(""); - }); - - it("keeps same-index commentary snapshot extensions on the original live item key", async () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - const createPartial = (text: string, id: string) => - createOpenAiResponsesPartial({ - text, - id, - signaturePhase: "commentary", - partialPhase: "commentary", - }); - const firstPartial = createPartial("Working", "item-1"); - const extendedPartial = createPartial("Working now", "item-2"); - - updateMessage(context, { - message: firstPartial, - assistantMessageEvent: { type: "text_start", contentIndex: 0, partial: firstPartial }, - }); - updateMessage(context, { - message: firstPartial, - assistantMessageEvent: { - type: "text_end", - contentIndex: 0, - content: "Working", - partial: firstPartial, - }, - }); - updateMessage(context, { - message: extendedPartial, - assistantMessageEvent: { - type: "text_end", - contentIndex: 0, - content: "Working now", - partial: extendedPartial, - }, - }); - await endMessage(context, { message: extendedPartial }); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - { - stream: "assistant", - data: { delta: "Working", phase: "commentary", itemId: "item-1" }, - }, - { - stream: "assistant", - data: { delta: " now", phase: "commentary", itemId: "item-1" }, - }, - ]); - expect(context.state.lastAssistantStreamItemId).toBe("item-1"); - expect(context.state.deltaBuffer).toBe("Working now"); - }); - - it("emits a commentary snapshot when Anthropic text is classified after deltas", () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - const narration = "I'll check the repo first."; - const commentaryPartial = { - role: "assistant", - api: "anthropic-messages", - content: [ - { - type: "text", - text: narration, - textSignature: JSON.stringify({ v: 1, id: "commentary-0", phase: "commentary" }), - }, - ], - }; - - updateMessage(context, { - message: { - role: "assistant", - api: "anthropic-messages", - content: [{ type: "text", text: narration }], - }, - assistantMessageEvent: { type: "text_delta", delta: narration }, - }); - updateMessage(context, { - message: { role: "assistant", api: "anthropic-messages", content: [] }, - assistantMessageEvent: { - type: "text_end", - content: narration, - partial: commentaryPartial, - }, - }); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toContainEqual( - expect.objectContaining({ - stream: "assistant", - data: expect.objectContaining({ - text: narration, - replace: true, - phase: "commentary", - itemId: "commentary-0", - }), - }), - ); - }); - - it("uses incremental deltas for same-item phased streams", () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - const signature = JSON.stringify({ v: 1, id: "item-final", phase: "final_answer" }); - const partial = { - role: "assistant", - phase: "final_answer", - content: [ - { - type: "text", - textSignature: signature, - get text() { - throw new Error("full partial text should not be read"); - }, - }, - ], - }; - - const createPhasedDelta = (delta: string) => - ({ - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - delta, - partial, - }, - }) as never; - - updateMessage(context, createPhasedDelta("Hello")); - updateMessage(context, createPhasedDelta(" world")); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - { - stream: "assistant", - data: { text: "Hello", delta: "Hello", phase: "final_answer" }, - }, - { - stream: "assistant", - data: { text: "Hello world", delta: " world", phase: "final_answer" }, - }, - ]); - }); - - it("keeps same-item phased stream deltas on the user-visible sanitizer path", () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - const signature = JSON.stringify({ v: 1, id: "item-final", phase: "final_answer" }); - const partial = { - role: "assistant", - phase: "final_answer", - content: [ - { - type: "text", - textSignature: signature, - get text() { - throw new Error("full partial text should not be read"); - }, - }, - ], - }; - - const createPhasedDelta = (delta: string) => - ({ - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - delta, - partial, - }, - }) as never; - - updateMessage(context, createPhasedDelta("Visible\n{")); - updateMessage( - context, - createPhasedDelta('"name":"read","arguments":{"file_path":"secret.md"}}'), - ); - updateMessage(context, createPhasedDelta("\nDone.")); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - { - stream: "assistant", - data: { text: "Visible", delta: "Visible", phase: "final_answer" }, - }, - { - stream: "assistant", - data: { text: "Visible\n\nDone.", delta: "\n\nDone.", phase: "final_answer" }, - }, - ]); - }); - - it("keeps sanitizer context when a same-item phased stream starts hidden", () => { - const onAgentEvent = vi.fn(); - const context = createMessageUpdateContext({ onAgentEvent }); - const signature = JSON.stringify({ v: 1, id: "item-final", phase: "final_answer" }); - const partial = { - role: "assistant", - phase: "final_answer", - content: [ - { - type: "text", - textSignature: signature, - get text() { - throw new Error("full partial text should not be read"); - }, - }, - ], - }; - - const createPhasedDelta = (delta: string) => - ({ - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - delta, - partial, - }, - }) as never; - - updateMessage(context, createPhasedDelta("{")); - updateMessage( - context, - createPhasedDelta('"name":"read","arguments":{"file_path":"secret.md"}}\nDone.'), - ); - - expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ - { - stream: "assistant", - data: { text: "Done.", delta: "Done.", phase: "final_answer" }, - }, - ]); - }); - - it("treats phased textSignature item changes as assistant-message boundaries", () => { - const flushBlockReplyBuffer = vi.fn(); - const resetAssistantMessageState = vi.fn(); - const onAssistantMessageStart = vi.fn(); - const onPartialReply = vi.fn(); - const context = createMessageUpdateContext({ - flushBlockReplyBuffer, - resetAssistantMessageState, - onPartialReply, - }); - context.params.onAssistantMessageStart = onAssistantMessageStart; - context.state.lastAssistantStreamContentIndex = 0; - context.state.lastAssistantStreamItemId = "item-1"; - context.state.assistantMessageIndex = 7; - - updateMessage(context, { - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - contentIndex: 1, - delta: "Second block", - partial: { - role: "assistant", - phase: "final_answer", - content: [ - createOpenAiResponsesTextBlock({ - text: "First block", - id: "item-1", - phase: "final_answer", - }), - createOpenAiResponsesTextBlock({ - text: "Second block", - id: "item-2", - phase: "final_answer", - }), - ], - stopReason: "stop", - api: "openai-responses", - provider: "openai", - model: "gpt-5.2", - usage: {}, - timestamp: 0, - }, - }, - }); - - expect(flushBlockReplyBuffer).toHaveBeenCalledWith({ assistantMessageIndex: 7 }); - expect(resetAssistantMessageState).toHaveBeenCalledWith(0); - expect(onAssistantMessageStart).toHaveBeenCalledTimes(1); - expect(onPartialReply).toHaveBeenCalledWith( - expect.objectContaining({ - text: "Second block", - delta: "Second block", - phase: "final_answer", - }), - ); - expect(onPartialReply).not.toHaveBeenCalledWith( - expect.objectContaining({ text: "First block\nSecond block" }), - ); - expect(context.state.lastAssistantStreamContentIndex).toBe(1); - expect(context.state.lastAssistantStreamItemId).toBe("item-2"); - }); - - it("does not replay a deferred item snapshot before its first delta", () => { - const flushBlockReplyBuffer = vi.fn(); - const resetAssistantMessageState = vi.fn(); - const onAssistantMessageStart = vi.fn(); - const onPartialReply = vi.fn(); - const context = createMessageUpdateContext({ - flushBlockReplyBuffer, - resetAssistantMessageState, - onPartialReply, - state: { - lastAssistantStreamContentIndex: 0, - lastAssistantStreamItemId: "item-1", - }, - }); - context.params.onAssistantMessageStart = onAssistantMessageStart; - const partial = { - role: "assistant", - phase: "final_answer", - content: [ - createOpenAiResponsesTextBlock({ - text: "First block", - id: "item-1", - phase: "final_answer", - }), - createOpenAiResponsesTextBlock({ - text: "Second block", - id: "item-2", - phase: "final_answer", - }), - ], - api: "openai-responses", - }; - - updateMessage(context, { - message: partial, - assistantMessageEvent: { - type: "text_start", - contentIndex: 1, - partial, - }, - }); - updateMessage(context, { - message: partial, - assistantMessageEvent: { - type: "text_delta", - contentIndex: 1, - delta: "Second block", - }, - }); - - expect(flushBlockReplyBuffer).toHaveBeenCalledTimes(1); - expect(resetAssistantMessageState).toHaveBeenCalledTimes(1); - expect(onAssistantMessageStart).toHaveBeenCalledTimes(1); - expect(onPartialReply).toHaveBeenCalledTimes(1); - expect(onPartialReply).toHaveBeenCalledWith( - expect.objectContaining({ - text: "Second block", - delta: "Second block", - phase: "final_answer", - }), - ); - }); - - it("keeps same-block OpenAI Responses snapshot extensions in one assistant message", () => { - const flushBlockReplyBuffer = vi.fn(); - const resetAssistantMessageState = vi.fn(); - const onAssistantMessageStart = vi.fn(); - const onPartialReply = vi.fn(); - const context = createMessageUpdateContext({ - flushBlockReplyBuffer, - resetAssistantMessageState, - onPartialReply, - state: { - deltaBuffer: "First block", - lastStreamedAssistant: "First block", - lastStreamedAssistantCleaned: "First block", - lastAssistantStreamContentIndex: 0, - lastAssistantStreamItemId: "item-1", - }, - }); - context.params.onAssistantMessageStart = onAssistantMessageStart; - - updateMessage(context, { - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_end", - contentIndex: 0, - content: "First block extended", - partial: createOpenAiResponsesPartial({ - text: "First block extended", - id: "item-2", - signaturePhase: "final_answer", - partialPhase: "final_answer", - }), - }, - }); - - expect(flushBlockReplyBuffer).not.toHaveBeenCalled(); - expect(resetAssistantMessageState).not.toHaveBeenCalled(); - expect(onAssistantMessageStart).not.toHaveBeenCalled(); - expect(onPartialReply).toHaveBeenCalledWith( - expect.objectContaining({ - text: "First block extended", - delta: " extended", - phase: "final_answer", - }), - ); - expect(context.state.lastAssistantStreamContentIndex).toBe(0); - expect(context.state.lastAssistantStreamItemId).toBe("item-1"); - }); - - it("scopes item-id fallback boundaries to the matching signed block", () => { - const onPartialReply = vi.fn(); - const resetAssistantMessageState = vi.fn(); - const context = createMessageUpdateContext({ - onPartialReply, - resetAssistantMessageState, - state: { lastAssistantStreamItemId: "item-1" }, - }); - - updateMessage(context, { - message: { role: "assistant", content: [] }, - assistantMessageEvent: { - type: "text_delta", - delta: "Second block", - partial: { - role: "assistant", - phase: "final_answer", - content: [ - createOpenAiResponsesTextBlock({ - text: "First block", - id: "item-1", - phase: "final_answer", - }), - createOpenAiResponsesTextBlock({ - text: "Second block", - id: "item-2", - phase: "final_answer", - }), - ], - api: "openai-responses", - }, - }, - }); - - expect(resetAssistantMessageState).toHaveBeenCalledTimes(1); - expect(onPartialReply).toHaveBeenCalledWith( - expect.objectContaining({ - text: "Second block", - delta: "Second block", - phase: "final_answer", - }), - ); - expect(onPartialReply).not.toHaveBeenCalledWith( - expect.objectContaining({ text: "First block\nSecond block" }), - ); - expect(context.state.lastAssistantStreamContentIndex).toBeUndefined(); - expect(context.state.lastAssistantStreamItemId).toBe("item-2"); - }); - - it("preserves phase-aware voice and reply directives while deferring final media delivery", () => { - const accumulator = createStreamingDirectiveAccumulator(); - const ctx = createMessageUpdateContext({ - consumePartialReplyDirectives: vi.fn((text: string, options?: { final?: boolean }) => - accumulator.consume(text, options), - ), - state: { - blockReplyBreak: "message_end", - }, - }); - const replyText = "Done.\n\n[[reply_to_current]]\n[[audio_as_voice]]\nMEDIA:/tmp/reply.ogg"; - - updateMessage( - ctx, - createTextUpdateEvent({ - type: "text_delta", - text: replyText, - id: "item-final", - signaturePhase: "final_answer", - partialPhase: "final_answer", - }), - ); - updateMessage( - ctx, - createTextUpdateEvent({ - type: "text_end", - text: replyText, - id: "item-final", - signaturePhase: "final_answer", - partialPhase: "final_answer", - }), - ); - - expect(ctx.state.blockBuffer).toBe("Done."); - expect( - consumePendingAssistantReplyDirectivesIntoReply(ctx.state, { - text: "Done.", - }), - ).toEqual({ - text: "Done.", - audioAsVoice: true, - replyToId: undefined, - replyToTag: true, - replyToCurrent: true, - }); - }); -}); - -describe("consumePendingToolMediaIntoReply", () => { - it("attaches queued tool media to the next assistant reply", () => { - const state = { - pendingToolMediaUrls: ["/tmp/a.png", "/tmp/a.png", "/tmp/b.png"], - pendingToolMediaAttachments: [ - { type: "image" as const, path: "/tmp/a.png", width: 640, height: 480 }, - { type: "image" as const, path: "/tmp/a.png", width: 1, height: 1 }, - { type: "image" as const, path: "/tmp/b.png", width: 800, height: 600 }, - ], - pendingToolMediaTrustByUrl: new Map([ - ["/tmp/a.png", true], - ["/tmp/b.png", false], - ]), - pendingToolAudioAsVoice: false, - }; - - expect( - consumePendingToolMediaIntoReply(state, { - text: "done", - }), - ).toEqual({ - text: "done", - mediaUrls: ["/tmp/a.png", "/tmp/b.png"], - attachments: [ - { - type: "image", - path: "/tmp/a.png", - width: 640, - height: 480, - trustedLocalMedia: true, - }, - { type: "image", path: "/tmp/b.png", width: 800, height: 600 }, - ], - audioAsVoice: undefined, - }); - expect(state.pendingToolMediaUrls).toStrictEqual([]); - expect(state.pendingToolMediaAttachments).toStrictEqual([]); - }); - - it("does not append queued image tool media when the reply already names media", () => { - const state = { - pendingToolMediaUrls: ["/tmp/generated.png"], - pendingToolMediaTrustByUrl: new Map([["/tmp/generated.png", true]]), - pendingToolAudioAsVoice: false, - }; - - expect( - consumePendingToolMediaIntoReply(state, { - text: "done", - mediaUrls: ["./selected.png"], - }), - ).toEqual({ - text: "done", - mediaUrls: ["./selected.png"], - }); - expect(state.pendingToolMediaUrls).toStrictEqual([]); - expect(state.pendingToolAudioAsVoice).toBe(false); - expect(state.pendingToolMediaTrustByUrl.size).toBe(0); - }); - - it("retains queued metadata for explicitly selected media", () => { - const state = { - pendingToolMediaUrls: ["/tmp/generated.mp3", "/tmp/generated.mp3", "/tmp/unselected.mp3"], - pendingToolMediaAttachments: [ - { type: "audio" as const, path: "/tmp/generated.mp3", durationMs: 2_000 }, - { type: "audio" as const, path: "/tmp/generated.mp3", durationMs: 9_999 }, - { type: "audio" as const, path: "/tmp/unselected.mp3", durationMs: 3_000 }, - ], - pendingToolMediaTrustByUrl: new Map([ - ["/tmp/generated.mp3", true], - ["/tmp/unselected.mp3", false], - ]), - pendingToolAudioAsVoice: false, - }; - - expect( - consumePendingToolMediaIntoReply(state, { - text: "done", - mediaUrls: [" /tmp/generated.mp3 "], - }), - ).toEqual({ - text: "done", - mediaUrls: [" /tmp/generated.mp3 "], - attachments: [ - { - type: "audio", - path: "/tmp/generated.mp3", - durationMs: 2_000, - trustedLocalMedia: true, - }, - ], - trustedLocalMedia: true, - }); - expect(state.pendingToolMediaAttachments).toStrictEqual([]); - }); - - it("does not trust an explicitly selected untrusted pending URL", () => { - const state = { - pendingToolMediaUrls: ["/tmp/generated.mp3", "/tmp/untrusted.mp3"], - pendingToolMediaAttachments: [ - { type: "audio" as const, path: "/tmp/generated.mp3" }, - { - type: "audio" as const, - path: "/tmp/untrusted.mp3", - trustedLocalMedia: true, - }, - ], - pendingToolMediaTrustByUrl: new Map([ - ["/tmp/generated.mp3", true], - ["/tmp/untrusted.mp3", false], - ]), - pendingToolAudioAsVoice: false, - }; - - expect( - consumePendingToolMediaIntoReply(state, { - text: "done", - mediaUrls: ["/tmp/untrusted.mp3"], - }), - ).toEqual({ - text: "done", - mediaUrls: ["/tmp/untrusted.mp3"], - attachments: [{ type: "audio", path: "/tmp/untrusted.mp3" }], - }); - }); - - it("does not append queued voice media when the reply already names media", () => { - const state = { - pendingToolMediaUrls: ["/tmp/reply.opus"], - pendingToolMediaTrustByUrl: new Map([["/tmp/reply.opus", true]]), - pendingToolAudioAsVoice: true, - }; - - expect( - consumePendingToolMediaIntoReply(state, { - text: "done", - mediaUrls: ["/tmp/assistant-provided.opus"], - }), - ).toEqual({ - text: "done", - mediaUrls: ["/tmp/assistant-provided.opus"], - }); - expect(state.pendingToolMediaUrls).toStrictEqual([]); - expect(state.pendingToolAudioAsVoice).toBe(false); - expect(state.pendingToolMediaTrustByUrl.size).toBe(0); - }); - - it("preserves reasoning replies without consuming queued media", () => { - const state = { - pendingToolMediaUrls: ["/tmp/a.png"], - pendingToolMediaTrustByUrl: new Map([["/tmp/a.png", false]]), - pendingToolAudioAsVoice: true, - }; - - expect( - consumePendingToolMediaIntoReply(state, { - text: "thinking", - isReasoning: true, - }), - ).toEqual({ - text: "thinking", - isReasoning: true, - }); - expect(state.pendingToolMediaUrls).toEqual(["/tmp/a.png"]); - expect(state.pendingToolAudioAsVoice).toBe(true); - }); -}); - -describe("consumePendingToolMediaReply", () => { - it("reads a media-only reply without consuming queued tool media", () => { - const state = { - pendingToolMediaUrls: ["/tmp/reply.opus"], - pendingToolMediaTrustByUrl: new Map([["/tmp/reply.opus", false]]), - pendingToolAudioAsVoice: true, - }; - - expect(readPendingToolMediaReply(state)).toEqual({ - mediaUrls: ["/tmp/reply.opus"], - audioAsVoice: true, - }); - expect(state.pendingToolMediaUrls).toEqual(["/tmp/reply.opus"]); - expect(state.pendingToolAudioAsVoice).toBe(true); - }); - - it("builds a media-only reply for orphaned tool media", () => { - const state = { - pendingToolMediaUrls: ["/tmp/reply.opus"], - pendingToolMediaTrustByUrl: new Map([["/tmp/reply.opus", false]]), - pendingToolAudioAsVoice: true, - }; - - expect(consumePendingToolMediaReply(state)).toEqual({ - mediaUrls: ["/tmp/reply.opus"], - audioAsVoice: true, - }); - expect(state.pendingToolMediaUrls).toStrictEqual([]); - expect(state.pendingToolAudioAsVoice).toBe(false); - }); -}); - -describe("handleMessageUpdate commentary phase", () => { - it("suppresses commentary-phase partial delivery and text_end flush", async () => { - const onAgentEvent = vi.fn(); - const onPartialReply = vi.fn(); - const flushBlockReplyBuffer = vi.fn(); - const ctx = createMessageUpdateContext({ - onAgentEvent, - onPartialReply, - flushBlockReplyBuffer, - }); - - updateMessage( - ctx, - createTextUpdateEvent({ type: "text_delta", text: "Need send.", messagePhase: "commentary" }), - ); - updateMessage( - ctx, - createTextUpdateEvent({ type: "text_end", text: "Need send.", messagePhase: "commentary" }), - ); - - await Promise.resolve(); - - expect(onAgentEvent).not.toHaveBeenCalled(); - expect(onPartialReply).not.toHaveBeenCalled(); - expect(flushBlockReplyBuffer).not.toHaveBeenCalled(); - }); - - it("suppresses commentary partials when phase exists only in textSignature metadata", async () => { - const onAgentEvent = vi.fn(); - const onPartialReply = vi.fn(); - const flushBlockReplyBuffer = vi.fn(); - const commentaryBlock = createOpenAiResponsesTextBlock({ - text: "Need send.", - id: "msg_sig", - phase: "commentary", - }); - const ctx = createMessageUpdateContext({ - onAgentEvent, - onPartialReply, - flushBlockReplyBuffer, - }); - - updateMessage( - ctx, - createTextUpdateEvent({ - type: "text_delta", - text: "Need send.", - content: [commentaryBlock], - }), - ); - updateMessage( - ctx, - createTextUpdateEvent({ - type: "text_end", - text: "Need send.", - content: [commentaryBlock], - }), - ); - - await Promise.resolve(); - - // Archive-always: commentary (textSignature-only phase — the F3 shape) is - // emitted on the bus for archival + window, but kept out of the reply lanes. - expect(onAgentEvent).toHaveBeenCalled(); - expect(onPartialReply).not.toHaveBeenCalled(); - expect(flushBlockReplyBuffer).not.toHaveBeenCalled(); - expect(ctx.state.deltaBuffer).toBe(""); - expect(ctx.state.blockBuffer).toBe(""); - }); - - it("keeps commentary partials out of reply lanes while emitting them on the bus", () => { - const onAgentEvent = vi.fn(); - const ctx = createMessageUpdateContext({ - onAgentEvent, - shouldEmitPartialReplies: false, - }); - - updateMessage( - ctx, - createTextUpdateEvent({ - type: "text_delta", - text: "Working...", - partial: createOpenAiResponsesPartial({ - text: "Working...", - id: "item_commentary", - signaturePhase: "commentary", - partialPhase: "commentary", - }), - }), - ); - - // Emit-always: the bus sees the commentary delta with its phase tag. The raw - // cumulative buffer retains it for end-event dedupe, but reply blocks stay untouched. - expect(onAgentEvent).toHaveBeenCalledTimes(1); - const commentaryEvent = firstMockArg(onAgentEvent, "agent event") as - | { stream?: string; data?: { delta?: string; phase?: string } } - | undefined; - expect(commentaryEvent?.stream).toBe("assistant"); - expect(commentaryEvent?.data?.phase).toBe("commentary"); - expect(commentaryEvent?.data?.delta).toBe("Working..."); - expect(ctx.state.deltaBuffer).toBe("Working..."); - expect(ctx.state.blockBuffer).toBe(""); - - updateMessage( - ctx, - createTextUpdateEvent({ - type: "text_delta", - text: "Done.", - partial: createOpenAiResponsesPartial({ - text: "Done.", - id: "item_final", - signaturePhase: "final_answer", - partialPhase: "final_answer", - }), - }), - ); - - expect(onAgentEvent).toHaveBeenCalledTimes(2); - const event = onAgentEvent.mock.calls[1]?.[0] as - | { stream?: string; data?: { text?: string; delta?: string } } - | undefined; - expect(event?.stream).toBe("assistant"); - expect(event?.data?.text).toBe("Done."); - expect(event?.data?.delta).toBe("Done."); - }); - - it("contains synchronous text_end flush failures", async () => { - const debug = vi.fn(); - const ctx = createMessageUpdateContext({ - debug, - shouldEmitPartialReplies: false, - flushBlockReplyBuffer: vi.fn(() => { - throw new Error("boom"); - }), - }); - - updateMessage(ctx, createTextUpdateEvent({ type: "text_end", text: "" })); - - await vi.waitFor(() => { - expect(debug).toHaveBeenCalledWith("text_end block reply flush failed: Error: boom"); - }); - }); -}); - -describe("handleMessageEnd", () => { - it.each(["answer part A msg [[E1008]timeout] answer part B", "answer ending ["])( - "keeps malformed directive-looking final text identical across delivery paths: %s", - (text) => { - const onAgentEvent = vi.fn(); - const emitBlockReply = vi.fn(); - const flushBlockReplyBuffer = vi.fn(); - const accumulator = createStreamingDirectiveAccumulator(); - const streamed = accumulator.consume(text)?.text ?? ""; - const ctx = createMessageEndContext({ - onAgentEvent, - emitBlockReply, - flushBlockReplyBuffer, - consumeReplyDirectives: vi.fn((chunk: string, options?: { final?: boolean }) => - accumulator.consume(chunk, options), - ), - blockChunker: { - hasBuffered: () => true, - reset: vi.fn(), - }, - state: { - blockBuffer: streamed, - deltaBuffer: streamed, - }, - }); - - void endMessage(ctx, { - message: { role: "assistant", content: [{ type: "text", text }] }, - }); - - expect(firstMockArg(onAgentEvent, "agent event")).toMatchObject({ - stream: "assistant", - data: { text, delta: text }, - }); - const finalBlockText = (firstMockArg(emitBlockReply, "block reply") as { text?: string }) - .text; - expect(`${streamed}${finalBlockText ?? ""}`).toBe(text); - expect(ctx.finalizeAssistantTexts).toHaveBeenCalledWith(expect.objectContaining({ text })); - }, - ); - - it("keeps exact NO_REPLY silent after a user-facing message send followed by sessions_send (#119383)", () => { - const emitBlockReply = vi.fn(); - const finalizeAssistantTexts = vi.fn(); - const ctx = createMessageEndContext({ - emitBlockReply, - finalizeAssistantTexts, - consumeReplyDirectives: vi.fn((text: string) => ({ text })), - state: { - blockBuffer: "", - deltaBuffer: "", - messagingToolSentTexts: ["", ""], - messagingToolSentTextsNormalized: ["", ""], - messagingToolSentTargets: [ - { - tool: "message", - provider: "whatsapp", - to: "user:123", - text: "", - }, - ], - }, - }); - - void endMessage(ctx, { - message: { role: "assistant", content: [{ type: "text", text: "NO_REPLY" }] }, - }); - - // The exact silent token must never be rewritten to the sessions_send body: - // the final assistant text keeps NO_REPLY and no block reply carries the note. - expect(finalizeAssistantTexts).toHaveBeenCalledWith( - expect.objectContaining({ text: "NO_REPLY" }), - ); - for (const call of emitBlockReply.mock.calls) { - expect(JSON.stringify(call)).not.toContain(""); - } - }); - - it("keeps exact NO_REPLY silent when only sessions_send delivered (#119383)", () => { - const emitBlockReply = vi.fn(); - const finalizeAssistantTexts = vi.fn(); - const ctx = createMessageEndContext({ - emitBlockReply, - finalizeAssistantTexts, - consumeReplyDirectives: vi.fn((text: string) => ({ text })), - state: { - blockBuffer: "", - deltaBuffer: "", - messagingToolSentTexts: [""], - messagingToolSentTextsNormalized: [""], - messagingToolSentTargets: [], - }, - }); - - void endMessage(ctx, { - message: { role: "assistant", content: [{ type: "text", text: "NO_REPLY" }] }, - }); - - expect(finalizeAssistantTexts).toHaveBeenCalledWith( - expect.objectContaining({ text: "NO_REPLY" }), - ); - for (const call of emitBlockReply.mock.calls) { - expect(JSON.stringify(call)).not.toContain(""); - } - }); - - it.each([ - { - name: "counts a completed provider assistant message", - message: { role: "assistant", content: [{ type: "text", text: "Done." }] }, - expected: 1, - }, - { - name: "ignores transcript-only mirrored assistant messages", - message: { - role: "assistant", - provider: "openclaw", - model: "delivery-mirror", - content: [{ type: "text", text: "Done." }], - }, - expected: 0, - }, - { - name: "ignores non-assistant messages", - message: { role: "user", content: [{ type: "text", text: "hi" }] }, - expected: 0, - }, - ])("$name for assistantTurnCount", ({ message, expected }) => { - const ctx = createMessageEndContext({ state: { assistantTurnCount: 0 } }); - - void endMessage(ctx, { message }); - - expect(ctx.state.assistantTurnCount).toBe(expected); - }); - - it("keeps duplicate-reply diagnostics free of lone surrogates", () => { - const text = `${"a".repeat(49)}😀tail`; - const ctx = createMessageEndContext({ - consumeReplyDirectives: vi.fn((value: string) => ({ text: value })), - state: { messagingToolSentTextsNormalized: [`${"a".repeat(49)}tail`] }, - }); - - void endMessage(ctx, { - message: { role: "assistant", content: [{ type: "text", text }] }, - }); - - const diagnostic = (ctx.log.debug as ReturnType).mock.calls - .flat() - .find((value) => String(value).startsWith("Skipping message_end block reply")); - expect(diagnostic).toEqual(expect.any(String)); - expect(Buffer.from(String(diagnostic)).toString()).toBe(diagnostic); - }); - - it("persists streamed usage when the final assistant snapshot is zeroed", () => { - const ctx = createMessageEndContext({ - state: { - pendingAssistantUsage: { input: 7, output: 5, reasoningTokens: 2, total: 12 }, - }, - }); - const message = { - role: "assistant", - api: "openai-completions", - content: [{ type: "text", text: "Done." }], - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - }, - }; - - void endMessage(ctx, { - message, - }); - - expect(firstMockArg(ctx.noteLastAssistant as never, "last assistant")).toMatchObject({ - usage: { - input: 7, - output: 5, - cacheRead: 0, - cacheWrite: 0, - reasoningTokens: 2, - totalTokens: 12, - }, - }); - expect(ctx.recordAssistantUsage).toHaveBeenCalledWith( - expect.objectContaining({ - input: 7, - output: 5, - reasoningTokens: 2, - totalTokens: 12, - }), - ); - }); - - it("keeps authoritative final usage instead of pending stream usage", () => { - const ctx = createMessageEndContext({ - state: { - pendingAssistantUsage: { input: 7, output: 5, total: 12 }, - }, - }); - const message = { - role: "assistant", - content: [{ type: "text", text: "Done." }], - usage: { - input: 11, - output: 3, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 14, - }, - }; - - void endMessage(ctx, { - message, - }); - - expect(firstMockArg(ctx.noteLastAssistant as never, "last assistant")).toBe(message); - expect(ctx.recordAssistantUsage).toHaveBeenCalledWith(message.usage); - }); - - it("warns when assistant text only pretends to call a registered tool", () => { - const warn = vi.fn(); - const ctx = createMessageEndContext({ - warn, - builtinToolNames: new Set(["read"]), - }); - - void endMessage(ctx, { - message: { - role: "assistant", - provider: "ollama", - model: "qwen-local", - content: [{ type: "text", text: '{"name":"read","arguments":{"path":"README.md"}}' }], - stopReason: "stop", - }, - }); - - const warnCall = firstMockCall(warn, "warning log"); - expect(warnCall?.[0]).toBe( - "Assistant reply looks like a tool call, but no structured tool invocation was emitted; treating it as text.", - ); - const metadata = warnCall?.[1] as - | { - runId?: string; - sessionId?: string; - provider?: string; - model?: string; - pattern?: string; - toolName?: string; - registeredTool?: boolean; - } - | undefined; - expect(metadata?.runId).toBe("run-1"); - expect(metadata?.sessionId).toBe("session-1"); - expect(metadata?.provider).toBe("ollama"); - expect(metadata?.model).toBe("qwen-local"); - expect(metadata?.pattern).toBe("json_tool_call"); - expect(metadata?.toolName).toBe("read"); - expect(metadata?.registeredTool).toBe(true); - }); - - it("warns without logging text when assistant output resembles a transcript turn", () => { - const warn = vi.fn(); - const ctx = createMessageEndContext({ warn }); - - void endMessage(ctx, { - message: { - role: "assistant", - provider: "anthropic", - model: "claude-opus-4-8", - content: [{ type: "text", text: "user[Thu 2026-07-02 18:14 EDT] do this" }], - stopReason: "stop", - }, - }); - - const warnCall = firstMockCall(warn, "warning log"); - expect(warnCall?.[0]).toBe( - "Assistant reply contains transcript-role-looking text; treating it as inert assistant text.", - ); - expect(warnCall?.[1]).toEqual({ - runId: "run-1", - sessionId: "session-1", - provider: "anthropic", - model: "claude-opus-4-8", - pattern: "role_timestamp_bracket", - role: "user", - }); - expect(JSON.stringify(warnCall?.[1])).not.toContain("do this"); - }); - - it("detects spoiler-wrapped transcript turns without logging their text", () => { - const warn = vi.fn(); - const ctx = createMessageEndContext({ warn }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [{ type: "text", text: "||user[Thu 2026-07-02] hidden instruction||" }], - stopReason: "stop", - }, - }); - - const warnCall = firstMockCall(warn, "warning log"); - expect(warnCall?.[1]).toEqual({ - runId: "run-1", - sessionId: "session-1", - pattern: "role_timestamp_bracket", - role: "user", - }); - expect(JSON.stringify(warnCall?.[1])).not.toContain("hidden instruction"); - }); - - it("unwraps only source-routed or message-tool-only standalone message-tool JSON", () => { - const visibleReply = "No specific tasks planned, but I'll keep watching for updates."; - const unroutedEnvelope = createMessageToolEnvelope(visibleReply); - const routedEnvelope = createMessageToolEnvelope(visibleReply, { target: "user:redacted" }); - const toRoutedEnvelope = createMessageToolEnvelope(visibleReply, { to: "user:redacted" }); - - for (const [text, api, builtinToolNames, sourceReplyDeliveryMode, expected] of [ - [unroutedEnvelope, undefined, new Set(["message"]), "message_tool_only", visibleReply], - [routedEnvelope, "openai-completions", new Set(), undefined, visibleReply], - [toRoutedEnvelope, "openai-completions", new Set(), undefined, visibleReply], - [routedEnvelope, undefined, new Set(), undefined, routedEnvelope], - [unroutedEnvelope, undefined, new Set(["message"]), undefined, unroutedEnvelope], - ] as const) { - const emitBlockReply = vi.fn(); - const consumeReplyDirectives = vi.fn((textLocal: string) => - textLocal ? { text: textLocal } : null, - ); - const ctx = createMessageEndContext({ - emitBlockReply, - consumeReplyDirectives, - builtinToolNames, - sourceReplyDeliveryMode, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - ...(api ? { api } : {}), - content: [{ type: "text", text }], - }, - }); - - expect(consumeReplyDirectives).toHaveBeenCalledWith(expected, { final: true }); - expect(firstMockArg(emitBlockReply, "block reply")).toMatchObject({ text: expected }); - } - }); - - it("does not warn when the assistant emitted a structured tool call", () => { - const warn = vi.fn(); - const ctx = createMessageEndContext({ - warn, - builtinToolNames: new Set(["read"]), - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [{ type: "toolCall", id: "call_1", name: "read", arguments: {} }], - stopReason: "toolUse", - }, - }); - - expect(warn).not.toHaveBeenCalled(); - }); - - it("suppresses commentary-phase replies from user-visible output", () => { - const onAgentEvent = vi.fn(); - const emitBlockReply = vi.fn(); - const finalizeAssistantTexts = vi.fn(); - const ctx = createMessageEndContext({ - onAgentEvent, - finalizeAssistantTexts, - emitBlockReply, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - phase: "commentary", - content: [{ type: "text", text: "Need send." }], - usage: { input: 1, output: 1, total: 2 }, - }, - }); - - // Archive-always: commentary reaches the bus/archive but not the visible reply. - expect(onAgentEvent).toHaveBeenCalled(); - expect(emitBlockReply).not.toHaveBeenCalled(); - expect(finalizeAssistantTexts).not.toHaveBeenCalled(); - }); - - it("suppresses commentary message_end when phase exists only in textSignature metadata", () => { - const onAgentEvent = vi.fn(); - const emitBlockReply = vi.fn(); - const finalizeAssistantTexts = vi.fn(); - const ctx = createMessageEndContext({ - onAgentEvent, - finalizeAssistantTexts, - emitBlockReply, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [ - createOpenAiResponsesTextBlock({ - text: "Need send.", - id: "msg_sig", - phase: "commentary", - }), - ], - usage: { input: 1, output: 1, total: 2 }, - }, - }); - - // Archive-always: commentary (textSignature-only phase) reaches the - // bus/archive but not the visible reply. - expect(onAgentEvent).toHaveBeenCalled(); - expect(emitBlockReply).not.toHaveBeenCalled(); - expect(finalizeAssistantTexts).not.toHaveBeenCalled(); - }); - - it("does not duplicate block reply for text_end channels when text was already delivered", () => { - const onBlockReply = vi.fn(); - const emitBlockReply = vi.fn(); - // In real usage, the directive accumulator returns null for empty/consumed - // input. The non-empty call shouldn't happen for text_end channels (that's - // the safety send we're guarding against). - const consumeReplyDirectives = vi.fn((text: string) => (text ? { text } : null)); - const ctx = createMessageEndContext({ - onBlockReply, - emitBlockReply, - consumeReplyDirectives, - state: { - emittedAssistantUpdate: true, - lastStreamedAssistantCleaned: "Hello world", - blockReplyBreak: "text_end", - // Simulate text_end already delivered this text through emitBlockChunk - lastBlockReplyText: "Hello world", - deltaBuffer: "", - blockBuffer: "", - }, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [{ type: "text", text: "Hello world" }], - usage: { input: 10, output: 5, total: 15 }, - }, - }); - - // The block reply should NOT fire again since text_end already delivered it. - // consumeReplyDirectives is called once with "" (the final flush for - // text_end channels) but returns null, so emitBlockReply is never called. - expect(emitBlockReply).not.toHaveBeenCalled(); - }); - - it("tags message-end safety replies with the current assistant message", () => { - const emitBlockReply = vi.fn(); - const ctx = createMessageEndContext({ - onBlockReply: vi.fn(), - emitBlockReply, - consumeReplyDirectives: vi.fn((text: string) => (text ? { text } : null)), - state: { - assistantMessageIndex: 7, - blockReplyBreak: "text_end", - lastBlockReplyText: null, - }, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [{ type: "text", text: "Final answer" }], - usage: { input: 10, output: 5, total: 15 }, - }, - }); - - expect(emitBlockReply).toHaveBeenCalledWith( - { text: "Final answer" }, - { assistantMessageIndex: 7 }, - ); - }); - - it("does not duplicate block reply for text_end channels even when stripping differs", () => { - const onBlockReply = vi.fn(); - const emitBlockReply = vi.fn(); - // Same pattern: directive accumulator returns null for empty final flush - const consumeReplyDirectives = vi.fn((text: string) => (text ? { text } : null)); - const ctx = createMessageEndContext({ - onBlockReply, - emitBlockReply, - consumeReplyDirectives, - state: { - emittedAssistantUpdate: true, - lastStreamedAssistantCleaned: "Hello world", - blockReplyBreak: "text_end", - // text_end delivered via emitBlockChunk which uses different stripping - lastBlockReplyText: "Hello world.", - deltaBuffer: "", - blockBuffer: "", - }, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - // The raw text differs slightly from lastBlockReplyText due to stripping - content: [{ type: "text", text: "Hello world" }], - usage: { input: 10, output: 5, total: 15 }, - }, - }); - - // Even though text !== lastBlockReplyText (different stripping), the safety - // send should NOT fire for text_end channels. The only consumeReplyDirectives - // call is the final empty flush which returns null. - expect(emitBlockReply).not.toHaveBeenCalled(); - }); - - it("emits final media and malformed pending text after flushing buffered message_end text", () => { - const emitBlockReply = vi.fn(); - const flushBlockReplyBuffer = vi.fn(); - const accumulator = createStreamingDirectiveAccumulator(); - const text = "Caption [[oops\nMEDIA:/tmp/final.png"; - const streamed = accumulator.consume(text)?.text ?? ""; - const consumeReplyDirectives = vi.fn((chunk: string, options?: { final?: boolean }) => - accumulator.consume(chunk, options), - ); - const ctx = createMessageEndContext({ - emitBlockReply, - flushBlockReplyBuffer, - consumeReplyDirectives, - blockChunker: { - hasBuffered: () => true, - reset: vi.fn(), - }, - state: { - emittedAssistantUpdate: true, - lastStreamedAssistantCleaned: "Caption [[oops", - blockReplyBreak: "message_end", - deltaBuffer: streamed, - blockBuffer: streamed, - }, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [{ type: "text", text }], - usage: { input: 10, output: 5, total: 15 }, - }, - }); - - expect(flushBlockReplyBuffer).toHaveBeenCalledWith({ - assistantMessageIndex: undefined, - final: true, - }); - expect(consumeReplyDirectives).toHaveBeenCalledWith("", { final: true }); - const finalReply = firstMockArg(emitBlockReply, "block reply") as { - text?: string; - mediaUrls?: string[]; - }; - expect(finalReply).toMatchObject({ - text: " [[oops", - mediaUrls: ["/tmp/final.png"], - }); - expect(`${streamed}${finalReply.text ?? ""}`).toBe("Caption [[oops"); - }); - - it("preserves literal reasoning-looking tags in unphased final visible text", () => { - const onAgentEvent = vi.fn(); - const stripBlockTags = vi.fn(() => "Before"); - const ctx = createMessageEndContext({ - onAgentEvent, - stripBlockTags, - consumeReplyDirectives: vi.fn((text: string) => ({ text })), - state: { - blockBuffer: "", - deltaBuffer: "", - }, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [ - { - type: "text", - text: "Before literal tag text after", - textSignature: JSON.stringify({ v: 1, id: "item_unphased" }), - }, - ], - usage: { input: 10, output: 5, total: 15 }, - }, - }); - - expect(stripBlockTags).not.toHaveBeenCalled(); - expect(firstMockArg(ctx.emitAssistantStreamData as never, "assistant stream")).toMatchObject({ - text: "Before literal tag text after", - delta: "Before literal tag text after", - }); - expect(ctx.finalizeAssistantTexts).toHaveBeenCalledWith( - expect.objectContaining({ text: "Before literal tag text after" }), - ); - }); - - it("keeps final-tag enforcement in message_end fallback", () => { - const onAgentEvent = vi.fn(); - const stripBlockTags = vi.fn(() => ""); - const ctx = createMessageEndContext({ - enforceFinalTag: true, - onAgentEvent, - stripBlockTags, - consumeReplyDirectives: vi.fn((text: string) => ({ text })), - state: { - blockBuffer: "", - deltaBuffer: "", - }, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: "Hello world", - usage: { input: 10, output: 5, total: 15 }, - }, - }); - - expect(stripBlockTags).toHaveBeenCalledWith( - "Hello world", - { thinking: false, final: false }, - { final: true }, - ); - expect(ctx.emitAssistantStreamData).not.toHaveBeenCalled(); - expect(ctx.finalizeAssistantTexts).toHaveBeenCalledWith(expect.objectContaining({ text: "" })); - }); - - it("emits a replacement final assistant event when final_answer appears only at message_end", () => { - const onAgentEvent = vi.fn(); - const ctx = createMessageEndContext({ - onAgentEvent, - state: { - emittedAssistantUpdate: true, - lastStreamedAssistantCleaned: "Working...", - blockReplyBreak: "text_end", - deltaBuffer: "", - blockBuffer: "", - }, - }); - - void endMessage(ctx, { - message: { - role: "assistant", - content: [ - createOpenAiResponsesTextBlock({ - text: "Working...", - id: "item_commentary", - phase: "commentary", - }), - createOpenAiResponsesTextBlock({ - text: "Done.", - id: "item_final", - phase: "final_answer", - }), - ], - stopReason: "stop", - api: "openai-responses", - provider: "openai", - model: "gpt-5.2", - usage: {}, - timestamp: 0, - }, - }); - - expect(onAgentEvent).toHaveBeenCalledTimes(1); - const event = firstMockArg(onAgentEvent, "agent event") as - | { stream?: string; data?: { text?: string; delta?: string; replace?: boolean } } - | undefined; - expect(event?.stream).toBe("assistant"); - expect(event?.data?.text).toBe("Done."); - expect(event?.data?.delta).toBe(""); - expect(event?.data?.replace).toBe(true); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.ts b/src/agents/embedded-agent-subscribe.handlers.messages.ts deleted file mode 100644 index 9762ff221038..000000000000 --- a/src/agents/embedded-agent-subscribe.handlers.messages.ts +++ /dev/null @@ -1,1647 +0,0 @@ -import { isPromiseLike } from "@openclaw/normalization-core/promise-like"; -/** - * Handles embedded-agent assistant message events, block replies, reasoning - * streams, reply directives, and pending tool media attachment handoff. - */ -import { asOptionalRecord as asRecord } from "@openclaw/normalization-core/record-coerce"; -import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; -import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; -import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; -import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload"; -import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; -import { - parseReplyDirectives, - type ReplyDirectiveParseResult, -} from "../auto-reply/reply/reply-directives.js"; -import { splitTrailingDirective } from "../auto-reply/reply/streaming-directives.js"; -import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; -import { emitAgentEvent } from "../infra/agent-events.js"; -import type { AssistantMessage } from "../llm/types.js"; -import { splitMediaFromOutput } from "../media/parse.js"; -import { coerceChatContentText } from "../shared/chat-content.js"; -import { - parseAssistantTextSignature, - resolveAssistantMessagePhase, - type AssistantPhase, -} from "../shared/chat-message-content.js"; -import { - isMessagingToolDuplicateNormalized, - normalizeTextForComparison, -} from "./embedded-agent-helpers.js"; -import { updateLiveEditDiffProgress } from "./embedded-agent-live-edit-diff.js"; -import type { BlockReplyPayload } from "./embedded-agent-payloads.js"; -import { runBestEffortCallback } from "./embedded-agent-subscribe.callback.js"; -import type { - EmbeddedAgentSubscribeContext, - EmbeddedAgentSubscribeState, -} from "./embedded-agent-subscribe.handlers.types.js"; -import { appendRawStream } from "./embedded-agent-subscribe.raw-stream.js"; -import { warnIfAssistantEmittedSuspiciousText } from "./embedded-agent-subscribe.tool-text-diagnostics.js"; -import { - extractEmbeddedAssistantText, - extractAssistantThinking, - extractAssistantCommentaryText, - extractAssistantVisibleText, - createThinkingTagStreamState, - extractThinkingFromTaggedStream, - extractThinkingFromTaggedText, - promoteThinkingTagsToBlocks, - sanitizeAssistantVisibleStreamText, -} from "./embedded-agent-utils.js"; -import type { AgentEvent, AgentMessage } from "./runtime/index.js"; -import { - hasNonzeroUsage, - makeZeroUsageSnapshot, - normalizeUsage, - type NormalizedUsage, - type UsageLike, -} from "./usage.js"; - -function shouldSuppressAssistantVisibleOutput(message: AgentMessage | undefined): boolean { - return resolveAssistantMessagePhase(message) === "commentary"; -} - -function isTranscriptOnlyOpenClawAssistantMessage(message: AgentMessage | undefined): boolean { - if (!message || message.role !== "assistant") { - return false; - } - const provider = normalizeOptionalString(message.provider) ?? ""; - const model = normalizeOptionalString(message.model) ?? ""; - return provider === "openclaw" && (model === "delivery-mirror" || model === "gateway-injected"); -} - -const RESPONSES_API_IDS = new Set([ - "openai-responses", - "openai-chatgpt-responses", - "azure-openai-responses", - "openclaw-openai-responses-transport", - "openclaw-openai-chatgpt-responses-transport", - "openclaw-azure-openai-responses-transport", -]); - -function isResponsesApiAssistantMessage(message: AgentMessage | undefined): boolean { - if (!message || message.role !== "assistant") { - return false; - } - const api = normalizeOptionalString((message as { api?: unknown }).api) ?? ""; - return RESPONSES_API_IDS.has(api); -} - -function isAnthropicAssistantMessage(message: AgentMessage | undefined): boolean { - if (!message || message.role !== "assistant") { - return false; - } - const api = normalizeOptionalString((message as { api?: unknown }).api) ?? ""; - return api === "anthropic-messages"; -} - -function isOpenAiCompletionsAssistantMessage(message: AgentMessage | undefined): boolean { - if (!message || message.role !== "assistant") { - return false; - } - const api = normalizeOptionalString((message as { api?: unknown }).api) ?? ""; - return api === "openai-completions" || api === "openclaw-openai-completions-transport"; -} - -export function preservePendingAssistantUsage( - message: AssistantMessage, - pendingUsage: NormalizedUsage | undefined, -): AssistantMessage { - if (isTranscriptOnlyOpenClawAssistantMessage(message) || !hasNonzeroUsage(pendingUsage)) { - return message; - } - const messageUsage = normalizeUsage((message as { usage?: UsageLike }).usage); - if (hasNonzeroUsage(messageUsage)) { - return message; - } - - // Pending usage resets at each assistant-message boundary, so it belongs to - // this final snapshot. Only replace missing/zero usage; provider totals win. - const input = pendingUsage.input ?? 0; - const output = pendingUsage.output ?? 0; - const cacheRead = pendingUsage.cacheRead ?? 0; - const cacheWrite = pendingUsage.cacheWrite ?? 0; - message.usage = { - ...makeZeroUsageSnapshot(), - input, - output, - cacheRead, - cacheWrite, - ...(pendingUsage.contextUsage ? { contextUsage: { ...pendingUsage.contextUsage } } : {}), - totalTokens: pendingUsage.total ?? input + output + cacheRead + cacheWrite, - ...(pendingUsage.reasoningTokens !== undefined - ? { reasoningTokens: pendingUsage.reasoningTokens } - : {}), - }; - return message; -} - -export function capturePendingAssistantUsage( - ctx: EmbeddedAgentSubscribeContext, - evt: AgentEvent & { message: AgentMessage; assistantMessageEvent?: unknown }, -): void { - const msg = evt.message; - if (msg?.role !== "assistant" || isTranscriptOnlyOpenClawAssistantMessage(msg)) { - return; - } - const assistantRecord = - evt.assistantMessageEvent && typeof evt.assistantMessageEvent === "object" - ? (evt.assistantMessageEvent as Record) - : undefined; - const evtType = typeof assistantRecord?.type === "string" ? assistantRecord.type : ""; - if (evtType === "text_end" || evtType === "done" || evtType === "error") { - ctx.recordAssistantUsage(assistantRecord); - } -} - -export function resetPendingAssistantUsage( - ctx: EmbeddedAgentSubscribeContext, - message: AgentMessage, -): void { - if (message?.role !== "assistant" || isTranscriptOnlyOpenClawAssistantMessage(message)) { - return; - } - ctx.state.pendingAssistantUsage = undefined; - ctx.state.assistantUsageCommitted = false; -} - -function extractStandaloneMessageToolText( - text: string, - params: { allowCurrentSourceReply?: boolean; allowRoutedReply?: boolean } = {}, -): string | undefined { - try { - const record = asRecord(JSON.parse(text.trim()) as unknown); - const args = asRecord(record?.arguments); - const hasRoute = Boolean( - normalizeOptionalString(args?.target) || - normalizeOptionalString(args?.to) || - normalizeOptionalString(args?.channel) || - normalizeOptionalString(args?.accountId) || - Array.isArray(args?.targets), - ); - if ( - normalizeOptionalString(record?.name) !== "message" || - normalizeOptionalString(args?.action) !== "send" || - (hasRoute ? !params.allowRoutedReply : !params.allowCurrentSourceReply) - ) { - return undefined; - } - return normalizeOptionalString(args?.message); - } catch { - return undefined; - } -} - -function resolveAssistantStreamItemId(params: { - contentIndex?: unknown; - message: AgentMessage | undefined; -}): string | undefined { - const content = (params.message as { content?: unknown } | undefined)?.content; - if (!Array.isArray(content)) { - return undefined; - } - const contentIndex = - typeof params.contentIndex === "number" && - Number.isInteger(params.contentIndex) && - params.contentIndex >= 0 - ? params.contentIndex - : undefined; - const indexedBlock = contentIndex !== undefined ? content[contentIndex] : undefined; - const indexedRecord = - indexedBlock && typeof indexedBlock === "object" - ? (indexedBlock as { type?: unknown }) - : undefined; - const hasIndexedTextBlock = indexedRecord?.type === "text"; - const candidateStart = - hasIndexedTextBlock && contentIndex !== undefined ? contentIndex : content.length - 1; - const candidateEnd = hasIndexedTextBlock ? candidateStart : 0; - for (let index = candidateStart; index >= candidateEnd; index -= 1) { - const block = content[index]; - if (!block || typeof block !== "object") { - continue; - } - const record = block as { type?: unknown; textSignature?: unknown }; - if (record.type !== "text") { - continue; - } - const signature = parseAssistantTextSignature(record); - if (signature?.id) { - return signature.id; - } - } - return undefined; -} - -function resolveAssistantStreamContentIndex(value: unknown): number | undefined { - return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined; -} - -function scopeAssistantMessageToStreamBlock( - message: AssistantMessage, - contentIndex: number | undefined, - itemId: string | undefined, -): AssistantMessage { - if (!Array.isArray(message.content)) { - return message; - } - const indexedBlock = contentIndex === undefined ? undefined : message.content[contentIndex]; - let block = - indexedBlock && typeof indexedBlock === "object" && indexedBlock.type === "text" - ? indexedBlock - : undefined; - if (!block && itemId) { - for (let index = message.content.length - 1; index >= 0; index -= 1) { - const candidate = message.content[index]; - if ( - candidate && - typeof candidate === "object" && - candidate.type === "text" && - parseAssistantTextSignature(candidate)?.id === itemId - ) { - block = candidate; - break; - } - } - } - if (!block) { - return message; - } - // Provider partials are cumulative across content blocks. Once a content - // index becomes a logical reply boundary, downstream snapshots must be - // cumulative only within that block or earlier text is replayed. - return { ...message, content: [block] }; -} - -function emitReasoningEnd(ctx: EmbeddedAgentSubscribeContext) { - if (!ctx.state.reasoningStreamOpen) { - return; - } - ctx.state.reasoningStreamOpen = false; - runBestEffortCallback({ - label: "reasoning end", - log: ctx.log, - callback: () => ctx.params.onReasoningEnd?.(), - }); -} - -function emitAssistantMessageStart(ctx: EmbeddedAgentSubscribeContext) { - runBestEffortCallback({ - label: "assistant message start", - log: ctx.log, - callback: () => ctx.params.onAssistantMessageStart?.(), - }); -} - -function openReasoningStream(ctx: EmbeddedAgentSubscribeContext) { - ctx.state.reasoningStreamOpen = true; -} - -function shouldSuppressDeterministicApprovalOutput( - state: Pick< - EmbeddedAgentSubscribeState, - "deterministicApprovalPromptPending" | "deterministicApprovalPromptSent" - >, -): boolean { - return state.deterministicApprovalPromptPending || state.deterministicApprovalPromptSent; -} - -function hasMessageToolOnlySourceDelivery(ctx: EmbeddedAgentSubscribeContext): boolean { - return ( - ctx.params.sourceReplyDeliveryMode === "message_tool_only" && - (ctx.state.messageToolOnlySourceReplyDelivered || - ctx.params.hasDeliveredMessageToolOnlySourceReply?.() === true || - (ctx.state.messagingToolSourceReplyPayloads?.length ?? 0) > 0) - ); -} - -function resolveCurrentSourceMessagingToolPartial( - state: Pick< - EmbeddedAgentSubscribeState, - "currentSourceMessagingToolHeldPartial" | "currentSourceMessagingToolSentTextsNormalized" - >, - params: { - evtType: "text_delta" | "text_start" | "text_end"; - text: string; - visibleDelta: string; - }, -): { hold: boolean; text: string } { - const held = state.currentSourceMessagingToolHeldPartial; - const text = - held && params.evtType === "text_delta" && !params.text.startsWith(held) - ? `${held}${params.visibleDelta || params.text}` - : params.text; - const normalized = normalizeTextForComparison(text); - if (!normalized) { - state.currentSourceMessagingToolHeldPartial = undefined; - return { hold: false, text }; - } - // A confirmed current-source tool send already made this prefix visible. - // Hold it until the assistant either repeats the sent text or diverges with new content. - const hold = state.currentSourceMessagingToolSentTextsNormalized.some( - (sentText) => sentText === normalized || sentText.startsWith(normalized), - ); - state.currentSourceMessagingToolHeldPartial = hold ? text : undefined; - return { hold, text }; -} - -function appendBlockReplyChunk(ctx: EmbeddedAgentSubscribeContext, chunk: string) { - if (ctx.blockChunker) { - ctx.blockChunker.append(chunk); - return; - } - ctx.state.blockBuffer += chunk; -} - -function replaceBlockReplyBuffer(ctx: EmbeddedAgentSubscribeContext, text: string) { - if (ctx.blockChunker) { - ctx.blockChunker.reset(); - ctx.blockChunker.append(text); - return; - } - ctx.state.blockBuffer = text; -} - -function resolveAssistantTextChunk(params: { - evtType: "text_delta" | "text_start" | "text_end"; - delta: string; - content: string; - accumulatedText: string; -}): string { - const { evtType, delta, content, accumulatedText } = params; - if (evtType === "text_delta") { - return delta; - } - if (delta) { - return delta; - } - if (!content) { - return ""; - } - // KNOWN: Some providers resend full content on `text_end`. - // We only append a suffix (or nothing) to keep output monotonic. - if (content.startsWith(accumulatedText)) { - return content.slice(accumulatedText.length); - } - if (accumulatedText.startsWith(content)) { - return ""; - } - if (!accumulatedText.includes(content)) { - return content; - } - return ""; -} - -const REASONING_TAG_RE = /<\s*\/?\s*(?:(?:antml:|mm:)?(?:think(?:ing)?|thought)|antthinking)\b/i; - -function resolveStreamVisibleText(params: { - previousRawText: string; - visibleDelta: string; - finalText?: string; -}): { rawText: string; visibleText: string } { - if (params.finalText !== undefined) { - const rawText = params.finalText; - return { rawText, visibleText: rawText.trim() }; - } - const rawText = `${params.previousRawText}${params.visibleDelta}`; - return { rawText, visibleText: rawText.trim() }; -} - -function resolveTextAppendDelta(previousText: string, nextText: string): string { - if (!nextText) { - return ""; - } - if (!previousText) { - return nextText; - } - if (nextText.startsWith(previousText)) { - return nextText.slice(previousText.length); - } - if (previousText.startsWith(nextText)) { - return ""; - } - return nextText; -} - -function copyPartialBlockState( - target: EmbeddedAgentSubscribeState["partialBlockState"], - source: EmbeddedAgentSubscribeState["partialBlockState"], -) { - const copyFenceState = (fence?: typeof source.fence) => - fence - ? { - atLineStart: fence.atLineStart, - ...(fence.open ? { open: { ...fence.open } } : {}), - } - : undefined; - target.thinking = source.thinking; - target.final = source.final; - target.inlineCode = { ...source.inlineCode }; - target.fence = copyFenceState(source.fence); - target.reasoningInlineCode = source.reasoningInlineCode - ? { ...source.reasoningInlineCode } - : undefined; - target.reasoningFence = copyFenceState(source.reasoningFence); - target.reasoningPendingFenceFragment = source.reasoningPendingFenceFragment; - target.finalInlineCode = source.finalInlineCode ? { ...source.finalInlineCode } : undefined; - target.finalFence = copyFenceState(source.finalFence); - target.pendingFenceFragment = source.pendingFenceFragment; - target.pendingTagFragment = source.pendingTagFragment; -} - -function clearPendingToolMedia( - state: Pick< - EmbeddedAgentSubscribeState, - | "pendingToolMediaUrls" - | "pendingToolMediaAttachments" - | "pendingToolMediaTrustByUrl" - | "pendingToolAudioAsVoice" - >, -) { - state.pendingToolMediaUrls = []; - state.pendingToolMediaAttachments = []; - state.pendingToolMediaTrustByUrl.clear(); - state.pendingToolAudioAsVoice = false; -} - -function hasReplyMedia(payload: BlockReplyPayload): boolean { - return (payload.mediaUrls ?? []).some((url) => url.trim().length > 0); -} - -function readAlignedPendingToolMedia( - state: Pick< - EmbeddedAgentSubscribeState, - "pendingToolMediaUrls" | "pendingToolMediaAttachments" | "pendingToolMediaTrustByUrl" - >, -) { - const seen = new Set(); - const mediaUrls: string[] = []; - const attachments: NonNullable = []; - for (const [index, url] of state.pendingToolMediaUrls.entries()) { - if (seen.has(url)) { - continue; - } - seen.add(url); - mediaUrls.push(url); - const { trustedLocalMedia: _untrustedInput, ...attachment } = - state.pendingToolMediaAttachments?.[index] ?? {}; - attachments.push({ - ...attachment, - ...(state.pendingToolMediaTrustByUrl.get(url) === true ? { trustedLocalMedia: true } : {}), - }); - } - return { - mediaUrls, - attachments: attachments.some((entry) => Object.keys(entry).length > 0) - ? attachments - : undefined, - }; -} - -/** Moves queued tool media into a non-reasoning assistant reply payload. */ -export function consumePendingToolMediaIntoReply( - state: Pick< - EmbeddedAgentSubscribeState, - | "pendingToolMediaUrls" - | "pendingToolMediaAttachments" - | "pendingToolMediaTrustByUrl" - | "pendingToolAudioAsVoice" - >, - payload: BlockReplyPayload, -): BlockReplyPayload { - if (payload.isReasoning) { - return payload; - } - if (state.pendingToolMediaUrls.length === 0 && !state.pendingToolAudioAsVoice) { - return payload; - } - if (hasReplyMedia(payload)) { - // Pending tool media is a fallback delivery queue; explicit final media is - // the assistant's user-visible selection, while tool output remains in the transcript. - const alignedPendingMedia = readAlignedPendingToolMedia(state); - const metadataByUrl = new Map( - alignedPendingMedia.mediaUrls.map((url, index) => [ - url, - alignedPendingMedia.attachments?.[index] ?? {}, - ]), - ); - const selectedAttachments = (payload.mediaUrls ?? []).map( - (url) => metadataByUrl.get(url.trim()) ?? {}, - ); - const allSelectedMediaIsPending = - (payload.mediaUrls?.length ?? 0) > 0 && - (payload.mediaUrls ?? []).every((url) => metadataByUrl.has(url.trim())); - const payloadWithMetadata = - payload.attachments?.length || - selectedAttachments.every((entry) => Object.keys(entry).length === 0) - ? payload - : { ...payload, attachments: selectedAttachments }; - const selectedPayload = - allSelectedMediaIsPending && - (payload.mediaUrls ?? []).every( - (url) => state.pendingToolMediaTrustByUrl.get(url.trim()) === true, - ) - ? { ...payloadWithMetadata, trustedLocalMedia: true } - : payloadWithMetadata; - clearPendingToolMedia(state); - return selectedPayload; - } - const pendingMedia = readAlignedPendingToolMedia(state); - const allPendingMediaTrusted = - pendingMedia.mediaUrls.length > 0 && - pendingMedia.mediaUrls.every((url) => state.pendingToolMediaTrustByUrl.get(url) === true); - const mergedPayload: BlockReplyPayload = { - ...payload, - mediaUrls: pendingMedia.mediaUrls.length ? pendingMedia.mediaUrls : undefined, - attachments: pendingMedia.attachments, - audioAsVoice: payload.audioAsVoice || state.pendingToolAudioAsVoice || undefined, - ...(payload.trustedLocalMedia || allPendingMediaTrusted ? { trustedLocalMedia: true } : {}), - }; - clearPendingToolMedia(state); - return mergedPayload; -} - -/** Consumes queued tool media as a standalone reply payload. */ -export function consumePendingToolMediaReply( - state: Pick< - EmbeddedAgentSubscribeState, - | "pendingToolMediaUrls" - | "pendingToolMediaAttachments" - | "pendingToolMediaTrustByUrl" - | "pendingToolAudioAsVoice" - >, -): BlockReplyPayload | null { - const payload = readPendingToolMediaReply(state); - if (!payload) { - return null; - } - clearPendingToolMedia(state); - return payload; -} - -/** Reads queued tool media without clearing it. */ -export function readPendingToolMediaReply( - state: Pick< - EmbeddedAgentSubscribeState, - | "pendingToolMediaUrls" - | "pendingToolMediaAttachments" - | "pendingToolMediaTrustByUrl" - | "pendingToolAudioAsVoice" - >, -): BlockReplyPayload | null { - if (state.pendingToolMediaUrls.length === 0 && !state.pendingToolAudioAsVoice) { - return null; - } - const pendingMedia = readAlignedPendingToolMedia(state); - const allPendingMediaTrusted = - pendingMedia.mediaUrls.length > 0 && - pendingMedia.mediaUrls.every((url) => state.pendingToolMediaTrustByUrl.get(url) === true); - return { - mediaUrls: pendingMedia.mediaUrls.length ? pendingMedia.mediaUrls : undefined, - attachments: pendingMedia.attachments, - audioAsVoice: state.pendingToolAudioAsVoice || undefined, - ...(allPendingMediaTrusted ? { trustedLocalMedia: true } : {}), - }; -} - -function hasReplyDirectiveMetadata(parsed: ReplyDirectiveParseResult | null | undefined): boolean { - return Boolean( - parsed && - ((parsed.mediaUrls?.length ?? 0) > 0 || - parsed.audioAsVoice || - parsed.replyToId || - parsed.replyToTag || - parsed.replyToCurrent), - ); -} - -function hasReplyDirectiveMetadataResult( - parsed: ReplyDirectiveParseResult | null | undefined, -): parsed is ReplyDirectiveParseResult { - return hasReplyDirectiveMetadata(parsed); -} - -function mergeReplyDirectiveResults( - first: ReplyDirectiveParseResult | null | undefined, - second: ReplyDirectiveParseResult | null | undefined, -): ReplyDirectiveParseResult | null { - if (!first) { - return second ?? null; - } - if (!second) { - return first; - } - const mediaUrls = uniqueStrings([...(first.mediaUrls ?? []), ...(second.mediaUrls ?? [])]); - return { - text: `${first.text ?? ""}${second.text ?? ""}`, - mediaUrls: mediaUrls.length ? mediaUrls : undefined, - replyToId: second.replyToId ?? first.replyToId, - replyToCurrent: first.replyToCurrent || second.replyToCurrent, - replyToTag: first.replyToTag || second.replyToTag, - audioAsVoice: first.audioAsVoice || second.audioAsVoice || undefined, - isSilent: first.isSilent || second.isSilent, - }; -} - -function containsCompleteMediaDirectiveLine(text: string): boolean { - return /(?:^|\n)\s*MEDIA:\s*\S[^\n]*(?:\n|$)/i.test(text); -} - -function resolveIncrementalStreamingReplyText(params: { - evtType: "text_delta" | "text_start" | "text_end"; - next: string; - previousRawText: string; - previousCleaned: string; - visibleDelta: string; - parsedStreamDirectives: ReplyDirectiveParseResult | null; - shouldUsePhaseAwareBlockReply: boolean; -}): string | undefined { - if ( - params.evtType === "text_end" || - !params.parsedStreamDirectives || - params.parsedStreamDirectives.isSilent || - hasReplyDirectiveMetadata(params.parsedStreamDirectives) || - containsCompleteMediaDirectiveLine(params.visibleDelta) || - params.parsedStreamDirectives.text !== params.visibleDelta - ) { - return undefined; - } - - if ( - !params.shouldUsePhaseAwareBlockReply && - params.previousCleaned === params.previousRawText.trim() - ) { - return params.next; - } - - const cleanedCandidate = `${params.previousCleaned}${params.parsedStreamDirectives.text}`.trim(); - return cleanedCandidate === params.next ? cleanedCandidate : undefined; -} - -function resolveStreamingReplyText(params: { - evtType: "text_delta" | "text_start" | "text_end"; - next: string; - previousRawText: string; - previousCleaned: string; - visibleDelta: string; - parsedStreamDirectives: ReplyDirectiveParseResult | null; - shouldUsePhaseAwareBlockReply: boolean; -}): string { - if (!params.parsedStreamDirectives && params.evtType === "text_delta") { - return params.previousCleaned; - } - - return ( - resolveIncrementalStreamingReplyText(params) ?? - parseReplyDirectives( - params.evtType === "text_end" ? params.next : splitTrailingDirective(params.next).text, - ).text - ); -} - -/** Records parsed reply directives until a sendable reply payload is built. */ -function recordPendingAssistantReplyDirectives( - state: Pick, - parsed: ReplyDirectiveParseResult | null | undefined, -) { - if (!hasReplyDirectiveMetadataResult(parsed)) { - return; - } - const current = state.pendingAssistantReplyDirectives; - const mediaUrls = Array.from( - new Set([...(current?.mediaUrls ?? []), ...(parsed.mediaUrls ?? [])]), - ); - state.pendingAssistantReplyDirectives = { - mediaUrls: mediaUrls.length ? mediaUrls : undefined, - audioAsVoice: current?.audioAsVoice || parsed?.audioAsVoice || undefined, - replyToId: parsed?.replyToId ?? current?.replyToId, - replyToTag: current?.replyToTag || parsed.replyToTag || undefined, - replyToCurrent: current?.replyToCurrent || parsed.replyToCurrent || undefined, - }; -} - -/** Merges pending reply directives into one reply payload and clears them. */ -export function consumePendingAssistantReplyDirectivesIntoReply( - state: Pick, - payload: BlockReplyPayload, -): BlockReplyPayload { - if (payload.isReasoning || !state.pendingAssistantReplyDirectives) { - return payload; - } - const pending = state.pendingAssistantReplyDirectives; - const mediaUrls = Array.from( - new Set([...(payload.mediaUrls ?? []), ...(pending.mediaUrls ?? [])]), - ); - state.pendingAssistantReplyDirectives = undefined; - return { - ...payload, - mediaUrls: mediaUrls.length ? mediaUrls : undefined, - audioAsVoice: payload.audioAsVoice || pending.audioAsVoice || undefined, - replyToId: payload.replyToId ?? pending.replyToId, - replyToTag: Boolean(payload.replyToTag || pending.replyToTag) || undefined, - replyToCurrent: Boolean(payload.replyToCurrent || pending.replyToCurrent) || undefined, - }; -} - -/** True when a reply payload has text, media, or voice content worth sending. */ -export function hasAssistantVisibleReply(params: { - text?: string; - mediaUrls?: string[]; - mediaUrl?: string; - audioAsVoice?: boolean; -}): boolean { - return resolveSendableOutboundReplyParts(params).hasContent || Boolean(params.audioAsVoice); -} - -/** Builds normalized stream payload data for assistant visible output. */ -function buildAssistantStreamData(params: { - text?: string; - delta?: string; - replace?: boolean; - mediaUrls?: string[]; - mediaUrl?: string; - phase?: AssistantPhase; - itemId?: string; -}): { - text: string; - delta: string; - replace?: true; - mediaUrls?: string[]; - phase?: AssistantPhase; - itemId?: string; -} { - const mediaUrls = resolveSendableOutboundReplyParts(params).mediaUrls; - return { - text: params.text ?? "", - delta: params.delta ?? "", - replace: params.replace ? true : undefined, - mediaUrls: mediaUrls.length ? mediaUrls : undefined, - phase: params.phase, - itemId: params.itemId, - }; -} - -/** Handles assistant message-start boundaries for streaming state. */ -export function handleMessageStart( - ctx: EmbeddedAgentSubscribeContext, - evt: AgentEvent & { message: AgentMessage }, -) { - const msg = evt.message; - if (msg?.role !== "assistant" || isTranscriptOnlyOpenClawAssistantMessage(msg)) { - return; - } - - // KNOWN: Resetting at `text_end` is unsafe (late/duplicate end events). - // ASSUME: `message_start` is the only reliable boundary for “new assistant message begins”. - // Start-of-message is a safer reset point than message_end: some providers - // may deliver late text_end updates after message_end, which would otherwise - // re-trigger block replies. - ctx.resetAssistantMessageState(ctx.state.assistantTexts.length); - // Use assistant message_start as the earliest "writing" signal for typing. - emitAssistantMessageStart(ctx); -} - -/** Handles assistant message deltas, reasoning, directives, and block replies. */ -export function handleMessageUpdate( - ctx: EmbeddedAgentSubscribeContext, - evt: AgentEvent & { message: AgentMessage; assistantMessageEvent?: unknown }, -) { - const msg = evt.message; - if (msg?.role !== "assistant" || isTranscriptOnlyOpenClawAssistantMessage(msg)) { - return; - } - - ctx.noteLastAssistant(msg); - const assistantEvent = evt.assistantMessageEvent; - const assistantRecord = - assistantEvent && typeof assistantEvent === "object" - ? (assistantEvent as Record) - : undefined; - const evtType = typeof assistantRecord?.type === "string" ? assistantRecord.type : ""; - const liveEditDiff = updateLiveEditDiffProgress(ctx.state.liveEditDiffStateById, assistantRecord); - if (liveEditDiff) { - const data = { phase: "input_delta", ...liveEditDiff }; - emitAgentEvent({ runId: ctx.params.runId, stream: "tool", data }); - runBestEffortCallback({ - label: "live edit diff agent event", - log: ctx.log, - callback: () => ctx.params.onAgentEvent?.({ stream: "tool", data }), - }); - } - const eventAssistantMessage = - assistantRecord?.partial && typeof assistantRecord.partial === "object" - ? (assistantRecord.partial as AssistantMessage) - : msg; - const isResponsesTextEvent = - isResponsesApiAssistantMessage(eventAssistantMessage) && - (evtType === "text_start" || evtType === "text_delta" || evtType === "text_end"); - const suppressVisibleAssistantOutput = shouldSuppressAssistantVisibleOutput(msg); - if (suppressVisibleAssistantOutput && !isResponsesTextEvent) { - const commentaryText = coerceChatContentText(extractAssistantCommentaryText(msg)); - if (commentaryText) { - appendRawStream({ - ts: Date.now(), - event: "assistant_text_stream", - runId: ctx.params.runId, - sessionId: (ctx.params.session as { id?: string }).id, - evtType: "commentary_update", - delta: "", - content: commentaryText, - }); - ctx.emitAssistantStreamData( - buildAssistantStreamData({ text: commentaryText, replace: true, phase: "commentary" }), - ); - } - return; - } - const suppressDeterministicApprovalOutput = shouldSuppressDeterministicApprovalOutput(ctx.state); - const suppressMessageToolOnlySourceReplyOutput = hasMessageToolOnlySourceDelivery(ctx); - - const assistantPhase = resolveAssistantMessagePhase(msg); - - if (evtType === "text_end" || evtType === "done" || evtType === "error") { - capturePendingAssistantUsage(ctx, evt); - if (evtType === "done" || evtType === "error") { - ctx.commitAssistantUsage(); - } - } - - if (evtType === "thinking_start" || evtType === "thinking_delta" || evtType === "thinking_end") { - if ( - !suppressMessageToolOnlySourceReplyOutput && - (evtType === "thinking_start" || evtType === "thinking_delta") - ) { - openReasoningStream(ctx); - } - const thinkingDelta = typeof assistantRecord?.delta === "string" ? assistantRecord.delta : ""; - const thinkingContent = - typeof assistantRecord?.content === "string" ? assistantRecord.content : ""; - appendRawStream({ - ts: Date.now(), - event: "assistant_thinking_stream", - runId: ctx.params.runId, - sessionId: (ctx.params.session as { id?: string }).id, - evtType, - delta: thinkingDelta, - content: thinkingContent, - }); - // Emit-always: emitReasoningStream always reaches the bus/archive; the - // streamReasoning rendering hook and message_tool_only source suppression - // are gated downstream (dispatch wrapProgressCallback, #92738), so emission - // here stays unconditional. - // Prefer full partial-message thinking when available; fall back to event payloads. - const partialThinking = extractAssistantThinking(msg); - ctx.emitReasoningStream(partialThinking || thinkingContent || thinkingDelta); - if (evtType === "thinking_end" && !suppressMessageToolOnlySourceReplyOutput) { - // Mirror the open gate above: when message-tool-only delivery has made the - // reasoning lane private, do not force-open it just to close it — that - // would fire the lane's end hook (onReasoningEnd) for a lane that never - // rendered, leaking the boundary signal. - if (!ctx.state.reasoningStreamOpen) { - openReasoningStream(ctx); - } - emitReasoningEnd(ctx); - } - return; - } - - if (evtType !== "text_delta" && evtType !== "text_start" && evtType !== "text_end") { - return; - } - - const delta = typeof assistantRecord?.delta === "string" ? assistantRecord.delta : ""; - const content = typeof assistantRecord?.content === "string" ? assistantRecord.content : ""; - - appendRawStream({ - ts: Date.now(), - event: "assistant_text_stream", - runId: ctx.params.runId, - sessionId: (ctx.params.session as { id?: string }).id, - evtType, - delta, - content, - }); - - const chunk = resolveAssistantTextChunk({ - evtType, - delta, - content, - accumulatedText: ctx.state.deltaBuffer, - }); - - const partialAssistant = eventAssistantMessage; - const streamContentIndex = resolveAssistantStreamContentIndex(assistantRecord?.contentIndex); - const streamItemId = resolveAssistantStreamItemId({ - contentIndex: streamContentIndex, - message: partialAssistant, - }); - const streamAssistant = scopeAssistantMessageToStreamBlock( - partialAssistant, - streamContentIndex, - streamItemId, - ); - const deliveryPhase = resolveAssistantMessagePhase(streamAssistant); - const isPhasePendingResponsesTextItem = - evtType !== "text_end" && - !deliveryPhase && - Boolean(streamItemId) && - isResponsesApiAssistantMessage(partialAssistant); - // These transports resolve commentary only at the tool boundary. Withhold - // early unphased deltas from durable block replies until that decision exists. - const isPhasePendingAnthropicText = - evtType !== "text_end" && !deliveryPhase && isAnthropicAssistantMessage(partialAssistant); - const isPhasePendingCompletionsText = - !deliveryPhase && isOpenAiCompletionsAssistantMessage(partialAssistant); - const hasResponsesContentIndex = - streamContentIndex !== undefined && isResponsesApiAssistantMessage(partialAssistant); - let streamItemChanged = false; - let deliveryItemId = streamItemId; - if ( - (deliveryPhase || isPhasePendingResponsesTextItem || hasResponsesContentIndex) && - (streamContentIndex !== undefined || streamItemId) - ) { - const previousStreamContentIndex = ctx.state.lastAssistantStreamContentIndex; - const previousStreamItemId = ctx.state.lastAssistantStreamItemId; - const contentIndexChanged = - previousStreamContentIndex !== undefined && - streamContentIndex !== undefined && - previousStreamContentIndex !== streamContentIndex; - const itemIdChangedWithoutIndexes = - (previousStreamContentIndex === undefined || streamContentIndex === undefined) && - Boolean(previousStreamItemId && streamItemId && previousStreamItemId !== streamItemId); - if (contentIndexChanged || itemIdChangedWithoutIndexes) { - streamItemChanged = true; - void ctx.flushBlockReplyBuffer({ assistantMessageIndex: ctx.state.assistantMessageIndex }); - ctx.resetAssistantMessageState(ctx.state.assistantTexts.length); - emitAssistantMessageStart(ctx); - } else if ( - previousStreamContentIndex !== undefined && - streamContentIndex === previousStreamContentIndex && - previousStreamItemId - ) { - // Snapshot-extension items can rotate provider ids while retaining one logical block. - // Keep the original live key so downstream commentary accumulators do not split it. - deliveryItemId = previousStreamItemId; - } - ctx.state.lastAssistantStreamContentIndex = streamContentIndex; - ctx.state.lastAssistantStreamItemId = deliveryItemId; - } - // Responses text_start snapshots may already contain text replayed by the first delta. - // Keep starts lifecycle-only so commentary and final-answer lanes consume each byte once. - if (evtType === "text_start" && isResponsesApiAssistantMessage(partialAssistant)) { - return; - } - if (deliveryPhase === "commentary") { - const isResponsesCommentary = isResponsesApiAssistantMessage(partialAssistant); - const hadResponsesCommentaryText = isResponsesCommentary && Boolean(ctx.state.deltaBuffer); - if (isResponsesCommentary && chunk) { - // Keep cumulative end events monotonic without feeding commentary into reply buffers. - ctx.state.deltaBuffer += chunk; - } - const commentaryText = - !chunk && (!isResponsesCommentary || !hadResponsesCommentaryText) - ? coerceChatContentText(extractAssistantCommentaryText(streamAssistant)) - : undefined; - const commentaryData = chunk - ? buildAssistantStreamData({ delta: chunk, phase: "commentary", itemId: deliveryItemId }) - : commentaryText - ? buildAssistantStreamData({ - text: commentaryText, - replace: true, - phase: "commentary", - itemId: deliveryItemId, - }) - : undefined; - if (commentaryData) { - ctx.emitAssistantStreamData(commentaryData); - } - return; - } - if (isPhasePendingResponsesTextItem) { - return; - } - // Subagents have no live consumer; their final result is delivered from - // message_end. Keep accumulating deltaBuffer, but skip per-chunk visible-text - // parsing so long parallel subagent streams do not monopolize the event loop. - const skipLiveStream = ctx.params.suppressLiveStreamOutput === true; - const shouldUsePhaseAwareBlockReply = Boolean(deliveryPhase); - - if (chunk) { - ctx.state.deltaBuffer += chunk; - if (!skipLiveStream && !shouldUsePhaseAwareBlockReply) { - if (!isPhasePendingAnthropicText && !isPhasePendingCompletionsText) { - appendBlockReplyChunk(ctx, chunk); - } - } - } - - if (skipLiveStream) { - return; - } - - // Handle partial tags: stream whatever reasoning is visible so far. - // Emit-always: emitReasoningStream reaches the bus/archive; rendering + - // message_tool_only suppression are gated downstream (#92738). - ctx.emitReasoningStream( - extractThinkingFromTaggedStream(ctx.state.deltaBuffer, ctx.state.thinkingTagStream), - ); - const wasThinking = ctx.state.partialBlockState.thinking; - let visibleDelta = ""; - // A text_start partial may already contain text that the following text_delta replays. - // Use starts only for lifecycle boundaries; consume their text from delta/end events. - const shouldReadScopedPartialText = - streamItemChanged || (shouldUsePhaseAwareBlockReply && (evtType === "text_end" || !chunk)); - let next = shouldReadScopedPartialText - ? coerceChatContentText(extractAssistantVisibleText(streamAssistant)).trim() - : ""; - let nextRawStreamText = next; - let shouldPersistRawStreamText = false; - if (shouldUsePhaseAwareBlockReply && !next && deliveryPhase === "final_answer" && chunk) { - visibleDelta = ctx.stripBlockTags(chunk, ctx.state.partialBlockState, { - final: evtType === "text_end", - }); - const streamVisibleText = resolveStreamVisibleText({ - previousRawText: ctx.state.lastStreamedAssistant ?? "", - visibleDelta, - }); - const previousVisibleText = sanitizeAssistantVisibleStreamText( - ctx.state.lastStreamedAssistant ?? "", - ).trim(); - next = sanitizeAssistantVisibleStreamText(streamVisibleText.rawText).trim(); - visibleDelta = resolveTextAppendDelta(previousVisibleText, next); - nextRawStreamText = streamVisibleText.rawText; - shouldPersistRawStreamText = true; - } else if (!next && deliveryPhase !== "final_answer") { - const pendingTagFragment = ctx.state.partialBlockState.pendingTagFragment; - const shouldRecomputeFullStream = Boolean(pendingTagFragment) || REASONING_TAG_RE.test(chunk); - if (shouldRecomputeFullStream) { - const recomputeState: EmbeddedAgentSubscribeState["partialBlockState"] = { - thinking: false, - final: false, - inlineCode: createInlineCodeState(), - }; - const recomputedRawText = ctx.stripBlockTags(ctx.state.deltaBuffer, recomputeState, { - final: evtType === "text_end", - }); - const previousRawText = ctx.state.lastStreamedAssistant ?? ""; - const isFullStreamReplacement = !recomputedRawText.startsWith(previousRawText); - next = recomputedRawText.trim(); - visibleDelta = isFullStreamReplacement - ? recomputedRawText - : recomputedRawText.slice(previousRawText.length); - nextRawStreamText = recomputedRawText; - copyPartialBlockState(ctx.state.partialBlockState, recomputeState); - } else { - visibleDelta = - chunk || evtType === "text_end" - ? ctx.stripBlockTags(chunk, ctx.state.partialBlockState, { - final: evtType === "text_end", - }) - : ""; - if (ctx.state.partialBlockState.pendingTagFragment) { - visibleDelta = ""; - next = ctx.state.lastStreamedAssistantCleaned ?? ""; - nextRawStreamText = ctx.state.lastStreamedAssistant ?? ""; - } else { - const streamVisibleText = resolveStreamVisibleText({ - previousRawText: ctx.state.lastStreamedAssistant ?? "", - visibleDelta, - }); - next = streamVisibleText.visibleText; - nextRawStreamText = streamVisibleText.rawText; - } - } - } else if (next && (chunk || evtType === "text_end")) { - visibleDelta = ctx.stripBlockTags(chunk, ctx.state.partialBlockState, { - final: evtType === "text_end", - }); - } - if (next) { - if ( - !suppressMessageToolOnlySourceReplyOutput && - !wasThinking && - ctx.state.partialBlockState.thinking - ) { - openReasoningStream(ctx); - } - // Detect when thinking block ends ( tag processed) - if ( - !suppressMessageToolOnlySourceReplyOutput && - wasThinking && - !ctx.state.partialBlockState.thinking - ) { - emitReasoningEnd(ctx); - } - const parsedDelta = visibleDelta ? ctx.consumePartialReplyDirectives(visibleDelta) : null; - const finalParsedDelta = - evtType === "text_end" ? ctx.consumePartialReplyDirectives("", { final: true }) : null; - const parsedStreamDirectives = mergeReplyDirectiveResults(parsedDelta, finalParsedDelta); - if (shouldUsePhaseAwareBlockReply) { - recordPendingAssistantReplyDirectives(ctx.state, parsedStreamDirectives); - } - const previousCleaned = ctx.state.lastStreamedAssistantCleaned ?? ""; - const cleanedText = resolveStreamingReplyText({ - evtType, - next, - previousRawText: ctx.state.lastStreamedAssistant ?? "", - previousCleaned, - visibleDelta, - parsedStreamDirectives, - shouldUsePhaseAwareBlockReply, - }); - const { mediaUrls, hasMedia } = resolveSendableOutboundReplyParts(parsedStreamDirectives ?? {}); - const hasAudio = Boolean(parsedStreamDirectives?.audioAsVoice); - - let shouldEmit; - let deltaText = ""; - let replace = false; - if (!hasAssistantVisibleReply({ text: cleanedText, mediaUrls, audioAsVoice: hasAudio })) { - shouldEmit = false; - } else { - replace = Boolean(previousCleaned && !cleanedText.startsWith(previousCleaned)); - deltaText = replace ? "" : cleanedText.slice(previousCleaned.length); - shouldEmit = replace - ? cleanedText !== previousCleaned || hasMedia || hasAudio - : Boolean(deltaText || hasMedia || hasAudio); - } - - if (shouldUsePhaseAwareBlockReply) { - if (replace) { - ctx.state.blockBuffer = ""; - ctx.blockChunker?.reset(); - } - const blockReplyChunk = replace ? cleanedText : deltaText; - if (blockReplyChunk) { - appendBlockReplyChunk(ctx, blockReplyChunk); - } - - if (evtType === "text_end" && !ctx.state.lastBlockReplyText && cleanedText) { - replaceBlockReplyBuffer(ctx, cleanedText); - } - } else if (streamItemChanged && !chunk) { - // An unphased equal/shrinking Responses item can end without a delta. - // Rebuild its block buffer from the scoped snapshot after the boundary reset. - appendBlockReplyChunk(ctx, cleanedText); - } - - ctx.state.lastStreamedAssistant = nextRawStreamText; - ctx.state.lastStreamedAssistantCleaned = cleanedText; - - if ( - ctx.params.silentExpected || - suppressDeterministicApprovalOutput || - suppressMessageToolOnlySourceReplyOutput - ) { - shouldEmit = false; - } - - if (shouldEmit) { - const currentSourcePartial = - ctx.params.sourceReplyDeliveryMode !== "message_tool_only" - ? resolveCurrentSourceMessagingToolPartial(ctx.state, { - evtType, - text: cleanedText, - visibleDelta, - }) - : { hold: false, text: cleanedText }; - const releaseHeldSnapshot = currentSourcePartial.text !== cleanedText; - const data = buildAssistantStreamData({ - text: currentSourcePartial.text, - delta: releaseHeldSnapshot ? currentSourcePartial.text : deltaText, - replace: releaseHeldSnapshot || replace, - mediaUrls, - phase: deliveryPhase ?? assistantPhase, - }); - ctx.emitAssistantStreamData(data, { emitPartialReply: !currentSourcePartial.hold }); - ctx.state.emittedAssistantUpdate = true; - } - } else if (shouldPersistRawStreamText) { - ctx.state.lastStreamedAssistant = nextRawStreamText; - } - - if ( - !ctx.params.silentExpected && - !suppressDeterministicApprovalOutput && - !suppressMessageToolOnlySourceReplyOutput && - ctx.params.onBlockReply && - ctx.blockChunking && - ctx.state.blockReplyBreak === "text_end" - ) { - ctx.blockChunker?.drain({ force: false, emit: ctx.emitBlockChunk }); - } - - if ( - !ctx.params.silentExpected && - !suppressDeterministicApprovalOutput && - !suppressMessageToolOnlySourceReplyOutput && - evtType === "text_end" && - ctx.state.blockReplyBreak === "text_end" - ) { - const assistantMessageIndex = ctx.state.assistantMessageIndex; - void Promise.resolve() - .then(() => ctx.flushBlockReplyBuffer({ assistantMessageIndex, final: true })) - .catch((err: unknown) => { - ctx.log.debug(`text_end block reply flush failed: ${String(err)}`); - }); - } -} - -if (process.env.VITEST || process.env.NODE_ENV === "test") { - (globalThis as Record)[ - Symbol.for("openclaw.embeddedSubscribeMessagesTestApi") - ] = { - buildAssistantStreamData, - recordPendingAssistantReplyDirectives, - resolveCurrentSourceMessagingToolPartial, - }; -} - -/** Handles assistant message-end finalization, block flush, and usage commit. */ -export function handleMessageEnd( - ctx: EmbeddedAgentSubscribeContext, - evt: AgentEvent & { message: AgentMessage }, -): void | Promise { - const msg = evt.message; - if (msg?.role !== "assistant" || isTranscriptOnlyOpenClawAssistantMessage(msg)) { - return; - } - - // Transcript-only messages never reach the provider, so this counts exactly - // the completed model round trips consumers see as `assistantTurns`. - ctx.state.assistantTurnCount += 1; - const assistantMessage = preservePendingAssistantUsage(msg, ctx.state.pendingAssistantUsage); - const assistantPhase = resolveAssistantMessagePhase(assistantMessage); - const suppressVisibleAssistantOutput = shouldSuppressAssistantVisibleOutput(assistantMessage); - const suppressDeterministicApprovalOutput = shouldSuppressDeterministicApprovalOutput(ctx.state); - const suppressMessageToolOnlySourceReplyOutput = hasMessageToolOnlySourceDelivery(ctx); - ctx.noteLastAssistant(assistantMessage); - ctx.noteCompletedAssistant(assistantMessage); - ctx.recordAssistantUsage((assistantMessage as { usage?: unknown }).usage); - ctx.commitAssistantUsage(); - if (suppressVisibleAssistantOutput) { - const isResponsesCommentary = isResponsesApiAssistantMessage(assistantMessage); - const commentaryMessage = isResponsesCommentary - ? scopeAssistantMessageToStreamBlock( - assistantMessage as AssistantMessage, - ctx.state.lastAssistantStreamContentIndex, - ctx.state.lastAssistantStreamItemId, - ) - : assistantMessage; - const commentaryText = coerceChatContentText(extractAssistantCommentaryText(commentaryMessage)); - appendRawStream({ - ts: Date.now(), - event: "assistant_message_end", - runId: ctx.params.runId, - sessionId: (ctx.params.session as { id?: string }).id, - rawText: coerceChatContentText(extractEmbeddedAssistantText(assistantMessage)), - rawThinking: extractAssistantThinking(assistantMessage), - }); - const commentaryAlreadyStreamed = - isResponsesCommentary && - Boolean(ctx.state.deltaBuffer) && - ctx.state.deltaBuffer === commentaryText; - if (commentaryText && !commentaryAlreadyStreamed) { - ctx.emitAssistantStreamData( - buildAssistantStreamData({ - text: commentaryText, - replace: true, - phase: "commentary", - itemId: isResponsesCommentary ? ctx.state.lastAssistantStreamItemId : undefined, - }), - ); - } - // Commentary-tagged tool turns can still carry durable reasoning under /reasoning on. - const suppressedTrimmedReasoning = ctx.state.includeReasoning - ? extractAssistantThinking(assistantMessage).trim() - : ""; - if ( - !ctx.params.silentExpected && - !suppressDeterministicApprovalOutput && - !suppressMessageToolOnlySourceReplyOutput && - ctx.state.includeReasoning && - suppressedTrimmedReasoning && - ctx.params.onBlockReply && - suppressedTrimmedReasoning !== ctx.state.lastReasoningSent - ) { - ctx.state.lastReasoningSent = suppressedTrimmedReasoning; - ctx.emitBlockReply({ text: suppressedTrimmedReasoning, isReasoning: true }); - } - return; - } - promoteThinkingTagsToBlocks(assistantMessage); - - const rawText = coerceChatContentText(extractEmbeddedAssistantText(assistantMessage)); - const rawVisibleText = coerceChatContentText(extractAssistantVisibleText(assistantMessage)); - appendRawStream({ - ts: Date.now(), - event: "assistant_message_end", - runId: ctx.params.runId, - sessionId: (ctx.params.session as { id?: string }).id, - rawText, - rawThinking: extractAssistantThinking(assistantMessage), - }); - warnIfAssistantEmittedSuspiciousText(ctx, assistantMessage); - const visibleText = - extractStandaloneMessageToolText(rawVisibleText, { - allowRoutedReply: isOpenAiCompletionsAssistantMessage(assistantMessage), - allowCurrentSourceReply: - ctx.params.sourceReplyDeliveryMode === "message_tool_only" && - ctx.builtinToolNames?.has("message") === true, - }) ?? rawVisibleText; - const finalVisibleText = ctx.params.enforceFinalTag - ? ctx.stripBlockTags(visibleText, { thinking: false, final: false }, { final: true }) - : visibleText; - - // Exact NO_REPLY stays silent. The legacy rewrite (silentReplyRewrite) was - // removed by contract; global messaging-tool send evidence is not a - // user-route reply and must never be mirrored into the final payload. - const text = finalVisibleText; - const rawThinking = - ctx.state.includeReasoning || ctx.state.streamReasoning - ? extractAssistantThinking(assistantMessage) || extractThinkingFromTaggedText(rawText) - : ""; - const trimmedReasoning = rawThinking ? rawThinking.trim() : ""; - const trimmedText = text.trim(); - const parsedText = trimmedText ? parseReplyDirectives(trimmedText) : null; - const cleanedText = parsedText?.text ?? ""; - const { mediaUrls, hasMedia } = resolveSendableOutboundReplyParts(parsedText ?? {}); - - const finalizeMessageEnd = () => { - ctx.state.deltaBuffer = ""; - ctx.state.thinkingTagStream = createThinkingTagStreamState(); - ctx.state.blockBuffer = ""; - ctx.blockChunker?.reset(); - ctx.state.blockState.thinking = false; - ctx.state.blockState.final = false; - ctx.state.blockState.inlineCode = createInlineCodeState(); - ctx.state.blockState.fence = undefined; - ctx.state.blockState.reasoningInlineCode = undefined; - ctx.state.blockState.reasoningFence = undefined; - ctx.state.blockState.reasoningPendingFenceFragment = undefined; - ctx.state.blockState.finalInlineCode = undefined; - ctx.state.blockState.finalFence = undefined; - ctx.state.blockState.pendingFenceFragment = undefined; - ctx.state.blockState.pendingTagFragment = undefined; - ctx.state.partialBlockState.fence = undefined; - ctx.state.partialBlockState.reasoningInlineCode = undefined; - ctx.state.partialBlockState.reasoningFence = undefined; - ctx.state.partialBlockState.reasoningPendingFenceFragment = undefined; - ctx.state.partialBlockState.finalInlineCode = undefined; - ctx.state.partialBlockState.finalFence = undefined; - ctx.state.partialBlockState.pendingFenceFragment = undefined; - ctx.state.partialBlockState.pendingTagFragment = undefined; - ctx.state.lastStreamedAssistant = undefined; - ctx.state.lastStreamedAssistantCleaned = undefined; - ctx.state.reasoningStreamOpen = false; - }; - - const previousStreamedText = ctx.state.lastStreamedAssistantCleaned ?? ""; - const shouldReplaceFinalStream = Boolean( - previousStreamedText && cleanedText && !cleanedText.startsWith(previousStreamedText), - ); - const didTextChangeWithinCurrentMessage = Boolean( - previousStreamedText && cleanedText !== previousStreamedText, - ); - const finalStreamDelta = shouldReplaceFinalStream - ? "" - : cleanedText.slice(previousStreamedText.length); - - if ( - !ctx.params.silentExpected && - !suppressDeterministicApprovalOutput && - !suppressMessageToolOnlySourceReplyOutput && - (cleanedText || hasMedia) && - (!ctx.state.emittedAssistantUpdate || - shouldReplaceFinalStream || - didTextChangeWithinCurrentMessage || - hasMedia) - ) { - const data = buildAssistantStreamData({ - text: cleanedText, - delta: finalStreamDelta, - replace: shouldReplaceFinalStream, - mediaUrls, - phase: assistantPhase, - }); - ctx.emitAssistantStreamData(data); - ctx.state.emittedAssistantUpdate = true; - ctx.state.lastStreamedAssistantCleaned = cleanedText; - } - - const silentExpectedWithoutSentinel = - ctx.params.silentExpected && !isSilentReplyText(trimmedText, SILENT_REPLY_TOKEN); - const finalAssistantText = silentExpectedWithoutSentinel ? "" : text; - const addedDuringMessage = ctx.state.assistantTexts.length > ctx.state.assistantTextBaseline; - const chunkerHasBuffered = ctx.blockChunker?.hasBuffered() ?? false; - ctx.finalizeAssistantTexts({ - text: finalAssistantText, - addedDuringMessage, - chunkerHasBuffered, - }); - - const onBlockReply = ctx.params.onBlockReply; - const shouldEmitReasoning = Boolean( - !ctx.params.silentExpected && - !suppressDeterministicApprovalOutput && - !suppressMessageToolOnlySourceReplyOutput && - ctx.state.includeReasoning && - trimmedReasoning && - onBlockReply && - trimmedReasoning !== ctx.state.lastReasoningSent, - ); - const shouldEmitReasoningBeforeAnswer = - shouldEmitReasoning && ctx.state.blockReplyBreak === "message_end" && !addedDuringMessage; - const maybeEmitReasoning = () => { - if (!shouldEmitReasoning || !trimmedReasoning) { - return; - } - ctx.state.lastReasoningSent = trimmedReasoning; - // Lane purity: the payload carries raw thinking only. Tool persistence is - // the verbose lane's job; interleaving comes from arrival order. - ctx.emitBlockReply({ text: trimmedReasoning, isReasoning: true }); - }; - - if (shouldEmitReasoningBeforeAnswer) { - maybeEmitReasoning(); - } - - const emitSplitResultAsBlockReply = ( - splitResult: ReturnType | null | undefined, - ) => { - if (!splitResult || !onBlockReply) { - return; - } - const { - text: cleanedTextLocal, - mediaUrls: mediaUrlsLocal, - audioAsVoice, - replyToId, - replyToTag, - replyToCurrent, - } = splitResult; - // Emit if there's content OR audioAsVoice flag (to propagate the flag). - if ( - hasAssistantVisibleReply({ text: cleanedTextLocal, mediaUrls: mediaUrlsLocal, audioAsVoice }) - ) { - ctx.emitBlockReply( - { - text: cleanedTextLocal, - mediaUrls: mediaUrlsLocal?.length ? mediaUrlsLocal : undefined, - audioAsVoice, - replyToId, - replyToTag, - replyToCurrent, - }, - { assistantMessageIndex: ctx.state.assistantMessageIndex }, - ); - } - }; - - const consumeFinalReplyDirectives = () => { - const bufferedResult = ctx.consumeReplyDirectives("", { final: true }); - if (!hasMedia || !parsedText) { - return bufferedResult; - } - const bufferedRawText = bufferedResult?.text ?? ""; - const leadingWhitespace = bufferedRawText.match(/^\s+/u)?.[0] ?? ""; - const strippedBufferedText = bufferedRawText ? splitMediaFromOutput(bufferedRawText).text : ""; - const bufferedText = - leadingWhitespace && - strippedBufferedText && - !strippedBufferedText.startsWith(leadingWhitespace) - ? `${leadingWhitespace}${strippedBufferedText}` - : strippedBufferedText; - return { - ...bufferedResult, - ...parsedText, - text: bufferedText, - }; - }; - - const hasBufferedBlockReply = ctx.blockChunker - ? ctx.blockChunker.hasBuffered() - : ctx.state.blockBuffer.length > 0; - - if ( - !ctx.params.silentExpected && - !suppressDeterministicApprovalOutput && - !suppressMessageToolOnlySourceReplyOutput && - text && - onBlockReply && - (ctx.state.blockReplyBreak === "message_end" || - hasBufferedBlockReply || - text !== ctx.state.lastBlockReplyText || - hasMedia) - ) { - if (hasBufferedBlockReply && ctx.blockChunker?.hasBuffered()) { - const flushBlockReplyBufferResult = ctx.flushBlockReplyBuffer({ - assistantMessageIndex: ctx.state.assistantMessageIndex, - final: true, - }); - if (isPromiseLike(flushBlockReplyBufferResult)) { - void flushBlockReplyBufferResult.catch((err: unknown) => { - ctx.log.debug(`message_end block reply flush failed: ${String(err)}`); - }); - } - // Final-flush the streaming directive accumulator so any partial - // inline reply/audio tag held back by splitTrailingDirective gets - // emitted on the message_end / blockReplyChunking path. - emitSplitResultAsBlockReply(consumeFinalReplyDirectives()); - } else if (text !== ctx.state.lastBlockReplyText || hasMedia) { - // Guard: for text_end channels, if text_end already delivered content - // (lastBlockReplyText is set), skip this safety send. The text comparison - // here uses a different stripping pipeline (stripBlockTags with reset state) - // than emitBlockChunk (stripBlockTags with running blockState + - // stripDowngradedToolCallText), which can false-positive. When text_end - // didn't deliver (e.g. commentary suppressed, provider skipped text_end), - // lastBlockReplyText is still null and message_end must deliver. - if ( - ctx.state.blockReplyBreak === "text_end" && - ctx.state.lastBlockReplyText != null && - !hasMedia - ) { - ctx.log.debug( - `Skipping message_end safety send for text_end channel - content already delivered via text_end`, - ); - } else { - // Check for duplicates before emitting (same logic as emitBlockChunk). - const normalizedText = normalizeTextForComparison(hasMedia ? cleanedText : text); - if ( - isMessagingToolDuplicateNormalized( - normalizedText, - ctx.state.messagingToolSentTextsNormalized, - ) - ) { - ctx.log.debug( - `Skipping message_end block reply - already sent via messaging tool: ${truncateUtf16Safe(text, 50)}...`, - ); - } else { - const alreadyDeliveredFinalText = Boolean( - hasMedia && cleanedText && cleanedText === ctx.state.lastBlockReplyText, - ); - ctx.state.lastBlockReplyText = hasMedia ? cleanedText || text : text; - ctx.state.lastDeliveredBlockReplyText = hasMedia ? cleanedText || text : text; - ctx.state.toolExecutionSinceLastBlockReply = false; - emitSplitResultAsBlockReply( - hasMedia && parsedText - ? { - ...parsedText, - text: alreadyDeliveredFinalText ? "" : cleanedText, - } - : ctx.consumeReplyDirectives(text, { final: true }), - ); - } - } - } - } - - if (!shouldEmitReasoningBeforeAnswer) { - maybeEmitReasoning(); - } - if (!ctx.params.silentExpected && rawThinking) { - // Emit-always: bus/archive get message-end thinking regardless of the - // streamReasoning rendering setting (gated inside emitReasoningStream). - ctx.emitReasoningStream(rawThinking); - } - - if ( - !ctx.params.silentExpected && - !suppressMessageToolOnlySourceReplyOutput && - ctx.state.blockReplyBreak === "text_end" && - onBlockReply - ) { - emitSplitResultAsBlockReply(ctx.consumeReplyDirectives("", { final: true })); - } - - if ( - !ctx.params.silentExpected && - ctx.state.blockReplyBreak === "message_end" && - ctx.params.onBlockReplyFlush - ) { - const flushBlockReplyBufferResult = ctx.flushBlockReplyBuffer(); - if (isPromiseLike(flushBlockReplyBufferResult)) { - return flushBlockReplyBufferResult - .then(() => { - const onBlockReplyFlushResult = ctx.params.onBlockReplyFlush?.({ - reason: "message_end", - }); - if (isPromiseLike(onBlockReplyFlushResult)) { - return onBlockReplyFlushResult; - } - return undefined; - }) - .finally(() => { - finalizeMessageEnd(); - }); - } - const onBlockReplyFlushResult = ctx.params.onBlockReplyFlush({ reason: "message_end" }); - if (isPromiseLike(onBlockReplyFlushResult)) { - return onBlockReplyFlushResult.finally(() => { - finalizeMessageEnd(); - }); - } - } - - finalizeMessageEnd(); - return undefined; -} -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.update-commentary.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.update-commentary.test.ts new file mode 100644 index 000000000000..1a1303bb5517 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.update-commentary.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createMessageUpdateContext, + firstMockArg, + updateMessage, +} from "./embedded-agent-subscribe.handlers.messages.test-helpers.js"; +import { + createOpenAiResponsesPartial, + createOpenAiResponsesTextBlock, + createOpenAiResponsesTextEvent as createTextUpdateEvent, +} from "./embedded-agent-subscribe.openai-responses.test-helpers.js"; + +describe("handleMessageUpdate commentary phase", () => { + it("suppresses commentary-phase partial delivery and text_end flush", async () => { + const onAgentEvent = vi.fn(); + const onPartialReply = vi.fn(); + const flushBlockReplyBuffer = vi.fn(); + const ctx = createMessageUpdateContext({ + onAgentEvent, + onPartialReply, + flushBlockReplyBuffer, + }); + + updateMessage( + ctx, + createTextUpdateEvent({ type: "text_delta", text: "Need send.", messagePhase: "commentary" }), + ); + updateMessage( + ctx, + createTextUpdateEvent({ type: "text_end", text: "Need send.", messagePhase: "commentary" }), + ); + + await Promise.resolve(); + + expect(onAgentEvent).not.toHaveBeenCalled(); + expect(onPartialReply).not.toHaveBeenCalled(); + expect(flushBlockReplyBuffer).not.toHaveBeenCalled(); + }); + + it("suppresses commentary partials when phase exists only in textSignature metadata", async () => { + const onAgentEvent = vi.fn(); + const onPartialReply = vi.fn(); + const flushBlockReplyBuffer = vi.fn(); + const commentaryBlock = createOpenAiResponsesTextBlock({ + text: "Need send.", + id: "msg_sig", + phase: "commentary", + }); + const ctx = createMessageUpdateContext({ + onAgentEvent, + onPartialReply, + flushBlockReplyBuffer, + }); + + updateMessage( + ctx, + createTextUpdateEvent({ + type: "text_delta", + text: "Need send.", + content: [commentaryBlock], + }), + ); + updateMessage( + ctx, + createTextUpdateEvent({ + type: "text_end", + text: "Need send.", + content: [commentaryBlock], + }), + ); + + await Promise.resolve(); + + // Archive-always: commentary (textSignature-only phase — the F3 shape) is + // emitted on the bus for archival + window, but kept out of the reply lanes. + expect(onAgentEvent).toHaveBeenCalled(); + expect(onPartialReply).not.toHaveBeenCalled(); + expect(flushBlockReplyBuffer).not.toHaveBeenCalled(); + expect(ctx.state.deltaBuffer).toBe(""); + expect(ctx.state.blockBuffer).toBe(""); + }); + + it("keeps commentary partials out of reply lanes while emitting them on the bus", () => { + const onAgentEvent = vi.fn(); + const ctx = createMessageUpdateContext({ + onAgentEvent, + shouldEmitPartialReplies: false, + }); + + updateMessage( + ctx, + createTextUpdateEvent({ + type: "text_delta", + text: "Working...", + partial: createOpenAiResponsesPartial({ + text: "Working...", + id: "item_commentary", + signaturePhase: "commentary", + partialPhase: "commentary", + }), + }), + ); + + // Emit-always: the bus sees the commentary delta with its phase tag. The raw + // cumulative buffer retains it for end-event dedupe, but reply blocks stay untouched. + expect(onAgentEvent).toHaveBeenCalledTimes(1); + const commentaryEvent = firstMockArg(onAgentEvent, "agent event") as + | { stream?: string; data?: { delta?: string; phase?: string } } + | undefined; + expect(commentaryEvent?.stream).toBe("assistant"); + expect(commentaryEvent?.data?.phase).toBe("commentary"); + expect(commentaryEvent?.data?.delta).toBe("Working..."); + expect(ctx.state.deltaBuffer).toBe("Working..."); + expect(ctx.state.blockBuffer).toBe(""); + + updateMessage( + ctx, + createTextUpdateEvent({ + type: "text_delta", + text: "Done.", + partial: createOpenAiResponsesPartial({ + text: "Done.", + id: "item_final", + signaturePhase: "final_answer", + partialPhase: "final_answer", + }), + }), + ); + + expect(onAgentEvent).toHaveBeenCalledTimes(2); + const event = onAgentEvent.mock.calls[1]?.[0] as + | { stream?: string; data?: { text?: string; delta?: string } } + | undefined; + expect(event?.stream).toBe("assistant"); + expect(event?.data?.text).toBe("Done."); + expect(event?.data?.delta).toBe("Done."); + }); + + it("contains synchronous text_end flush failures", async () => { + const debug = vi.fn(); + const ctx = createMessageUpdateContext({ + debug, + shouldEmitPartialReplies: false, + flushBlockReplyBuffer: vi.fn(() => { + throw new Error("boom"); + }), + }); + + updateMessage(ctx, createTextUpdateEvent({ type: "text_end", text: "" })); + + await vi.waitFor(() => { + expect(debug).toHaveBeenCalledWith("text_end block reply flush failed: Error: boom"); + }); + }); +}); diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.update-source.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.update-source.test.ts new file mode 100644 index 000000000000..5ab8c6569655 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.update-source.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createMessageUpdateContext, + updateMessage, +} from "./embedded-agent-subscribe.handlers.messages.test-helpers.js"; +import { resolveCurrentSourceMessagingToolPartial } from "./embedded-agent-subscribe.handlers.messages.test-support.js"; +import { createOpenAiResponsesTextEvent as createTextUpdateEvent } from "./embedded-agent-subscribe.openai-responses.test-helpers.js"; + +describe("handleMessageUpdate current-source message-tool previews", () => { + it("holds delta-only continuation fragments and releases one full divergent snapshot", () => { + const state = { + currentSourceMessagingToolHeldPartial: undefined as string | undefined, + currentSourceMessagingToolSentTextsNormalized: ["qa-msteams-dm-ok"], + }; + + expect( + resolveCurrentSourceMessagingToolPartial(state, { + evtType: "text_delta", + text: "QA-MSTEAMS", + visibleDelta: "QA-MSTEAMS", + }), + ).toEqual({ hold: true, text: "QA-MSTEAMS" }); + expect( + resolveCurrentSourceMessagingToolPartial(state, { + evtType: "text_delta", + text: "-DM-OK", + visibleDelta: "-DM-OK", + }), + ).toEqual({ hold: true, text: "QA-MSTEAMS-DM-OK" }); + expect( + resolveCurrentSourceMessagingToolPartial(state, { + evtType: "text_delta", + text: " with more detail", + visibleDelta: " with more detail", + }), + ).toEqual({ hold: false, text: "QA-MSTEAMS-DM-OK with more detail" }); + expect(state.currentSourceMessagingToolHeldPartial).toBeUndefined(); + }); + + it("holds automatic partial prefixes and exact duplicates after source delivery", () => { + const onAgentEvent = vi.fn(); + const onPartialReply = vi.fn(); + const sentText = "QA-MSTEAMS-DM-OK"; + const context = createMessageUpdateContext({ + onAgentEvent, + onPartialReply, + sourceReplyDeliveryMode: "automatic", + state: { + currentSourceMessagingToolSentTextsNormalized: [sentText.toLowerCase()], + }, + }); + + updateMessage( + context, + createTextUpdateEvent({ + type: "text_delta", + text: "QA-MSTEAMS", + id: "msg_source_duplicate", + }), + ); + updateMessage( + context, + createTextUpdateEvent({ + type: "text_end", + text: sentText, + id: "msg_source_duplicate", + }), + ); + + expect(onAgentEvent).toHaveBeenCalledTimes(1); + expect(onPartialReply).not.toHaveBeenCalled(); + }); + + it("releases the full cumulative snapshot when automatic text diverges", () => { + const onPartialReply = vi.fn(); + const sentText = "QA-MSTEAMS-DM-OK"; + const context = createMessageUpdateContext({ + onPartialReply, + sourceReplyDeliveryMode: "automatic", + state: { + currentSourceMessagingToolSentTextsNormalized: [sentText.toLowerCase()], + }, + }); + + updateMessage( + context, + createTextUpdateEvent({ + type: "text_delta", + text: "QA-MSTEAMS", + id: "msg_source_diverges", + }), + ); + updateMessage( + context, + createTextUpdateEvent({ + type: "text_end", + text: `${sentText} with more detail`, + id: "msg_source_diverges", + }), + ); + + expect(onPartialReply).toHaveBeenCalledTimes(1); + expect(onPartialReply).toHaveBeenCalledWith( + expect.objectContaining({ text: `${sentText} with more detail` }), + ); + }); + + it("keeps unrelated automatic partial text visible", () => { + const onPartialReply = vi.fn(); + const context = createMessageUpdateContext({ + onPartialReply, + sourceReplyDeliveryMode: "automatic", + state: { + currentSourceMessagingToolSentTextsNormalized: ["qa-msteams-dm-ok"], + }, + }); + + updateMessage( + context, + createTextUpdateEvent({ + type: "text_end", + text: "A genuinely different answer", + id: "msg_source_different", + }), + ); + + expect(onPartialReply).toHaveBeenCalledWith( + expect.objectContaining({ text: "A genuinely different answer" }), + ); + }); +}); diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.update-stream-items.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.update-stream-items.test.ts new file mode 100644 index 000000000000..4c2b31449082 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.update-stream-items.test.ts @@ -0,0 +1,399 @@ +import { describe, expect, it, vi } from "vitest"; +import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js"; +import { + createMessageUpdateContext, + endMessage, + firstMockArg, + updateMessage, +} from "./embedded-agent-subscribe.handlers.messages.test-helpers.js"; +import { + createOpenAiResponsesPartial, + createOpenAiResponsesTextEvent as createTextUpdateEvent, +} from "./embedded-agent-subscribe.openai-responses.test-helpers.js"; + +describe("handleMessageUpdate text signatures", () => { + it("emits the full incrementally extracted reasoning value on every delta", () => { + const emitReasoningStream = vi.fn(); + const context = createMessageUpdateContext({ emitReasoningStream }); + + for (const chunk of ["reason", "ing"]) { + updateMessage( + context, + createTextUpdateEvent({ type: "text_delta", text: chunk, delta: chunk }), + ); + } + + expect(emitReasoningStream.mock.calls.map(([text]) => text)).toEqual([ + "", + "reason", + "reasoning", + ]); + }); + + it("uses incremental text deltas for unphased OpenAI Responses streams", () => { + const onAgentEvent = vi.fn(); + const stripBlockTags = vi.fn((text: string) => text); + const context = createMessageUpdateContext({ onAgentEvent, stripBlockTags }); + + const createNonPhaseEvent = (text: string, delta: string) => + ({ + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + contentIndex: 0, + delta, + partial: { + role: "assistant", + content: [{ type: "text", text }], + stopReason: "stop", + api: "openai-responses", + provider: "openai", + model: "gpt-5.2", + usage: {}, + timestamp: 0, + }, + }, + }) as never; + + updateMessage(context, createNonPhaseEvent("Hello ", "Hello ")); + updateMessage(context, createNonPhaseEvent("Hello world", "world")); + + expect(stripBlockTags.mock.calls.map(([text]) => text)).toEqual(["Hello ", "world"]); + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + { + stream: "assistant", + data: { text: "Hello", delta: "Hello" }, + }, + { + stream: "assistant", + data: { text: "Hello world", delta: " world" }, + }, + ]); + }); + + it("treats unphased OpenAI Responses content-index changes as message boundaries", () => { + const flushBlockReplyBuffer = vi.fn(); + const onAssistantMessageStart = vi.fn(); + const onPartialReply = vi.fn(); + const context = createMessageUpdateContext({ + flushBlockReplyBuffer, + onPartialReply, + state: { + deltaBuffer: "First block", + lastStreamedAssistant: "First block", + lastStreamedAssistantCleaned: "First block", + lastAssistantStreamContentIndex: 0, + }, + }); + const resetAssistantMessageState = vi.fn(() => { + context.state.deltaBuffer = ""; + context.state.lastStreamedAssistant = undefined; + context.state.lastStreamedAssistantCleaned = undefined; + }); + context.resetAssistantMessageState = resetAssistantMessageState; + context.params.onAssistantMessageStart = onAssistantMessageStart; + + updateMessage(context, { + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_end", + contentIndex: 1, + content: "First block", + partial: { + role: "assistant", + content: [ + { type: "text", text: "First block" }, + { type: "text", text: "First block" }, + ], + api: "openai-responses", + }, + }, + }); + + expect(flushBlockReplyBuffer).toHaveBeenCalledTimes(1); + expect(resetAssistantMessageState).toHaveBeenCalledTimes(1); + expect(onAssistantMessageStart).toHaveBeenCalledTimes(1); + expect(onPartialReply).toHaveBeenCalledWith( + expect.objectContaining({ text: "First block", delta: "First block" }), + ); + expect(context.state.blockBuffer).toBe("First block"); + expect(context.state.lastAssistantStreamContentIndex).toBe(1); + }); + + it("holds incomplete streaming directive tails without emitting them as text", () => { + const onAgentEvent = vi.fn(); + const accumulator = createStreamingDirectiveAccumulator(); + const context = createMessageUpdateContext({ + onAgentEvent, + consumePartialReplyDirectives: vi.fn((text: string, options?: { final?: boolean }) => + accumulator.consume(text, options), + ), + }); + + const createNonPhaseEvent = (delta: string) => + ({ + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + delta, + }, + }) as never; + + updateMessage(context, createNonPhaseEvent("Hello\n")); + updateMessage(context, createNonPhaseEvent("M")); + + expect(onAgentEvent).toHaveBeenCalledTimes(1); + expect(firstMockArg(onAgentEvent, "agent event")).toMatchObject({ + stream: "assistant", + data: { text: "Hello", delta: "Hello" }, + }); + expect(context.state.lastStreamedAssistantCleaned).toBe("Hello"); + }); + + it.each([ + { + name: "the directive accumulator has no parsed result", + text: "answer part A msg [[E1008]timeout] answer part B", + hasParsedDirectives: false, + }, + { + name: "the directive accumulator flushes a buffered tail", + text: "answer part A msg [[E1008]timeout] answer part B", + hasParsedDirectives: true, + }, + { + name: "the final text ends with one bracket", + text: "answer part A [", + hasParsedDirectives: true, + }, + ])("keeps literal final text when $name", ({ text, hasParsedDirectives }) => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ + onAgentEvent, + ...(hasParsedDirectives ? {} : { consumePartialReplyDirectives: vi.fn(() => null) }), + }); + + updateMessage(context, { + message: { role: "assistant", content: [] }, + assistantMessageEvent: { type: "text_end", content: text }, + }); + + expect(context.state.lastStreamedAssistantCleaned).toBe(text); + expect(firstMockArg(onAgentEvent, "final assistant event")).toMatchObject({ + stream: "assistant", + data: { text }, + }); + }); + + it("keeps stripped reply directives out of later plain deltas", () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + + const createNonPhaseEvent = (delta: string) => + ({ + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + delta, + }, + }) as never; + + updateMessage(context, createNonPhaseEvent("[[reply_to_current]]\nHello")); + updateMessage(context, createNonPhaseEvent(" world")); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + { + stream: "assistant", + data: { text: "Hello", delta: "Hello" }, + }, + { + stream: "assistant", + data: { text: "Hello world", delta: " world" }, + }, + ]); + }); + + it("does not expose complete legacy media directives on plain deltas", () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + + updateMessage(context, { + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + delta: "Here it is.\nMEDIA:/tmp/final.png\n", + }, + }); + + expect(firstMockArg(onAgentEvent, "agent event")).toMatchObject({ + stream: "assistant", + data: { text: "Here it is.", delta: "Here it is." }, + }); + }); + + it("uses full partial text for suffix deltas after a suppressed commentary item", () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + + updateMessage( + context, + createTextUpdateEvent({ + type: "text_delta", + text: "Hello", + delta: "Hello", + id: "item-commentary", + signaturePhase: "commentary", + partialPhase: "commentary", + }), + ); + updateMessage( + context, + createTextUpdateEvent({ + type: "text_delta", + text: "Hello world", + delta: " world", + id: "item-final", + signaturePhase: "final_answer", + partialPhase: "final_answer", + }), + ); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + // Emit-always: the commentary delta reaches the bus tagged with its + // phase; reply lanes still exclude it (covered below). + { + stream: "assistant", + data: { delta: "Hello", phase: "commentary", itemId: "item-commentary" }, + }, + { + stream: "assistant", + data: { text: "Hello world", delta: "Hello world", phase: "final_answer" }, + }, + ]); + }); + + it.each([ + "openai-responses", + "openai-chatgpt-responses", + "openclaw-openai-responses-transport", + "openclaw-openai-chatgpt-responses-transport", + "openclaw-azure-openai-responses-transport", + ])("streams %s commentary bytes exactly once across start, deltas, and end", async (api) => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + const createPartial = (text: string) => ({ + ...createOpenAiResponsesPartial({ + text, + id: "item-commentary", + signaturePhase: "commentary", + partialPhase: "commentary", + }), + api, + }); + const startPartial = createPartial("Work"); + const finalPartial = createPartial("Working..."); + + updateMessage(context, { + message: startPartial, + assistantMessageEvent: { + type: "text_start", + contentIndex: 0, + partial: startPartial, + }, + }); + updateMessage(context, { + message: startPartial, + assistantMessageEvent: { + type: "text_delta", + contentIndex: 0, + delta: "Work", + partial: startPartial, + }, + }); + updateMessage(context, { + message: finalPartial, + assistantMessageEvent: { + type: "text_delta", + contentIndex: 0, + delta: "ing...", + partial: finalPartial, + }, + }); + updateMessage(context, { + message: finalPartial, + assistantMessageEvent: { + type: "text_end", + contentIndex: 0, + content: "Working...", + partial: finalPartial, + }, + }); + await endMessage(context, { + message: finalPartial, + }); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + { + stream: "assistant", + data: { delta: "Work", phase: "commentary", itemId: "item-commentary" }, + }, + { + stream: "assistant", + data: { delta: "ing...", phase: "commentary", itemId: "item-commentary" }, + }, + ]); + expect(context.state.deltaBuffer).toBe("Working..."); + expect(context.state.blockBuffer).toBe(""); + }); + + it("keeps same-index commentary snapshot extensions on the original live item key", async () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + const createPartial = (text: string, id: string) => + createOpenAiResponsesPartial({ + text, + id, + signaturePhase: "commentary", + partialPhase: "commentary", + }); + const firstPartial = createPartial("Working", "item-1"); + const extendedPartial = createPartial("Working now", "item-2"); + + updateMessage(context, { + message: firstPartial, + assistantMessageEvent: { type: "text_start", contentIndex: 0, partial: firstPartial }, + }); + updateMessage(context, { + message: firstPartial, + assistantMessageEvent: { + type: "text_end", + contentIndex: 0, + content: "Working", + partial: firstPartial, + }, + }); + updateMessage(context, { + message: extendedPartial, + assistantMessageEvent: { + type: "text_end", + contentIndex: 0, + content: "Working now", + partial: extendedPartial, + }, + }); + await endMessage(context, { message: extendedPartial }); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + { + stream: "assistant", + data: { delta: "Working", phase: "commentary", itemId: "item-1" }, + }, + { + stream: "assistant", + data: { delta: " now", phase: "commentary", itemId: "item-1" }, + }, + ]); + expect(context.state.lastAssistantStreamItemId).toBe("item-1"); + expect(context.state.deltaBuffer).toBe("Working now"); + }); +}); diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.update-stream-phases.test.ts b/src/agents/embedded-agent-subscribe.handlers.messages.update-stream-phases.test.ts new file mode 100644 index 000000000000..0ae89a5140ef --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.update-stream-phases.test.ts @@ -0,0 +1,462 @@ +import { describe, expect, it, vi } from "vitest"; +import { createStreamingDirectiveAccumulator } from "../auto-reply/reply/streaming-directives.js"; +import { consumePendingAssistantReplyDirectivesIntoReply } from "./embedded-agent-subscribe.handlers.messages.replies.js"; +import { + createMessageUpdateContext, + updateMessage, +} from "./embedded-agent-subscribe.handlers.messages.test-helpers.js"; +import { + createOpenAiResponsesPartial, + createOpenAiResponsesTextBlock, + createOpenAiResponsesTextEvent as createTextUpdateEvent, +} from "./embedded-agent-subscribe.openai-responses.test-helpers.js"; + +describe("handleMessageUpdate text signatures", () => { + it("emits a commentary snapshot when Anthropic text is classified after deltas", () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + const narration = "I'll check the repo first."; + const commentaryPartial = { + role: "assistant", + api: "anthropic-messages", + content: [ + { + type: "text", + text: narration, + textSignature: JSON.stringify({ v: 1, id: "commentary-0", phase: "commentary" }), + }, + ], + }; + + updateMessage(context, { + message: { + role: "assistant", + api: "anthropic-messages", + content: [{ type: "text", text: narration }], + }, + assistantMessageEvent: { type: "text_delta", delta: narration }, + }); + updateMessage(context, { + message: { role: "assistant", api: "anthropic-messages", content: [] }, + assistantMessageEvent: { + type: "text_end", + content: narration, + partial: commentaryPartial, + }, + }); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toContainEqual( + expect.objectContaining({ + stream: "assistant", + data: expect.objectContaining({ + text: narration, + replace: true, + phase: "commentary", + itemId: "commentary-0", + }), + }), + ); + }); + + it("uses incremental deltas for same-item phased streams", () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + const signature = JSON.stringify({ v: 1, id: "item-final", phase: "final_answer" }); + const partial = { + role: "assistant", + phase: "final_answer", + content: [ + { + type: "text", + textSignature: signature, + get text() { + throw new Error("full partial text should not be read"); + }, + }, + ], + }; + + const createPhasedDelta = (delta: string) => + ({ + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + delta, + partial, + }, + }) as never; + + updateMessage(context, createPhasedDelta("Hello")); + updateMessage(context, createPhasedDelta(" world")); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + { + stream: "assistant", + data: { text: "Hello", delta: "Hello", phase: "final_answer" }, + }, + { + stream: "assistant", + data: { text: "Hello world", delta: " world", phase: "final_answer" }, + }, + ]); + }); + + it("keeps same-item phased stream deltas on the user-visible sanitizer path", () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + const signature = JSON.stringify({ v: 1, id: "item-final", phase: "final_answer" }); + const partial = { + role: "assistant", + phase: "final_answer", + content: [ + { + type: "text", + textSignature: signature, + get text() { + throw new Error("full partial text should not be read"); + }, + }, + ], + }; + + const createPhasedDelta = (delta: string) => + ({ + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + delta, + partial, + }, + }) as never; + + updateMessage(context, createPhasedDelta("Visible\n{")); + updateMessage( + context, + createPhasedDelta('"name":"read","arguments":{"file_path":"secret.md"}}'), + ); + updateMessage(context, createPhasedDelta("\nDone.")); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + { + stream: "assistant", + data: { text: "Visible", delta: "Visible", phase: "final_answer" }, + }, + { + stream: "assistant", + data: { text: "Visible\n\nDone.", delta: "\n\nDone.", phase: "final_answer" }, + }, + ]); + }); + + it("keeps sanitizer context when a same-item phased stream starts hidden", () => { + const onAgentEvent = vi.fn(); + const context = createMessageUpdateContext({ onAgentEvent }); + const signature = JSON.stringify({ v: 1, id: "item-final", phase: "final_answer" }); + const partial = { + role: "assistant", + phase: "final_answer", + content: [ + { + type: "text", + textSignature: signature, + get text() { + throw new Error("full partial text should not be read"); + }, + }, + ], + }; + + const createPhasedDelta = (delta: string) => + ({ + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + delta, + partial, + }, + }) as never; + + updateMessage(context, createPhasedDelta("{")); + updateMessage( + context, + createPhasedDelta('"name":"read","arguments":{"file_path":"secret.md"}}\nDone.'), + ); + + expect(onAgentEvent.mock.calls.map(([event]) => event)).toMatchObject([ + { + stream: "assistant", + data: { text: "Done.", delta: "Done.", phase: "final_answer" }, + }, + ]); + }); + + it("treats phased textSignature item changes as assistant-message boundaries", () => { + const flushBlockReplyBuffer = vi.fn(); + const resetAssistantMessageState = vi.fn(); + const onAssistantMessageStart = vi.fn(); + const onPartialReply = vi.fn(); + const context = createMessageUpdateContext({ + flushBlockReplyBuffer, + resetAssistantMessageState, + onPartialReply, + }); + context.params.onAssistantMessageStart = onAssistantMessageStart; + context.state.lastAssistantStreamContentIndex = 0; + context.state.lastAssistantStreamItemId = "item-1"; + context.state.assistantMessageIndex = 7; + + updateMessage(context, { + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + contentIndex: 1, + delta: "Second block", + partial: { + role: "assistant", + phase: "final_answer", + content: [ + createOpenAiResponsesTextBlock({ + text: "First block", + id: "item-1", + phase: "final_answer", + }), + createOpenAiResponsesTextBlock({ + text: "Second block", + id: "item-2", + phase: "final_answer", + }), + ], + stopReason: "stop", + api: "openai-responses", + provider: "openai", + model: "gpt-5.2", + usage: {}, + timestamp: 0, + }, + }, + }); + + expect(flushBlockReplyBuffer).toHaveBeenCalledWith({ assistantMessageIndex: 7 }); + expect(resetAssistantMessageState).toHaveBeenCalledWith(0); + expect(onAssistantMessageStart).toHaveBeenCalledTimes(1); + expect(onPartialReply).toHaveBeenCalledWith( + expect.objectContaining({ + text: "Second block", + delta: "Second block", + phase: "final_answer", + }), + ); + expect(onPartialReply).not.toHaveBeenCalledWith( + expect.objectContaining({ text: "First block\nSecond block" }), + ); + expect(context.state.lastAssistantStreamContentIndex).toBe(1); + expect(context.state.lastAssistantStreamItemId).toBe("item-2"); + }); + + it("does not replay a deferred item snapshot before its first delta", () => { + const flushBlockReplyBuffer = vi.fn(); + const resetAssistantMessageState = vi.fn(); + const onAssistantMessageStart = vi.fn(); + const onPartialReply = vi.fn(); + const context = createMessageUpdateContext({ + flushBlockReplyBuffer, + resetAssistantMessageState, + onPartialReply, + state: { + lastAssistantStreamContentIndex: 0, + lastAssistantStreamItemId: "item-1", + }, + }); + context.params.onAssistantMessageStart = onAssistantMessageStart; + const partial = { + role: "assistant", + phase: "final_answer", + content: [ + createOpenAiResponsesTextBlock({ + text: "First block", + id: "item-1", + phase: "final_answer", + }), + createOpenAiResponsesTextBlock({ + text: "Second block", + id: "item-2", + phase: "final_answer", + }), + ], + api: "openai-responses", + }; + + updateMessage(context, { + message: partial, + assistantMessageEvent: { + type: "text_start", + contentIndex: 1, + partial, + }, + }); + updateMessage(context, { + message: partial, + assistantMessageEvent: { + type: "text_delta", + contentIndex: 1, + delta: "Second block", + }, + }); + + expect(flushBlockReplyBuffer).toHaveBeenCalledTimes(1); + expect(resetAssistantMessageState).toHaveBeenCalledTimes(1); + expect(onAssistantMessageStart).toHaveBeenCalledTimes(1); + expect(onPartialReply).toHaveBeenCalledTimes(1); + expect(onPartialReply).toHaveBeenCalledWith( + expect.objectContaining({ + text: "Second block", + delta: "Second block", + phase: "final_answer", + }), + ); + }); + + it("keeps same-block OpenAI Responses snapshot extensions in one assistant message", () => { + const flushBlockReplyBuffer = vi.fn(); + const resetAssistantMessageState = vi.fn(); + const onAssistantMessageStart = vi.fn(); + const onPartialReply = vi.fn(); + const context = createMessageUpdateContext({ + flushBlockReplyBuffer, + resetAssistantMessageState, + onPartialReply, + state: { + deltaBuffer: "First block", + lastStreamedAssistant: "First block", + lastStreamedAssistantCleaned: "First block", + lastAssistantStreamContentIndex: 0, + lastAssistantStreamItemId: "item-1", + }, + }); + context.params.onAssistantMessageStart = onAssistantMessageStart; + + updateMessage(context, { + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_end", + contentIndex: 0, + content: "First block extended", + partial: createOpenAiResponsesPartial({ + text: "First block extended", + id: "item-2", + signaturePhase: "final_answer", + partialPhase: "final_answer", + }), + }, + }); + + expect(flushBlockReplyBuffer).not.toHaveBeenCalled(); + expect(resetAssistantMessageState).not.toHaveBeenCalled(); + expect(onAssistantMessageStart).not.toHaveBeenCalled(); + expect(onPartialReply).toHaveBeenCalledWith( + expect.objectContaining({ + text: "First block extended", + delta: " extended", + phase: "final_answer", + }), + ); + expect(context.state.lastAssistantStreamContentIndex).toBe(0); + expect(context.state.lastAssistantStreamItemId).toBe("item-1"); + }); + + it("scopes item-id fallback boundaries to the matching signed block", () => { + const onPartialReply = vi.fn(); + const resetAssistantMessageState = vi.fn(); + const context = createMessageUpdateContext({ + onPartialReply, + resetAssistantMessageState, + state: { lastAssistantStreamItemId: "item-1" }, + }); + + updateMessage(context, { + message: { role: "assistant", content: [] }, + assistantMessageEvent: { + type: "text_delta", + delta: "Second block", + partial: { + role: "assistant", + phase: "final_answer", + content: [ + createOpenAiResponsesTextBlock({ + text: "First block", + id: "item-1", + phase: "final_answer", + }), + createOpenAiResponsesTextBlock({ + text: "Second block", + id: "item-2", + phase: "final_answer", + }), + ], + api: "openai-responses", + }, + }, + }); + + expect(resetAssistantMessageState).toHaveBeenCalledTimes(1); + expect(onPartialReply).toHaveBeenCalledWith( + expect.objectContaining({ + text: "Second block", + delta: "Second block", + phase: "final_answer", + }), + ); + expect(onPartialReply).not.toHaveBeenCalledWith( + expect.objectContaining({ text: "First block\nSecond block" }), + ); + expect(context.state.lastAssistantStreamContentIndex).toBeUndefined(); + expect(context.state.lastAssistantStreamItemId).toBe("item-2"); + }); + + it("preserves phase-aware voice and reply directives while deferring final media delivery", () => { + const accumulator = createStreamingDirectiveAccumulator(); + const ctx = createMessageUpdateContext({ + consumePartialReplyDirectives: vi.fn((text: string, options?: { final?: boolean }) => + accumulator.consume(text, options), + ), + state: { + blockReplyBreak: "message_end", + }, + }); + const replyText = "Done.\n\n[[reply_to_current]]\n[[audio_as_voice]]\nMEDIA:/tmp/reply.ogg"; + + updateMessage( + ctx, + createTextUpdateEvent({ + type: "text_delta", + text: replyText, + id: "item-final", + signaturePhase: "final_answer", + partialPhase: "final_answer", + }), + ); + updateMessage( + ctx, + createTextUpdateEvent({ + type: "text_end", + text: replyText, + id: "item-final", + signaturePhase: "final_answer", + partialPhase: "final_answer", + }), + ); + + expect(ctx.state.blockBuffer).toBe("Done."); + expect( + consumePendingAssistantReplyDirectivesIntoReply(ctx.state, { + text: "Done.", + }), + ).toEqual({ + text: "Done.", + audioAsVoice: true, + replyToId: undefined, + replyToTag: true, + replyToCurrent: true, + }); + }); +}); diff --git a/src/agents/embedded-agent-subscribe.handlers.messages.update.ts b/src/agents/embedded-agent-subscribe.handlers.messages.update.ts new file mode 100644 index 000000000000..9c7c4b0e2b02 --- /dev/null +++ b/src/agents/embedded-agent-subscribe.handlers.messages.update.ts @@ -0,0 +1,510 @@ +/** + * Handles assistant message deltas, reasoning, directives, and block replies. + */ +import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload"; +import { createInlineCodeState } from "../../packages/markdown-core/src/code-spans.js"; +import { emitAgentEvent } from "../infra/agent-events.js"; +import type { AssistantMessage } from "../llm/types.js"; +import { coerceChatContentText } from "../shared/chat-content.js"; +import { resolveAssistantMessagePhase } from "../shared/chat-message-content.js"; +import { updateLiveEditDiffProgress } from "./embedded-agent-live-edit-diff.js"; +import { runBestEffortCallback } from "./embedded-agent-subscribe.callback.js"; +import { capturePendingAssistantUsage } from "./embedded-agent-subscribe.handlers.messages.lifecycle.js"; +import { + hasAssistantVisibleReply, + mergeReplyDirectiveResults, + recordPendingAssistantReplyDirectives, +} from "./embedded-agent-subscribe.handlers.messages.replies.js"; +import { + appendBlockReplyChunk, + buildAssistantStreamData, + copyPartialBlockState, + emitAssistantMessageStart, + emitReasoningEnd, + hasMessageToolOnlySourceDelivery, + isAnthropicAssistantMessage, + isOpenAiCompletionsAssistantMessage, + isResponsesApiAssistantMessage, + isSubscribeTranscriptOnlyOpenClawAssistantMessage, + openReasoningStream, + replaceBlockReplyBuffer, + resolveAssistantStreamContentIndex, + resolveAssistantStreamItemId, + resolveAssistantTextChunk, + resolveCurrentSourceMessagingToolPartial, + resolveStreamVisibleText, + resolveStreamingReplyText, + resolveTextAppendDelta, + scopeAssistantMessageToStreamBlock, + shouldSuppressAssistantVisibleOutput, + shouldSuppressDeterministicApprovalOutput, +} from "./embedded-agent-subscribe.handlers.messages.stream.js"; +import type { + EmbeddedAgentSubscribeContext, + EmbeddedAgentSubscribeState, +} from "./embedded-agent-subscribe.handlers.types.js"; +import { appendRawStream } from "./embedded-agent-subscribe.raw-stream.js"; +import { + extractAssistantCommentaryText, + extractAssistantThinking, + extractAssistantVisibleText, + extractThinkingFromTaggedStream, + sanitizeAssistantVisibleStreamText, +} from "./embedded-agent-utils.js"; +import type { AgentEvent, AgentMessage } from "./runtime/index.js"; + +const REASONING_TAG_RE = /<\s*\/?\s*(?:(?:antml:|mm:)?(?:think(?:ing)?|thought)|antthinking)\b/i; + +export function handleMessageUpdate( + ctx: EmbeddedAgentSubscribeContext, + evt: AgentEvent & { message: AgentMessage; assistantMessageEvent?: unknown }, +) { + const msg = evt.message; + if (msg?.role !== "assistant" || isSubscribeTranscriptOnlyOpenClawAssistantMessage(msg)) { + return; + } + + ctx.noteLastAssistant(msg); + const assistantEvent = evt.assistantMessageEvent; + const assistantRecord = + assistantEvent && typeof assistantEvent === "object" + ? (assistantEvent as Record) + : undefined; + const evtType = typeof assistantRecord?.type === "string" ? assistantRecord.type : ""; + const liveEditDiff = updateLiveEditDiffProgress(ctx.state.liveEditDiffStateById, assistantRecord); + if (liveEditDiff) { + const data = { phase: "input_delta", ...liveEditDiff }; + emitAgentEvent({ runId: ctx.params.runId, stream: "tool", data }); + runBestEffortCallback({ + label: "live edit diff agent event", + log: ctx.log, + callback: () => ctx.params.onAgentEvent?.({ stream: "tool", data }), + }); + } + const eventAssistantMessage = + assistantRecord?.partial && typeof assistantRecord.partial === "object" + ? (assistantRecord.partial as AssistantMessage) + : msg; + const isResponsesTextEvent = + isResponsesApiAssistantMessage(eventAssistantMessage) && + (evtType === "text_start" || evtType === "text_delta" || evtType === "text_end"); + const suppressVisibleAssistantOutput = shouldSuppressAssistantVisibleOutput(msg); + if (suppressVisibleAssistantOutput && !isResponsesTextEvent) { + const commentaryText = coerceChatContentText(extractAssistantCommentaryText(msg)); + if (commentaryText) { + appendRawStream({ + ts: Date.now(), + event: "assistant_text_stream", + runId: ctx.params.runId, + sessionId: (ctx.params.session as { id?: string }).id, + evtType: "commentary_update", + delta: "", + content: commentaryText, + }); + ctx.emitAssistantStreamData( + buildAssistantStreamData({ text: commentaryText, replace: true, phase: "commentary" }), + ); + } + return; + } + const suppressDeterministicApprovalOutput = shouldSuppressDeterministicApprovalOutput(ctx.state); + const suppressMessageToolOnlySourceReplyOutput = hasMessageToolOnlySourceDelivery(ctx); + + const assistantPhase = resolveAssistantMessagePhase(msg); + + if (evtType === "text_end" || evtType === "done" || evtType === "error") { + capturePendingAssistantUsage(ctx, evt); + if (evtType === "done" || evtType === "error") { + ctx.commitAssistantUsage(); + } + } + + if (evtType === "thinking_start" || evtType === "thinking_delta" || evtType === "thinking_end") { + if ( + !suppressMessageToolOnlySourceReplyOutput && + (evtType === "thinking_start" || evtType === "thinking_delta") + ) { + openReasoningStream(ctx); + } + const thinkingDelta = typeof assistantRecord?.delta === "string" ? assistantRecord.delta : ""; + const thinkingContent = + typeof assistantRecord?.content === "string" ? assistantRecord.content : ""; + appendRawStream({ + ts: Date.now(), + event: "assistant_thinking_stream", + runId: ctx.params.runId, + sessionId: (ctx.params.session as { id?: string }).id, + evtType, + delta: thinkingDelta, + content: thinkingContent, + }); + // Emit-always: emitReasoningStream always reaches the bus/archive; the + // streamReasoning rendering hook and message_tool_only source suppression + // are gated downstream (dispatch wrapProgressCallback, #92738), so emission + // here stays unconditional. + // Prefer full partial-message thinking when available; fall back to event payloads. + const partialThinking = extractAssistantThinking(msg); + ctx.emitReasoningStream(partialThinking || thinkingContent || thinkingDelta); + if (evtType === "thinking_end" && !suppressMessageToolOnlySourceReplyOutput) { + // Mirror the open gate above: when message-tool-only delivery has made the + // reasoning lane private, do not force-open it just to close it — that + // would fire the lane's end hook (onReasoningEnd) for a lane that never + // rendered, leaking the boundary signal. + if (!ctx.state.reasoningStreamOpen) { + openReasoningStream(ctx); + } + emitReasoningEnd(ctx); + } + return; + } + + if (evtType !== "text_delta" && evtType !== "text_start" && evtType !== "text_end") { + return; + } + + const delta = typeof assistantRecord?.delta === "string" ? assistantRecord.delta : ""; + const content = typeof assistantRecord?.content === "string" ? assistantRecord.content : ""; + + appendRawStream({ + ts: Date.now(), + event: "assistant_text_stream", + runId: ctx.params.runId, + sessionId: (ctx.params.session as { id?: string }).id, + evtType, + delta, + content, + }); + + const chunk = resolveAssistantTextChunk({ + evtType, + delta, + content, + accumulatedText: ctx.state.deltaBuffer, + }); + + const partialAssistant = eventAssistantMessage; + const streamContentIndex = resolveAssistantStreamContentIndex(assistantRecord?.contentIndex); + const streamItemId = resolveAssistantStreamItemId({ + contentIndex: streamContentIndex, + message: partialAssistant, + }); + const streamAssistant = scopeAssistantMessageToStreamBlock( + partialAssistant, + streamContentIndex, + streamItemId, + ); + const deliveryPhase = resolveAssistantMessagePhase(streamAssistant); + const isPhasePendingResponsesTextItem = + evtType !== "text_end" && + !deliveryPhase && + Boolean(streamItemId) && + isResponsesApiAssistantMessage(partialAssistant); + // These transports resolve commentary only at the tool boundary. Withhold + // early unphased deltas from durable block replies until that decision exists. + const isPhasePendingAnthropicText = + evtType !== "text_end" && !deliveryPhase && isAnthropicAssistantMessage(partialAssistant); + const isPhasePendingCompletionsText = + !deliveryPhase && isOpenAiCompletionsAssistantMessage(partialAssistant); + const hasResponsesContentIndex = + streamContentIndex !== undefined && isResponsesApiAssistantMessage(partialAssistant); + let streamItemChanged = false; + let deliveryItemId = streamItemId; + if ( + (deliveryPhase || isPhasePendingResponsesTextItem || hasResponsesContentIndex) && + (streamContentIndex !== undefined || streamItemId) + ) { + const previousStreamContentIndex = ctx.state.lastAssistantStreamContentIndex; + const previousStreamItemId = ctx.state.lastAssistantStreamItemId; + const contentIndexChanged = + previousStreamContentIndex !== undefined && + streamContentIndex !== undefined && + previousStreamContentIndex !== streamContentIndex; + const itemIdChangedWithoutIndexes = + (previousStreamContentIndex === undefined || streamContentIndex === undefined) && + Boolean(previousStreamItemId && streamItemId && previousStreamItemId !== streamItemId); + if (contentIndexChanged || itemIdChangedWithoutIndexes) { + streamItemChanged = true; + void ctx.flushBlockReplyBuffer({ assistantMessageIndex: ctx.state.assistantMessageIndex }); + ctx.resetAssistantMessageState(ctx.state.assistantTexts.length); + emitAssistantMessageStart(ctx); + } else if ( + previousStreamContentIndex !== undefined && + streamContentIndex === previousStreamContentIndex && + previousStreamItemId + ) { + // Snapshot-extension items can rotate provider ids while retaining one logical block. + // Keep the original live key so downstream commentary accumulators do not split it. + deliveryItemId = previousStreamItemId; + } + ctx.state.lastAssistantStreamContentIndex = streamContentIndex; + ctx.state.lastAssistantStreamItemId = deliveryItemId; + } + // Responses text_start snapshots may already contain text replayed by the first delta. + // Keep starts lifecycle-only so commentary and final-answer lanes consume each byte once. + if (evtType === "text_start" && isResponsesApiAssistantMessage(partialAssistant)) { + return; + } + if (deliveryPhase === "commentary") { + const isResponsesCommentary = isResponsesApiAssistantMessage(partialAssistant); + const hadResponsesCommentaryText = isResponsesCommentary && Boolean(ctx.state.deltaBuffer); + if (isResponsesCommentary && chunk) { + // Keep cumulative end events monotonic without feeding commentary into reply buffers. + ctx.state.deltaBuffer += chunk; + } + const commentaryText = + !chunk && (!isResponsesCommentary || !hadResponsesCommentaryText) + ? coerceChatContentText(extractAssistantCommentaryText(streamAssistant)) + : undefined; + const commentaryData = chunk + ? buildAssistantStreamData({ delta: chunk, phase: "commentary", itemId: deliveryItemId }) + : commentaryText + ? buildAssistantStreamData({ + text: commentaryText, + replace: true, + phase: "commentary", + itemId: deliveryItemId, + }) + : undefined; + if (commentaryData) { + ctx.emitAssistantStreamData(commentaryData); + } + return; + } + if (isPhasePendingResponsesTextItem) { + return; + } + // Subagents have no live consumer; their final result is delivered from + // message_end. Keep accumulating deltaBuffer, but skip per-chunk visible-text + // parsing so long parallel subagent streams do not monopolize the event loop. + const skipLiveStream = ctx.params.suppressLiveStreamOutput === true; + const shouldUsePhaseAwareBlockReply = Boolean(deliveryPhase); + + if (chunk) { + ctx.state.deltaBuffer += chunk; + if (!skipLiveStream && !shouldUsePhaseAwareBlockReply) { + if (!isPhasePendingAnthropicText && !isPhasePendingCompletionsText) { + appendBlockReplyChunk(ctx, chunk); + } + } + } + + if (skipLiveStream) { + return; + } + + // Handle partial tags: stream whatever reasoning is visible so far. + // Emit-always: emitReasoningStream reaches the bus/archive; rendering + + // message_tool_only suppression are gated downstream (#92738). + ctx.emitReasoningStream( + extractThinkingFromTaggedStream(ctx.state.deltaBuffer, ctx.state.thinkingTagStream), + ); + const wasThinking = ctx.state.partialBlockState.thinking; + let visibleDelta = ""; + // A text_start partial may already contain text that the following text_delta replays. + // Use starts only for lifecycle boundaries; consume their text from delta/end events. + const shouldReadScopedPartialText = + streamItemChanged || (shouldUsePhaseAwareBlockReply && (evtType === "text_end" || !chunk)); + let next = shouldReadScopedPartialText + ? coerceChatContentText(extractAssistantVisibleText(streamAssistant)).trim() + : ""; + let nextRawStreamText = next; + let shouldPersistRawStreamText = false; + if (shouldUsePhaseAwareBlockReply && !next && deliveryPhase === "final_answer" && chunk) { + visibleDelta = ctx.stripBlockTags(chunk, ctx.state.partialBlockState, { + final: evtType === "text_end", + }); + const streamVisibleText = resolveStreamVisibleText({ + previousRawText: ctx.state.lastStreamedAssistant ?? "", + visibleDelta, + }); + const previousVisibleText = sanitizeAssistantVisibleStreamText( + ctx.state.lastStreamedAssistant ?? "", + ).trim(); + next = sanitizeAssistantVisibleStreamText(streamVisibleText.rawText).trim(); + visibleDelta = resolveTextAppendDelta(previousVisibleText, next); + nextRawStreamText = streamVisibleText.rawText; + shouldPersistRawStreamText = true; + } else if (!next && deliveryPhase !== "final_answer") { + const pendingTagFragment = ctx.state.partialBlockState.pendingTagFragment; + const shouldRecomputeFullStream = Boolean(pendingTagFragment) || REASONING_TAG_RE.test(chunk); + if (shouldRecomputeFullStream) { + const recomputeState: EmbeddedAgentSubscribeState["partialBlockState"] = { + thinking: false, + final: false, + inlineCode: createInlineCodeState(), + }; + const recomputedRawText = ctx.stripBlockTags(ctx.state.deltaBuffer, recomputeState, { + final: evtType === "text_end", + }); + const previousRawText = ctx.state.lastStreamedAssistant ?? ""; + const isFullStreamReplacement = !recomputedRawText.startsWith(previousRawText); + next = recomputedRawText.trim(); + visibleDelta = isFullStreamReplacement + ? recomputedRawText + : recomputedRawText.slice(previousRawText.length); + nextRawStreamText = recomputedRawText; + copyPartialBlockState(ctx.state.partialBlockState, recomputeState); + } else { + visibleDelta = + chunk || evtType === "text_end" + ? ctx.stripBlockTags(chunk, ctx.state.partialBlockState, { + final: evtType === "text_end", + }) + : ""; + if (ctx.state.partialBlockState.pendingTagFragment) { + visibleDelta = ""; + next = ctx.state.lastStreamedAssistantCleaned ?? ""; + nextRawStreamText = ctx.state.lastStreamedAssistant ?? ""; + } else { + const streamVisibleText = resolveStreamVisibleText({ + previousRawText: ctx.state.lastStreamedAssistant ?? "", + visibleDelta, + }); + next = streamVisibleText.visibleText; + nextRawStreamText = streamVisibleText.rawText; + } + } + } else if (next && (chunk || evtType === "text_end")) { + visibleDelta = ctx.stripBlockTags(chunk, ctx.state.partialBlockState, { + final: evtType === "text_end", + }); + } + if (next) { + if ( + !suppressMessageToolOnlySourceReplyOutput && + !wasThinking && + ctx.state.partialBlockState.thinking + ) { + openReasoningStream(ctx); + } + // Detect when thinking block ends ( tag processed) + if ( + !suppressMessageToolOnlySourceReplyOutput && + wasThinking && + !ctx.state.partialBlockState.thinking + ) { + emitReasoningEnd(ctx); + } + const parsedDelta = visibleDelta ? ctx.consumePartialReplyDirectives(visibleDelta) : null; + const finalParsedDelta = + evtType === "text_end" ? ctx.consumePartialReplyDirectives("", { final: true }) : null; + const parsedStreamDirectives = mergeReplyDirectiveResults(parsedDelta, finalParsedDelta); + if (shouldUsePhaseAwareBlockReply) { + recordPendingAssistantReplyDirectives(ctx.state, parsedStreamDirectives); + } + const previousCleaned = ctx.state.lastStreamedAssistantCleaned ?? ""; + const cleanedText = resolveStreamingReplyText({ + evtType, + next, + previousRawText: ctx.state.lastStreamedAssistant ?? "", + previousCleaned, + visibleDelta, + parsedStreamDirectives, + shouldUsePhaseAwareBlockReply, + }); + const { mediaUrls, hasMedia } = resolveSendableOutboundReplyParts(parsedStreamDirectives ?? {}); + const hasAudio = Boolean(parsedStreamDirectives?.audioAsVoice); + + let shouldEmit; + let deltaText = ""; + let replace = false; + if (!hasAssistantVisibleReply({ text: cleanedText, mediaUrls, audioAsVoice: hasAudio })) { + shouldEmit = false; + } else { + replace = Boolean(previousCleaned && !cleanedText.startsWith(previousCleaned)); + deltaText = replace ? "" : cleanedText.slice(previousCleaned.length); + shouldEmit = replace + ? cleanedText !== previousCleaned || hasMedia || hasAudio + : Boolean(deltaText || hasMedia || hasAudio); + } + + if (shouldUsePhaseAwareBlockReply) { + if (replace) { + ctx.state.blockBuffer = ""; + ctx.blockChunker?.reset(); + } + const blockReplyChunk = replace ? cleanedText : deltaText; + if (blockReplyChunk) { + appendBlockReplyChunk(ctx, blockReplyChunk); + } + + if (evtType === "text_end" && !ctx.state.lastBlockReplyText && cleanedText) { + replaceBlockReplyBuffer(ctx, cleanedText); + } + } else if (streamItemChanged && !chunk) { + // An unphased equal/shrinking Responses item can end without a delta. + // Rebuild its block buffer from the scoped snapshot after the boundary reset. + appendBlockReplyChunk(ctx, cleanedText); + } + + ctx.state.lastStreamedAssistant = nextRawStreamText; + ctx.state.lastStreamedAssistantCleaned = cleanedText; + + if ( + ctx.params.silentExpected || + suppressDeterministicApprovalOutput || + suppressMessageToolOnlySourceReplyOutput + ) { + shouldEmit = false; + } + + if (shouldEmit) { + const currentSourcePartial = + ctx.params.sourceReplyDeliveryMode !== "message_tool_only" + ? resolveCurrentSourceMessagingToolPartial(ctx.state, { + evtType, + text: cleanedText, + visibleDelta, + }) + : { hold: false, text: cleanedText }; + const releaseHeldSnapshot = currentSourcePartial.text !== cleanedText; + const data = buildAssistantStreamData({ + text: currentSourcePartial.text, + delta: releaseHeldSnapshot ? currentSourcePartial.text : deltaText, + replace: releaseHeldSnapshot || replace, + mediaUrls, + phase: deliveryPhase ?? assistantPhase, + }); + ctx.emitAssistantStreamData(data, { emitPartialReply: !currentSourcePartial.hold }); + ctx.state.emittedAssistantUpdate = true; + } + } else if (shouldPersistRawStreamText) { + ctx.state.lastStreamedAssistant = nextRawStreamText; + } + + if ( + !ctx.params.silentExpected && + !suppressDeterministicApprovalOutput && + !suppressMessageToolOnlySourceReplyOutput && + ctx.params.onBlockReply && + ctx.blockChunking && + ctx.state.blockReplyBreak === "text_end" + ) { + ctx.blockChunker?.drain({ force: false, emit: ctx.emitBlockChunk }); + } + + if ( + !ctx.params.silentExpected && + !suppressDeterministicApprovalOutput && + !suppressMessageToolOnlySourceReplyOutput && + evtType === "text_end" && + ctx.state.blockReplyBreak === "text_end" + ) { + const assistantMessageIndex = ctx.state.assistantMessageIndex; + void Promise.resolve() + .then(() => ctx.flushBlockReplyBuffer({ assistantMessageIndex, final: true })) + .catch((err: unknown) => { + ctx.log.debug(`text_end block reply flush failed: ${String(err)}`); + }); + } +} + +if (process.env.VITEST || process.env.NODE_ENV === "test") { + (globalThis as Record)[ + Symbol.for("openclaw.embeddedSubscribeMessagesTestApi") + ] = { + buildAssistantStreamData, + recordPendingAssistantReplyDirectives, + resolveCurrentSourceMessagingToolPartial, + }; +} diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.completion.ts b/src/agents/embedded-agent-subscribe.handlers.tools.completion.ts index ca8117645e7f..3bdacb0452d1 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.completion.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.completion.ts @@ -26,6 +26,11 @@ import { readMessageToolSourceReplyText, resolveMessageToolSourceReplyFinal, } from "./embedded-agent-message-tool-source-reply.js"; +import { + extractMessagingToolSend, + extractMessagingToolSendResult, + extractMessagingToolSourceReplyPayload, +} from "./embedded-agent-messaging-extraction.js"; import { isMessagingTool, isMessagingToolSendAction, @@ -73,16 +78,14 @@ import type { ToolHandlerContext } from "./embedded-agent-subscribe.handlers.typ import { collectMessagingMediaUrlsFromRecord, collectMessagingMediaUrlsFromToolResult, +} from "./embedded-agent-tool-media.js"; +import { capLiveExecResult, - extractMessagingToolSourceReplyPayload, extractToolErrorCode, - extractMessagingToolSend, - extractMessagingToolSendResult, extractToolErrorMessage, - isToolResultError, isToolResultTimedOut, sanitizeToolResult, -} from "./embedded-agent-subscribe.tools.js"; +} from "./embedded-agent-tool-results.js"; import { parseExecApprovalResultText } from "./exec-approval-result.js"; import { readMcpConnectAction } from "./mcp-connect-action.js"; import { readMcpAppChannelView } from "./mcp-ui-resource.js"; @@ -93,6 +96,7 @@ import { } from "./tool-error-summary.js"; import { resolveFileMutationToolName } from "./tool-mutation-names.js"; import { normalizeToolPolicyName } from "./tool-policy.js"; +import { isToolResultError } from "./tool-result-error.js"; import { cancelAskUserPromptDelivery } from "./tools/ask-user-tool.js"; import { isAutomationsToolName } from "./tools/automations-tool-name.js"; diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.progress.ts b/src/agents/embedded-agent-subscribe.handlers.tools.progress.ts index b473aec2b2bb..c41b787babfd 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.progress.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.progress.ts @@ -24,7 +24,7 @@ import { capLiveExecResult, sanitizeToolResult, truncateLiveExecOutput, -} from "./embedded-agent-subscribe.tools.js"; +} from "./embedded-agent-tool-results.js"; import type { AgentEvent } from "./runtime/index.js"; import { normalizeToolPolicyName } from "./tool-policy.js"; diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.results.ts b/src/agents/embedded-agent-subscribe.handlers.tools.results.ts index 8e831177a00d..513e926f9e89 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.results.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.results.ts @@ -21,10 +21,9 @@ import type { ExecToolDetails } from "./bash-tools.exec-types.js"; import type { ToolHandlerContext } from "./embedded-agent-subscribe.handlers.types.js"; import { extractToolResultMediaArtifact, - extractToolResultText, filterToolResultMediaUrls, - truncateLiveExecOutput, -} from "./embedded-agent-subscribe.tools.js"; +} from "./embedded-agent-tool-media.js"; +import { extractToolResultText, truncateLiveExecOutput } from "./embedded-agent-tool-results.js"; import type { ProcessTerminalDiagnostic } from "./tool-error-summary.js"; import { readToolResultDetails } from "./tool-result-error.js"; import { createToolTerminalObserver } from "./tool-terminal-outcome.js"; diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.start.ts b/src/agents/embedded-agent-subscribe.handlers.tools.start.ts index cdfa34867928..4a7c1f100127 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.start.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.start.ts @@ -9,6 +9,7 @@ import { emitAgentActivityEvent, type AgentItemEventData } from "../infra/agent- import { emitAgentEvent } from "../infra/agent-events.js"; import { REQUIRED_PARAM_GROUPS, type RequiredParamGroup } from "./agent-tools.params.js"; import { sanitizeForConsole } from "./console-sanitize.js"; +import { extractMessagingToolSend } from "./embedded-agent-messaging-extraction.js"; import { isMessagingTool, isMessagingToolSendAction, @@ -23,11 +24,8 @@ import type { ToolCallSummary, ToolHandlerContext, } from "./embedded-agent-subscribe.handlers.types.js"; -import { - collectMessagingMediaUrlsFromRecord, - extractMessagingToolSend, - sanitizeToolArgs, -} from "./embedded-agent-subscribe.tools.js"; +import { collectMessagingMediaUrlsFromRecord } from "./embedded-agent-tool-media.js"; +import { sanitizeToolArgs } from "./embedded-agent-tool-results.js"; import { buildAgentHarnessQuestionPromptPayload } from "./harness/user-input-bridge.js"; import type { AgentEvent } from "./runtime/index.js"; import { inferToolMetaFromArgsCore, isCommandBearingToolCall } from "./tool-display.js"; diff --git a/src/agents/embedded-agent-subscribe.handlers.ts b/src/agents/embedded-agent-subscribe.handlers.ts index efdbc343dbc8..7b94bbb2c1b8 100644 --- a/src/agents/embedded-agent-subscribe.handlers.ts +++ b/src/agents/embedded-agent-subscribe.handlers.ts @@ -10,12 +10,12 @@ import { } from "./embedded-agent-subscribe.handlers.lifecycle.js"; import { capturePendingAssistantUsage, - handleMessageEnd, handleMessageStart, - handleMessageUpdate, preservePendingAssistantUsage, resetPendingAssistantUsage, -} from "./embedded-agent-subscribe.handlers.messages.js"; + handleMessageEnd, +} from "./embedded-agent-subscribe.handlers.messages.lifecycle.js"; +import { handleMessageUpdate } from "./embedded-agent-subscribe.handlers.messages.update.js"; import { handleToolExecutionEnd, handleToolExecutionStart, diff --git a/src/agents/embedded-agent-subscribe.tools.extract.test.ts b/src/agents/embedded-agent-subscribe.tools.extract.test.ts index e0f95f5ace14..7f06d6a871bd 100644 --- a/src/agents/embedded-agent-subscribe.tools.extract.test.ts +++ b/src/agents/embedded-agent-subscribe.tools.extract.test.ts @@ -3,7 +3,7 @@ import { beforeEach, describe, expect, it } from "vitest"; import { setActivePluginRegistry } from "../plugins/runtime.js"; import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js"; -import { extractMessagingToolSend } from "./embedded-agent-subscribe.tools.js"; +import { extractMessagingToolSend } from "./embedded-agent-messaging-extraction.js"; function normalizeTelegramMessagingTargetForTest(raw: string): string | undefined { // Test normalizer mirrors channel plugins that canonicalize human targets diff --git a/src/agents/embedded-agent-subscribe.tools.media.test.ts b/src/agents/embedded-agent-subscribe.tools.media.test.ts index a52b73d7b357..69cd0b428162 100644 --- a/src/agents/embedded-agent-subscribe.tools.media.test.ts +++ b/src/agents/embedded-agent-subscribe.tools.media.test.ts @@ -1,11 +1,11 @@ // Tool media extraction tests cover structured media payloads, image fallbacks, // trust decisions, and filtering of local/remote media URLs. import { describe, expect, it } from "vitest"; +import { isToolResultMediaTrusted } from "./embedded-agent-subscribe.tools.test-support.js"; import { extractToolResultMediaArtifact, filterToolResultMediaUrls, -} from "./embedded-agent-subscribe.tools.js"; -import { isToolResultMediaTrusted } from "./embedded-agent-subscribe.tools.test-support.js"; +} from "./embedded-agent-tool-media.js"; describe("extractToolResultMediaArtifact", () => { it("returns undefined for null/undefined", () => { diff --git a/src/agents/embedded-agent-subscribe.tools.reconcile-thread.test.ts b/src/agents/embedded-agent-subscribe.tools.reconcile-thread.test.ts index 94f03e6a6f35..a71d3277fff4 100644 --- a/src/agents/embedded-agent-subscribe.tools.reconcile-thread.test.ts +++ b/src/agents/embedded-agent-subscribe.tools.reconcile-thread.test.ts @@ -5,7 +5,7 @@ import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/c import { extractMessagingToolSend, extractMessagingToolSendResult, -} from "./embedded-agent-subscribe.tools.js"; +} from "./embedded-agent-messaging-extraction.js"; const PARTIAL_RESULT_PROVIDER = "partialthreadprovider"; diff --git a/src/agents/embedded-agent-subscribe.tools.test-support.ts b/src/agents/embedded-agent-subscribe.tools.test-support.ts index 6e12209eca52..befc19c5795d 100644 --- a/src/agents/embedded-agent-subscribe.tools.test-support.ts +++ b/src/agents/embedded-agent-subscribe.tools.test-support.ts @@ -1,4 +1,4 @@ -import "./embedded-agent-subscribe.tools.js"; +import "./embedded-agent-tool-media.js"; type EmbeddedSubscribeToolsTestApi = { isToolResultMediaTrusted( diff --git a/src/agents/embedded-agent-subscribe.tools.test.ts b/src/agents/embedded-agent-subscribe.tools.test.ts index cd13e4b9395b..f4a64268e32a 100644 --- a/src/agents/embedded-agent-subscribe.tools.test.ts +++ b/src/agents/embedded-agent-subscribe.tools.test.ts @@ -9,10 +9,10 @@ import { extractToolResultText, extractToolErrorCode, extractToolErrorMessage, - isToolResultError, sanitizeToolArgs, sanitizeToolResult, -} from "./embedded-agent-subscribe.tools.js"; +} from "./embedded-agent-tool-results.js"; +import { isToolResultError } from "./tool-result-error.js"; afterEach(() => { // Logging config spies are global module state; restore after every sanitizer diff --git a/src/agents/embedded-agent-subscribe.tools.ts b/src/agents/embedded-agent-subscribe.tools.ts deleted file mode 100644 index 19515e6f2c08..000000000000 --- a/src/agents/embedded-agent-subscribe.tools.ts +++ /dev/null @@ -1,1180 +0,0 @@ -/** Sanitizes, extracts, and classifies embedded-agent tool execution results. */ -import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64"; -import { asOptionalRecord as readRecord } from "@openclaw/normalization-core/record-coerce"; -import { - normalizeOptionalLowercaseString, - normalizeOptionalString, - normalizeOptionalStringifiedId, - readStringValue, -} from "@openclaw/normalization-core/string-coerce"; -import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; -import { getChannelPlugin, normalizeChannelId } from "../channels/plugins/index.js"; -import type { ChannelMessageActionName } from "../channels/plugins/types.public.js"; -import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { normalizeTargetForProvider } from "../infra/outbound/target-normalization.js"; -import { - normalizeLegacyInteractiveReply, - normalizeMessagePresentation, -} from "../interactive/payload.js"; -import { - redactSecrets, - redactSensitiveFieldValue, - redactToolPayloadText, -} from "../logging/redact.js"; -import { truncateUtf16Safe } from "../utils.js"; -import { collectTextContentBlocks } from "./content-blocks.js"; -import { isMessagingToolTargetEvidenceAction } from "./embedded-agent-messaging.js"; -import type { - MessagingToolSend, - MessagingToolSourceReplyPayload, -} from "./embedded-agent-messaging.types.js"; -import { normalizeToolPolicyName } from "./tool-policy.js"; -import { - isToolResultError, - readToolResultDetails, - readToolResultStatus, -} from "./tool-result-error.js"; -import { AUTOMATIONS_TOOL_NAME } from "./tools/automations-tool-name.js"; - -export { isToolResultError }; - -const TOOL_RESULT_MAX_CHARS = 8000; -const TOOL_ERROR_MAX_CHARS = 400; -const LIVE_EXEC_OUTPUT_MAX_CHARS = 8000; -const TOOL_DENIAL_ERROR_CODES = ["SYSTEM_RUN_DENIED", "INVALID_REQUEST"] as const; -const OPAQUE_STRUCTURED_RESULT_FIELDS = new Set(["encrypted_content", "encrypted_stdout"]); -const SENSITIVE_STRUCTURED_HEADER_FIELDS = new Set([ - "authorization", - "proxy-authorization", - "cookie", - "set-cookie", - "x-api-key", - "x-auth-token", -]); - -function truncateToolText(text: string): string { - if (text.length <= TOOL_RESULT_MAX_CHARS) { - return text; - } - return `${truncateUtf16Safe(text, TOOL_RESULT_MAX_CHARS)}\n…(truncated)…`; -} - -export function truncateLiveExecOutput(text: string): string { - if (text.length <= LIVE_EXEC_OUTPUT_MAX_CHARS) { - return text; - } - return `${truncateUtf16Safe(text, LIVE_EXEC_OUTPUT_MAX_CHARS)}\n...(live output truncated)...`; -} - -export function capLiveExecResult(result: unknown): unknown { - const details = readToolResultDetails(result); - if (!details || typeof details.status !== "string" || typeof details.aggregated !== "string") { - return result; - } - const aggregated = truncateLiveExecOutput(details.aggregated); - if (aggregated === details.aggregated) { - return result; - } - if (!result || typeof result !== "object" || Array.isArray(result)) { - return result; - } - return { - ...(result as Record), - details: { - ...details, - aggregated, - }, - }; -} - -function normalizeToolErrorText(text: string): string | undefined { - const trimmed = text.trim(); - if (!trimmed) { - return undefined; - } - const firstLine = trimmed.split(/\r?\n/)[0]?.trim() ?? ""; - if (!firstLine) { - return undefined; - } - return firstLine.length > TOOL_ERROR_MAX_CHARS - ? `${truncateUtf16Safe(firstLine, TOOL_ERROR_MAX_CHARS)}…` - : firstLine; -} - -function isErrorLikeStatus(status: string): boolean { - const normalized = normalizeOptionalLowercaseString(status); - if (!normalized) { - return false; - } - if ( - normalized === "0" || - normalized === "ok" || - normalized === "success" || - normalized === "completed" || - normalized === "running" - ) { - return false; - } - return /error|fail|timeout|timed[_\s-]?out|denied|cancel|invalid|forbidden/.test(normalized); -} - -function readErrorCandidate(value: unknown): string | undefined { - if (typeof value === "string") { - return normalizeToolErrorText(value); - } - if (!value || typeof value !== "object") { - return undefined; - } - const record = value as Record; - if (typeof record.message === "string") { - return normalizeToolErrorText(record.message); - } - if (typeof record.error === "string") { - return normalizeToolErrorText(record.error); - } - return undefined; -} - -function extractErrorField(value: unknown): string | undefined { - if (!value || typeof value !== "object") { - return undefined; - } - const record = value as Record; - const direct = extractDirectErrorField(record); - if (direct) { - return direct; - } - const status = normalizeOptionalString(record.status) ?? ""; - if (!status || !isErrorLikeStatus(status)) { - return undefined; - } - return normalizeToolErrorText(status); -} - -function extractDirectErrorField(value: unknown): string | undefined { - if (!value || typeof value !== "object") { - return undefined; - } - const record = value as Record; - return ( - readErrorCandidate(record.error) ?? - readErrorCandidate(record.message) ?? - readErrorCandidate(record.reason) - ); -} - -function readErrorCodeField(value: unknown): string | undefined { - return typeof value === "string" ? normalizeOptionalString(value) : undefined; -} - -function readDenialErrorCodeFromMessage(value: unknown): string | undefined { - const message = typeof value === "string" ? normalizeOptionalString(value) : undefined; - if (!message) { - return undefined; - } - for (const code of TOOL_DENIAL_ERROR_CODES) { - if (message === code || message.startsWith(`${code}:`)) { - return code; - } - } - return undefined; -} - -function readNestedErrorCodeField(value: unknown): string | undefined { - if (!value || typeof value !== "object") { - return undefined; - } - const record = value as Record; - return ( - readDenialErrorCodeFromMessage(record.message) ?? - readDenialErrorCodeFromMessage(record.error) ?? - readErrorCodeField(record.code) ?? - readErrorCodeField(record.gatewayCode) - ); -} - -function extractDirectErrorCodeField(value: unknown): string | undefined { - if (!value || typeof value !== "object") { - return undefined; - } - const record = value as Record; - return ( - readNestedErrorCodeField(record.error) ?? - readNestedErrorCodeField(record.nodeError) ?? - readErrorCodeField(record.code) ?? - readErrorCodeField(record.gatewayCode) - ); -} - -export function buildToolLifecycleErrorResult(error: unknown): { - details: Record; -} { - const errorRecord = readRecord(error); - const rawDetails = readRecord(errorRecord?.details); - const nodeError = readRecord(rawDetails?.nodeError); - const gatewayCode = - readErrorCodeField(errorRecord?.gatewayCode) ?? readErrorCodeField(errorRecord?.code); - const message = error instanceof Error ? error.message : String(error); - return { - details: { - status: "error", - error: message, - ...(gatewayCode ? { gatewayCode } : {}), - ...(nodeError ? { nodeError } : {}), - }, - }; -} - -function extractAggregatedErrorField(value: unknown): string | undefined { - if (!value || typeof value !== "object") { - return undefined; - } - const record = value as Record; - return readErrorCandidate(record.aggregated); -} - -function redactStringsDeep(value: unknown, seen = new WeakSet()): unknown { - if (typeof value === "string") { - return redactToolPayloadText(value); - } - if (Array.isArray(value)) { - if (seen.has(value)) { - return "[Circular]"; - } - seen.add(value); - return value.map((item) => redactStringsDeep(item, seen)); - } - if (value && typeof value === "object") { - if (seen.has(value)) { - return "[Circular]"; - } - seen.add(value); - const out: Record = {}; - for (const [key, child] of Object.entries(value as Record)) { - out[key] = - typeof child === "string" - ? redactSensitiveFieldValue(key, child) - : redactStringsDeep(child, seen); - } - return out; - } - return value; -} - -export function sanitizeToolArgs(args: unknown): unknown { - return redactStringsDeep(args); -} - -export function sanitizeToolResult(result: unknown): unknown { - if (typeof result === "string") { - return redactToolPayloadText(result); - } - if (Array.isArray(result)) { - return redactSecrets(result); - } - if (!result || typeof result !== "object") { - return result; - } - const record = result as Record; - // Strip image data first so the deep redaction pass doesn't waste work - // scanning base64 payloads (and so we capture the original byte counts). - const preCleaned: Record = { ...record }; - const originalContent = Array.isArray(record.content) ? record.content : null; - if (originalContent) { - preCleaned.content = originalContent.map((item) => { - if (!item || typeof item !== "object") { - return item; - } - const entry = item as Record; - if (readStringValue(entry.type) === "image") { - const data = readStringValue(entry.data); - const existingBytes = typeof entry.bytes === "number" ? entry.bytes : undefined; - const bytes = data === undefined ? existingBytes : estimateBase64DecodedBytes(data); - const cleaned = { ...entry }; - delete cleaned.data; - return Object.assign({}, cleaned, { bytes, omitted: true }); - } - return entry; - }); - } - // Deep-redact the entire result so any top-level or nested string is - // protected, not just `details` and text content blocks. - const baseline = redactSecrets(preCleaned); - const out: Record = { ...baseline }; - const content = Array.isArray(baseline.content) ? baseline.content : null; - if (content) { - out.content = content.map((item) => { - if (!item || typeof item !== "object") { - return item; - } - const entry = item as Record; - if (readStringValue(entry.type) === "text" && typeof entry.text === "string") { - return Object.assign({}, entry, { text: truncateToolText(entry.text) }); - } - return entry; - }); - } - return out; -} - -const INLINE_DATA_URI_VALUE_PATTERN = - /^data:(?:[a-z][a-z0-9.+-]*\/[a-z0-9.+-]+)?(?:;[a-z0-9.+-]+(?:=[^,;"'\s]+)?)*,/i; - -function redactInlineDataUriValue(value: string): string { - const trimmed = value.trimStart(); - if (!INLINE_DATA_URI_VALUE_PATTERN.test(trimmed)) { - return value; - } - return `[inline data URI: ${value.length} chars]`; -} - -function carriesBinaryData(record: Record): boolean { - const type = normalizeOptionalLowercaseString(record.type); - if (type === "audio" || type === "image" || type === "base64") { - return true; - } - const mediaType = normalizeOptionalLowercaseString(record.media_type ?? record.mimeType); - return ( - mediaType?.startsWith("image/") === true || - mediaType?.startsWith("audio/") === true || - mediaType?.startsWith("video/") === true || - mediaType === "application/pdf" - ); -} - -function sanitizeStructuredToolResultValue( - value: unknown, - key = "", - parentCarriesBinaryData = false, - seen = new WeakSet(), -): unknown { - if (typeof value === "string") { - if (SENSITIVE_STRUCTURED_HEADER_FIELDS.has(key.toLowerCase())) { - return "***"; - } - if (key === "blob" || (key === "data" && parentCarriesBinaryData)) { - return `[binary omitted: ${value.length} chars]`; - } - // Claude CLI result blocks carry replay-only ciphertext that is not useful display text. - if (OPAQUE_STRUCTURED_RESULT_FIELDS.has(key)) { - return `[opaque data omitted: ${value.length} chars]`; - } - return truncateToolText(redactInlineDataUriValue(redactSensitiveFieldValue(key, value))); - } - if (typeof value === "bigint") { - return value.toString(); - } - if (!value || typeof value !== "object") { - return value; - } - if (seen.has(value)) { - return "[Circular]"; - } - seen.add(value); - if (Array.isArray(value)) { - // Keep the owning key so arrays of credentials inherit the same redaction policy. - return value.map((item) => - sanitizeStructuredToolResultValue(item, key, parentCarriesBinaryData, seen), - ); - } - const record = value as Record; - const hasBinaryData = carriesBinaryData(record); - return Object.fromEntries( - Object.entries(record).map(([childKey, child]) => [ - childKey, - sanitizeStructuredToolResultValue(child, childKey, hasBinaryData, seen), - ]), - ); -} - -function stringifyStructuredToolResultContent(block: unknown): string | undefined { - if (!block || typeof block !== "object") { - return undefined; - } - const record = block as Record; - const type = readStringValue(record.type); - if (type === "text" || type === "image" || type === "image_url" || type === "audio") { - return undefined; - } - try { - const serialized = JSON.stringify(sanitizeStructuredToolResultValue(record)); - const redacted = serialized ? redactToolPayloadText(serialized) : serialized; - return redacted && redacted !== "{}" ? redacted : undefined; - } catch { - return undefined; - } -} - -function resolveToolResultContentBlocks(result: object): unknown[] { - if (Array.isArray(result)) { - return result; - } - const record = result as Record; - // Typed provider blocks own their `content`; only untyped tool-result envelopes unwrap it. - if (readStringValue(record.type)) { - return [record]; - } - if (Array.isArray(record.content)) { - return record.content; - } - if (record.content && typeof record.content === "object") { - return [record.content]; - } - return [record]; -} - -export function extractToolResultText(result: unknown): string | undefined { - if (typeof result === "string") { - const trimmed = redactToolPayloadText(redactInlineDataUriValue(result)).trim(); - return trimmed ? truncateToolText(trimmed) : undefined; - } - if (!result || typeof result !== "object") { - return undefined; - } - const content = resolveToolResultContentBlocks(result); - const texts = collectTextContentBlocks(content) - .map((item) => { - const trimmed = item.trim(); - return trimmed ? trimmed : undefined; - }) - .filter((value): value is string => Boolean(value)); - if (texts.length > 0) { - return truncateToolText(texts.join("\n")); - } - const structuredTexts: string[] = []; - for (const item of content) { - const structured = stringifyStructuredToolResultContent(item); - if (structured) { - structuredTexts.push(structured); - } - } - if (structuredTexts.length === 0) { - return undefined; - } - return truncateToolText(structuredTexts.join("\n")); -} - -function pushUniqueMessagingMediaUrl(urls: string[], seen: Set, value: unknown): void { - if (typeof value !== "string") { - return; - } - const normalized = value.trim(); - if (!normalized || seen.has(normalized)) { - return; - } - seen.add(normalized); - urls.push(normalized); -} - -/** Collects messaging attachment references from tool-call arguments or result records. */ -export function collectMessagingMediaUrlsFromRecord(record: Record): string[] { - const urls: string[] = []; - const seen = new Set(); - const pushAttachment = (value: unknown) => { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return; - } - const attachment = value as Record; - for (const candidate of [ - attachment.media, - attachment.mediaUrl, - attachment.path, - attachment.filePath, - attachment.fileUrl, - attachment.url, - ]) { - pushUniqueMessagingMediaUrl(urls, seen, candidate); - } - }; - - for (const candidate of [ - record.media, - record.mediaUrl, - record.path, - record.filePath, - record.fileUrl, - ]) { - pushUniqueMessagingMediaUrl(urls, seen, candidate); - } - if (Array.isArray(record.mediaUrls)) { - for (const mediaUrl of record.mediaUrls) { - pushUniqueMessagingMediaUrl(urls, seen, mediaUrl); - } - } - if (Array.isArray(record.attachments)) { - for (const attachment of record.attachments) { - pushAttachment(attachment); - } - } - return urls; -} - -/** Collects messaging attachment references from a completed tool result. */ -export function collectMessagingMediaUrlsFromToolResult(result: unknown): string[] { - const urls: string[] = []; - const seen = new Set(); - const appendFromRecord = (value: unknown) => { - if (!value || typeof value !== "object") { - return; - } - for (const url of collectMessagingMediaUrlsFromRecord(value as Record)) { - if (!seen.has(url)) { - seen.add(url); - urls.push(url); - } - } - }; - - appendFromRecord(result); - if (result && typeof result === "object") { - appendFromRecord((result as Record).details); - } - const outputText = extractToolResultText(result); - if (outputText) { - try { - appendFromRecord(JSON.parse(outputText)); - } catch { - // Ignore non-JSON tool output. - } - } - return urls; -} - -/** Extract an internal source-reply payload from a completed message tool result. */ -export function extractMessagingToolSourceReplyPayload( - result: unknown, -): MessagingToolSourceReplyPayload | undefined { - const details = readToolResultDetails(result); - if (!details || details.sourceReplySink !== "internal-ui") { - return undefined; - } - const status = normalizeOptionalLowercaseString(details.deliveryStatus); - if (status && status !== "sent") { - return undefined; - } - const sourceReply = readRecord(details.sourceReply) ?? details; - const payload: MessagingToolSourceReplyPayload = {}; - const text = readStringValue(sourceReply.text) ?? readStringValue(details.message); - if (text) { - payload.text = text; - } - const mediaUrl = readStringValue(sourceReply.mediaUrl) ?? readStringValue(details.mediaUrl); - if (mediaUrl) { - payload.mediaUrl = mediaUrl; - } - const rawMediaUrls = Array.isArray(sourceReply.mediaUrls) - ? sourceReply.mediaUrls - : Array.isArray(details.mediaUrls) - ? details.mediaUrls - : []; - const mediaUrls = uniqueStrings( - rawMediaUrls.filter((value): value is string => typeof value === "string"), - ); - if (mediaUrls.length > 0) { - payload.mediaUrls = mediaUrls; - } - if (sourceReply.audioAsVoice === true || details.audioAsVoice === true) { - payload.audioAsVoice = true; - } - const presentation = normalizeMessagePresentation(sourceReply.presentation); - if (presentation) { - payload.presentation = presentation; - } - const interactive = normalizeLegacyInteractiveReply(sourceReply.interactive); - if (interactive) { - payload.interactive = interactive; - } - const channelData = readRecord(sourceReply.channelData); - if (channelData) { - payload.channelData = { ...channelData }; - } - const idempotencyKey = - readStringValue(sourceReply.idempotencyKey) ?? readStringValue(details.idempotencyKey); - if (idempotencyKey) { - payload.idempotencyKey = idempotencyKey; - } - return Object.keys(payload).length > 0 ? payload : undefined; -} - -// Core tool names that are allowed to emit trusted local media artifacts. -// Plugin tools must be explicitly passed as trusted run-local names by the caller. -const TRUSTED_TOOL_RESULT_MEDIA = new Set([ - "agents_list", - "apply_patch", - "browser", - "canvas", - AUTOMATIONS_TOOL_NAME, - "edit", - "exec", - "gateway", - "image", - "image_generate", - "memory_get", - "memory_search", - "message", - "music_generate", - "nodes", - "process", - "read", - "session_status", - "sessions_history", - "sessions_list", - "sessions_search", - "sessions_send", - "sessions_spawn", - "subagents", - "tts", - "video_generate", - "web_fetch", - "web_search", - "x_search", - "write", -]); -const HTTP_URL_RE = /^https?:\/\//i; - -function isCoreToolResultMediaTrustedName(toolName?: string): boolean { - if (!toolName) { - return false; - } - return TRUSTED_TOOL_RESULT_MEDIA.has(normalizeToolPolicyName(toolName)); -} - -function isExternalToolResult(result: unknown): boolean { - const details = readToolResultDetails(result); - if (!details) { - return false; - } - return typeof details.mcpServer === "string" || typeof details.mcpTool === "string"; -} - -function isToolResultMediaTrusted( - toolName?: string, - result?: unknown, - trustedLocalMediaToolNames?: ReadonlySet, -): boolean { - if (!toolName || isExternalToolResult(result)) { - return false; - } - const registeredName = toolName.trim(); - if (registeredName && trustedLocalMediaToolNames?.has(registeredName) === true) { - return true; - } - return isCoreToolResultMediaTrustedName(toolName); -} - -if (process.env.VITEST || process.env.NODE_ENV === "test") { - (globalThis as Record)[ - Symbol.for("openclaw.embeddedSubscribeToolsTestApi") - ] = { isToolResultMediaTrusted }; -} - -function isTrustedOwnedTtsLocalMedia( - toolName: string | undefined, - result: unknown, - trustedLocalMediaToolNames?: ReadonlySet, -): boolean { - if ( - !toolName || - !isToolResultMediaTrusted(toolName, result, trustedLocalMediaToolNames) || - normalizeToolPolicyName(toolName) !== "tts" - ) { - return false; - } - const media = readToolResultDetails(result)?.media; - if (!media || typeof media !== "object" || Array.isArray(media)) { - return false; - } - return (media as Record).trustedLocalMedia === true; -} - -export function filterToolResultMediaUrls( - toolName: string | undefined, - mediaUrls: string[], - result?: unknown, - trustedLocalMediaToolNames?: ReadonlySet, -): string[] { - if (mediaUrls.length === 0) { - return mediaUrls; - } - const trustedOwnedTtsLocalMedia = isTrustedOwnedTtsLocalMedia( - toolName, - result, - trustedLocalMediaToolNames, - ); - if (isToolResultMediaTrusted(toolName, result, trustedLocalMediaToolNames)) { - // When the current run provides its exact trusted local-media tool names, - // require the raw emitted tool name to match one of them before allowing - // local media paths. - // This blocks normalized aliases and case-variant collisions such as - // "Bash" -> "bash" or "Web_Search" -> "web_search" from inheriting a - // registered tool's media trust. TTS-generated local files carry a - // separate trusted-media flag from the owned tool result, so they can - // survive runs whose exact trusted set omitted the raw tts name. - if (trustedLocalMediaToolNames !== undefined) { - if (!trustedOwnedTtsLocalMedia) { - const registeredName = toolName?.trim(); - if (!registeredName || !trustedLocalMediaToolNames.has(registeredName)) { - return mediaUrls.filter((url) => HTTP_URL_RE.test(url.trim())); - } - } - } - return mediaUrls; - } - return mediaUrls.filter((url) => HTTP_URL_RE.test(url.trim())); -} - -/** - * Extract media file paths from a tool result. - * - * Strategy (first match wins): - * 1. Read structured `details.media` attachments from tool details. - * 2. Fall back to `details.path` when image content exists (legacy imageResult). - * - * Returns an empty array when no media is found (e.g. embedded `read` tool - * returns base64 image data but no file path; those need a different delivery - * path like saving to a temp file). - */ -type ToolResultMediaArtifact = { - mediaUrls: string[]; - audioAsVoice?: boolean; - trustedLocalMedia?: boolean; -}; - -function readToolResultDetailsMedia( - result: Record, -): Record | undefined { - const details = readToolResultDetails(result); - const media = - details?.media && typeof details.media === "object" && !Array.isArray(details.media) - ? (details.media as Record) - : undefined; - return media; -} - -function collectStructuredMediaUrls(media: Record): string[] { - const urls: string[] = []; - const pushString = (value: unknown) => { - if (typeof value !== "string") { - return; - } - const normalized = value.trim(); - if (normalized) { - urls.push(normalized); - } - }; - const pushAttachment = (value: unknown) => { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return; - } - const attachment = value as Record; - pushString(attachment.media); - pushString(attachment.path); - pushString(attachment.url); - pushString(attachment.mediaUrl); - pushString(attachment.filePath); - pushString(attachment.fileUrl); - }; - pushString(media.media); - pushString(media.path); - pushString(media.url); - pushString(media.mediaUrl); - pushString(media.filePath); - pushString(media.fileUrl); - if (Array.isArray(media.mediaUrls)) { - for (const value of media.mediaUrls) { - pushString(value); - } - } - if (Array.isArray(media.attachments)) { - for (const attachment of media.attachments) { - pushAttachment(attachment); - } - } - return uniqueStrings(urls); -} - -function isNonOutboundToolResultMedia(media: Record): boolean { - return media.outbound === false; -} - -function hasImageContentBlock(content: unknown[]): boolean { - for (const item of content) { - if (!item || typeof item !== "object") { - continue; - } - const entry = item as Record; - if (entry.type === "image") { - return true; - } - } - return false; -} - -export function extractToolResultMediaArtifact( - result: unknown, -): ToolResultMediaArtifact | undefined { - if (!result || typeof result !== "object") { - return undefined; - } - const record = result as Record; - const detailsMedia = readToolResultDetailsMedia(record); - if (detailsMedia) { - if (isNonOutboundToolResultMedia(detailsMedia)) { - return undefined; - } - const mediaUrls = collectStructuredMediaUrls(detailsMedia); - if (mediaUrls.length > 0) { - return { - mediaUrls, - ...(detailsMedia.audioAsVoice === true ? { audioAsVoice: true } : {}), - ...(detailsMedia.trustedLocalMedia === true ? { trustedLocalMedia: true } : {}), - }; - } - } - - const content = Array.isArray(record.content) ? record.content : null; - if (!content) { - return undefined; - } - - // Fall back to legacy details.path when image content exists but no - // structured media details. - if (hasImageContentBlock(content)) { - const details = record.details as Record | undefined; - const p = normalizeOptionalString(details?.path) ?? ""; - if (p) { - return { mediaUrls: [p] }; - } - } - - return undefined; -} - -export function extractToolErrorCode(result: unknown): string | undefined { - if (!result || typeof result !== "object") { - return undefined; - } - const record = result as Record; - return extractDirectErrorCodeField(record.details) ?? extractDirectErrorCodeField(record); -} - -export function isToolResultTimedOut(result: unknown): boolean { - const normalizedStatus = readToolResultStatus(result); - if (normalizedStatus === "timeout") { - return true; - } - return readToolResultDetails(result)?.timedOut === true; -} - -export function extractToolErrorMessage(result: unknown): string | undefined { - if (!result || typeof result !== "object") { - return undefined; - } - const record = result as Record; - const fromDetails = extractDirectErrorField(record.details); - if (fromDetails) { - return fromDetails; - } - const fromDetailsAggregated = extractAggregatedErrorField(record.details); - if (fromDetailsAggregated) { - return fromDetailsAggregated; - } - const fromRoot = extractDirectErrorField(record); - if (fromRoot) { - return fromRoot; - } - const text = extractToolResultText(result); - if (text) { - try { - const parsed = JSON.parse(text) as unknown; - const fromJson = extractErrorField(parsed); - if (fromJson) { - return fromJson; - } - } catch { - // Fall through to status/text fallback. - } - } - const fromDetailsStatus = extractErrorField(record.details); - if (fromDetailsStatus) { - return fromDetailsStatus; - } - const fromRootStatus = extractErrorField(record); - if (fromRootStatus) { - return fromRootStatus; - } - const status = readToolResultStatus(result); - if (status && !isToolResultError(result)) { - return undefined; - } - return text ? normalizeToolErrorText(text) : undefined; -} - -function resolveMessageToolTarget(params: { - action: string; - args: Record; - providerId: string | null; - currentChannelId?: string; - currentMessagingTarget?: string; -}): string | undefined { - const directTarget = - normalizeOptionalString(params.args.target) ?? - normalizeOptionalString(params.args.to) ?? - normalizeOptionalString(params.args.channelId); - if (directTarget) { - return directTarget; - } - const aliases = params.providerId - ? getChannelPlugin(params.providerId)?.actions?.messageActionTargetAliases?.[ - params.action as ChannelMessageActionName - ]?.deliveryTargetAliases - : undefined; - for (const alias of aliases ?? []) { - const aliasTarget = normalizeOptionalStringifiedId(params.args[alias]); - if (aliasTarget) { - return aliasTarget; - } - } - return params.currentMessagingTarget ?? params.currentChannelId; -} - -function resolveMessagingToolThreadEvidence(params: { - providerId: string; - to: string; - accountId?: string; - threadId?: string; - replyToId?: string; - allowImplicitThread: boolean; - threadSuppressed: boolean; - options?: { - config?: OpenClawConfig; - currentChannelId?: string; - currentMessagingTarget?: string; - currentThreadId?: string; - currentMessageId?: string | number; - replyToMode?: "off" | "first" | "all" | "batched"; - hasRepliedRef?: { value: boolean }; - }; -}): Pick { - const threading = getChannelPlugin(params.providerId)?.threading; - const autoThreadResolver = params.allowImplicitThread - ? threading?.resolveAutoThreadId - : undefined; - const replyTransport = params.replyToId - ? threading?.resolveReplyTransport?.({ - cfg: params.options?.config ?? {}, - accountId: params.accountId, - threadId: params.threadId, - replyToId: params.replyToId, - }) - : undefined; - const transportThreadId = normalizeOptionalStringifiedId(replyTransport?.threadId); - const replyToThreadId = - replyTransport?.threadId === null - ? normalizeOptionalString(replyTransport.replyToId) - : undefined; - const explicitThreadId = transportThreadId ?? replyToThreadId ?? params.threadId; - const currentChannelId = normalizeOptionalString(params.options?.currentChannelId); - const currentMessagingTarget = normalizeOptionalString(params.options?.currentMessagingTarget); - const currentThreadId = normalizeOptionalString(params.options?.currentThreadId); - const replyToMode = params.options?.replyToMode ?? (currentThreadId ? "all" : undefined); - const canResolveCurrentThread = Boolean( - (currentChannelId || currentMessagingTarget) && currentThreadId, - ); - const resolvedCurrentThreadId = - !explicitThreadId && !params.threadSuppressed && autoThreadResolver && canResolveCurrentThread - ? autoThreadResolver({ - cfg: params.options?.config ?? {}, - accountId: params.accountId, - to: params.to, - replyToId: params.replyToId, - toolContext: { - currentChannelId, - currentMessagingTarget, - currentThreadTs: currentThreadId, - currentMessageId: params.options?.currentMessageId, - replyToMode, - hasRepliedRef: params.options?.hasRepliedRef, - }, - }) - : undefined; - const threadImplicit = - !explicitThreadId && - !params.threadSuppressed && - Boolean(autoThreadResolver) && - (!canResolveCurrentThread || Boolean(resolvedCurrentThreadId)); - return { - ...((explicitThreadId ?? resolvedCurrentThreadId) - ? { threadId: explicitThreadId ?? resolvedCurrentThreadId } - : {}), - ...(threadImplicit ? { threadImplicit: true } : {}), - ...(params.threadSuppressed ? { threadSuppressed: true } : {}), - }; -} - -export function extractMessagingToolSend( - toolName: string, - args: Record, - options?: { - config?: OpenClawConfig; - currentChannelId?: string; - currentMessagingTarget?: string; - currentThreadId?: string; - currentMessageId?: string | number; - replyToMode?: "off" | "first" | "all" | "batched"; - hasRepliedRef?: { value: boolean }; - }, -): MessagingToolSend | undefined { - // Provider docking: new provider tools must implement plugin.actions.extractToolSend. - const action = normalizeOptionalString(args.action) ?? ""; - const accountId = normalizeOptionalString(args.accountId); - if (toolName === "conversations_send" || toolName === "conversations_turn") { - const conversationRef = normalizeOptionalString(args.conversationRef); - return conversationRef - ? { - tool: toolName, - provider: "conversation", - to: conversationRef, - } - : undefined; - } - if (toolName === "message") { - if (!isMessagingToolTargetEvidenceAction(toolName, args)) { - return undefined; - } - const providerRaw = normalizeOptionalString(args.provider) ?? ""; - const channelRaw = normalizeOptionalString(args.channel) ?? ""; - const providerHint = providerRaw || channelRaw; - const providerId = providerHint ? normalizeChannelId(providerHint) : null; - const toRaw = resolveMessageToolTarget({ - action, - args, - providerId, - currentChannelId: options?.currentChannelId, - currentMessagingTarget: options?.currentMessagingTarget, - }); - if (!toRaw) { - return undefined; - } - const provider = providerId ?? normalizeOptionalLowercaseString(providerHint) ?? "message"; - const to = normalizeTargetForProvider(provider, toRaw); - const pluginExtractionArgs = { ...args, to: toRaw }; - const pluginExtracted = providerId - ? getChannelPlugin(providerId)?.actions?.extractToolSend?.({ args: pluginExtractionArgs }) - : null; - const resolvedAccountId = normalizeOptionalString(pluginExtracted?.accountId) ?? accountId; - const threadId = - normalizeOptionalString(pluginExtracted?.threadId) ?? normalizeOptionalString(args.threadId); - const replyToId = normalizeOptionalString(args.replyTo); - // Normal sends use prepared core delivery, where provider transport owns - // reply/thread precedence. Other send-like actions use plugin dispatch. - const outboundReplyToId = action === "send" ? replyToId : undefined; - const threadSuppressed = - pluginExtracted?.threadSuppressed === true || - args.topLevel === true || - args.threadId === null; - return to - ? { - tool: toolName, - provider, - accountId: resolvedAccountId, - to, - ...(providerId - ? resolveMessagingToolThreadEvidence({ - providerId, - to, - accountId: resolvedAccountId, - threadId, - replyToId: outboundReplyToId, - allowImplicitThread: pluginExtracted - ? pluginExtracted.threadImplicit === true - : true, - threadSuppressed, - options, - }) - : { - ...(threadId ? { threadId } : {}), - ...(threadSuppressed ? { threadSuppressed: true } : {}), - }), - } - : undefined; - } - - const providerId = normalizeChannelId(toolName); - if (!providerId) { - return undefined; - } - const plugin = getChannelPlugin(providerId); - const extracted = plugin?.actions?.extractToolSend?.({ args }); - if (!extracted?.to) { - return undefined; - } - const to = normalizeTargetForProvider(providerId, extracted.to); - const threadId = normalizeOptionalString(extracted.threadId); - const threadSuppressed = extracted.threadSuppressed === true; - const extractedAccountId = normalizeOptionalString(extracted.accountId) ?? accountId; - const nativeReplyToMode = options?.replyToMode; - const nativeSingleUseMode = nativeReplyToMode === "first" || nativeReplyToMode === "batched"; - const canResolveNativeImplicitThread = - extracted.threadImplicit === true && - nativeReplyToMode !== undefined && - (!nativeSingleUseMode || options?.hasRepliedRef !== undefined); - return to - ? { - tool: toolName, - provider: providerId, - accountId: extractedAccountId, - to, - ...resolveMessagingToolThreadEvidence({ - providerId, - to, - accountId: extractedAccountId, - threadId, - allowImplicitThread: canResolveNativeImplicitThread, - threadSuppressed, - options, - }), - } - : undefined; -} - -/** Reconciles pending send evidence with the provider's successful action result. */ -export function extractMessagingToolSendResult( - pending: MessagingToolSend, - result: unknown, -): MessagingToolSend { - const providerId = normalizeChannelId(pending.provider); - const extracted = providerId - ? getChannelPlugin(providerId)?.actions?.extractToolSendResult?.({ - result, - send: { - to: pending.to ?? "", - accountId: pending.accountId, - threadId: pending.threadId, - threadImplicit: pending.threadImplicit, - threadSuppressed: pending.threadSuppressed, - }, - }) - : null; - if (!extracted?.to) { - return pending; - } - const extractedThreadId = normalizeOptionalString(extracted.threadId); - const providerReportedThread = - extractedThreadId != null || - extracted.threadImplicit === true || - extracted.threadSuppressed === true; - // Thread route fields are one state. Mixing provider and pending values can - // create contradictory implicit and suppressed evidence. - const threadEvidence = providerReportedThread ? extracted : pending; - return { - ...pending, - ...extracted, - accountId: normalizeOptionalString(extracted.accountId) ?? pending.accountId, - to: normalizeTargetForProvider(providerId ?? pending.provider, extracted.to), - threadId: normalizeOptionalString(threadEvidence.threadId), - threadImplicit: threadEvidence.threadImplicit === true ? true : undefined, - threadSuppressed: threadEvidence.threadSuppressed === true ? true : undefined, - }; -} -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-subscribe.ts b/src/agents/embedded-agent-subscribe.ts index 0e167bf68e54..a46b44a680bb 100644 --- a/src/agents/embedded-agent-subscribe.ts +++ b/src/agents/embedded-agent-subscribe.ts @@ -42,7 +42,7 @@ import { consumePendingToolMediaIntoReply, hasAssistantVisibleReply, readPendingToolMediaReply, -} from "./embedded-agent-subscribe.handlers.messages.js"; +} from "./embedded-agent-subscribe.handlers.messages.replies.js"; import { cleanupRunToolStartData, handleToolExecutionEnd, @@ -52,12 +52,12 @@ import type { EmbeddedAgentSubscribeContext, EmbeddedAgentSubscribeState, } from "./embedded-agent-subscribe.handlers.types.js"; +import type { SubscribeEmbeddedAgentSessionParams } from "./embedded-agent-subscribe.types.js"; import { - buildToolLifecycleErrorResult, extractToolResultMediaArtifact, filterToolResultMediaUrls, -} from "./embedded-agent-subscribe.tools.js"; -import type { SubscribeEmbeddedAgentSessionParams } from "./embedded-agent-subscribe.types.js"; +} from "./embedded-agent-tool-media.js"; +import { buildToolLifecycleErrorResult } from "./embedded-agent-tool-results.js"; import { createThinkingTagStreamState, stripDowngradedToolCallText, diff --git a/src/agents/embedded-agent-tool-media.ts b/src/agents/embedded-agent-tool-media.ts new file mode 100644 index 000000000000..9310f8da12c6 --- /dev/null +++ b/src/agents/embedded-agent-tool-media.ts @@ -0,0 +1,347 @@ +/** Extracts and trust-filters media from embedded-agent tool results. */ +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import { extractToolResultText } from "./embedded-agent-tool-results.js"; +import { normalizeToolPolicyName } from "./tool-policy.js"; +import { readToolResultDetails } from "./tool-result-error.js"; +import { AUTOMATIONS_TOOL_NAME } from "./tools/automations-tool-name.js"; + +function pushUniqueMessagingMediaUrl(urls: string[], seen: Set, value: unknown): void { + if (typeof value !== "string") { + return; + } + const normalized = value.trim(); + if (!normalized || seen.has(normalized)) { + return; + } + seen.add(normalized); + urls.push(normalized); +} + +/** Collects messaging attachment references from tool-call arguments or result records. */ +export function collectMessagingMediaUrlsFromRecord(record: Record): string[] { + const urls: string[] = []; + const seen = new Set(); + const pushAttachment = (value: unknown) => { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return; + } + const attachment = value as Record; + for (const candidate of [ + attachment.media, + attachment.mediaUrl, + attachment.path, + attachment.filePath, + attachment.fileUrl, + attachment.url, + ]) { + pushUniqueMessagingMediaUrl(urls, seen, candidate); + } + }; + + for (const candidate of [ + record.media, + record.mediaUrl, + record.path, + record.filePath, + record.fileUrl, + ]) { + pushUniqueMessagingMediaUrl(urls, seen, candidate); + } + if (Array.isArray(record.mediaUrls)) { + for (const mediaUrl of record.mediaUrls) { + pushUniqueMessagingMediaUrl(urls, seen, mediaUrl); + } + } + if (Array.isArray(record.attachments)) { + for (const attachment of record.attachments) { + pushAttachment(attachment); + } + } + return urls; +} + +/** Collects messaging attachment references from a completed tool result. */ +export function collectMessagingMediaUrlsFromToolResult(result: unknown): string[] { + const urls: string[] = []; + const seen = new Set(); + const appendFromRecord = (value: unknown) => { + if (!value || typeof value !== "object") { + return; + } + for (const url of collectMessagingMediaUrlsFromRecord(value as Record)) { + if (!seen.has(url)) { + seen.add(url); + urls.push(url); + } + } + }; + + appendFromRecord(result); + if (result && typeof result === "object") { + appendFromRecord((result as Record).details); + } + const outputText = extractToolResultText(result); + if (outputText) { + try { + appendFromRecord(JSON.parse(outputText)); + } catch { + // Ignore non-JSON tool output. + } + } + return urls; +} + +/** Extract an internal source-reply payload from a completed message tool result. */ + +const TRUSTED_TOOL_RESULT_MEDIA = new Set([ + "agents_list", + "apply_patch", + "browser", + "canvas", + AUTOMATIONS_TOOL_NAME, + "edit", + "exec", + "gateway", + "image", + "image_generate", + "memory_get", + "memory_search", + "message", + "music_generate", + "nodes", + "process", + "read", + "session_status", + "sessions_history", + "sessions_list", + "sessions_search", + "sessions_send", + "sessions_spawn", + "subagents", + "tts", + "video_generate", + "web_fetch", + "web_search", + "x_search", + "write", +]); +const HTTP_URL_RE = /^https?:\/\//i; + +function isCoreToolResultMediaTrustedName(toolName?: string): boolean { + if (!toolName) { + return false; + } + return TRUSTED_TOOL_RESULT_MEDIA.has(normalizeToolPolicyName(toolName)); +} + +function isExternalToolResult(result: unknown): boolean { + const details = readToolResultDetails(result); + if (!details) { + return false; + } + return typeof details.mcpServer === "string" || typeof details.mcpTool === "string"; +} + +function isToolResultMediaTrusted( + toolName?: string, + result?: unknown, + trustedLocalMediaToolNames?: ReadonlySet, +): boolean { + if (!toolName || isExternalToolResult(result)) { + return false; + } + const registeredName = toolName.trim(); + if (registeredName && trustedLocalMediaToolNames?.has(registeredName) === true) { + return true; + } + return isCoreToolResultMediaTrustedName(toolName); +} + +if (process.env.VITEST || process.env.NODE_ENV === "test") { + (globalThis as Record)[ + Symbol.for("openclaw.embeddedSubscribeToolsTestApi") + ] = { isToolResultMediaTrusted }; +} + +function isTrustedOwnedTtsLocalMedia( + toolName: string | undefined, + result: unknown, + trustedLocalMediaToolNames?: ReadonlySet, +): boolean { + if ( + !toolName || + !isToolResultMediaTrusted(toolName, result, trustedLocalMediaToolNames) || + normalizeToolPolicyName(toolName) !== "tts" + ) { + return false; + } + const media = readToolResultDetails(result)?.media; + if (!media || typeof media !== "object" || Array.isArray(media)) { + return false; + } + return (media as Record).trustedLocalMedia === true; +} + +export function filterToolResultMediaUrls( + toolName: string | undefined, + mediaUrls: string[], + result?: unknown, + trustedLocalMediaToolNames?: ReadonlySet, +): string[] { + if (mediaUrls.length === 0) { + return mediaUrls; + } + const trustedOwnedTtsLocalMedia = isTrustedOwnedTtsLocalMedia( + toolName, + result, + trustedLocalMediaToolNames, + ); + if (isToolResultMediaTrusted(toolName, result, trustedLocalMediaToolNames)) { + // When the current run provides its exact trusted local-media tool names, + // require the raw emitted tool name to match one of them before allowing + // local media paths. + // This blocks normalized aliases and case-variant collisions such as + // "Bash" -> "bash" or "Web_Search" -> "web_search" from inheriting a + // registered tool's media trust. TTS-generated local files carry a + // separate trusted-media flag from the owned tool result, so they can + // survive runs whose exact trusted set omitted the raw tts name. + if (trustedLocalMediaToolNames !== undefined) { + if (!trustedOwnedTtsLocalMedia) { + const registeredName = toolName?.trim(); + if (!registeredName || !trustedLocalMediaToolNames.has(registeredName)) { + return mediaUrls.filter((url) => HTTP_URL_RE.test(url.trim())); + } + } + } + return mediaUrls; + } + return mediaUrls.filter((url) => HTTP_URL_RE.test(url.trim())); +} + +/** + * Extract media file paths from a tool result. + * + * Strategy (first match wins): + * 1. Read structured `details.media` attachments from tool details. + * 2. Fall back to `details.path` when image content exists (legacy imageResult). + * + * Returns an empty array when no media is found (e.g. embedded `read` tool + * returns base64 image data but no file path; those need a different delivery + * path like saving to a temp file). + */ +type ToolResultMediaArtifact = { + mediaUrls: string[]; + audioAsVoice?: boolean; + trustedLocalMedia?: boolean; +}; + +function readToolResultDetailsMedia( + result: Record, +): Record | undefined { + const details = readToolResultDetails(result); + const media = + details?.media && typeof details.media === "object" && !Array.isArray(details.media) + ? (details.media as Record) + : undefined; + return media; +} + +function collectStructuredMediaUrls(media: Record): string[] { + const urls: string[] = []; + const pushString = (value: unknown) => { + if (typeof value !== "string") { + return; + } + const normalized = value.trim(); + if (normalized) { + urls.push(normalized); + } + }; + const pushAttachment = (value: unknown) => { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return; + } + const attachment = value as Record; + pushString(attachment.media); + pushString(attachment.path); + pushString(attachment.url); + pushString(attachment.mediaUrl); + pushString(attachment.filePath); + pushString(attachment.fileUrl); + }; + pushString(media.media); + pushString(media.path); + pushString(media.url); + pushString(media.mediaUrl); + pushString(media.filePath); + pushString(media.fileUrl); + if (Array.isArray(media.mediaUrls)) { + for (const value of media.mediaUrls) { + pushString(value); + } + } + if (Array.isArray(media.attachments)) { + for (const attachment of media.attachments) { + pushAttachment(attachment); + } + } + return uniqueStrings(urls); +} + +function isNonOutboundToolResultMedia(media: Record): boolean { + return media.outbound === false; +} + +function hasImageContentBlock(content: unknown[]): boolean { + for (const item of content) { + if (!item || typeof item !== "object") { + continue; + } + const entry = item as Record; + if (entry.type === "image") { + return true; + } + } + return false; +} + +export function extractToolResultMediaArtifact( + result: unknown, +): ToolResultMediaArtifact | undefined { + if (!result || typeof result !== "object") { + return undefined; + } + const record = result as Record; + const detailsMedia = readToolResultDetailsMedia(record); + if (detailsMedia) { + if (isNonOutboundToolResultMedia(detailsMedia)) { + return undefined; + } + const mediaUrls = collectStructuredMediaUrls(detailsMedia); + if (mediaUrls.length > 0) { + return { + mediaUrls, + ...(detailsMedia.audioAsVoice === true ? { audioAsVoice: true } : {}), + ...(detailsMedia.trustedLocalMedia === true ? { trustedLocalMedia: true } : {}), + }; + } + } + + const content = Array.isArray(record.content) ? record.content : null; + if (!content) { + return undefined; + } + + // Fall back to legacy details.path when image content exists but no + // structured media details. + if (hasImageContentBlock(content)) { + const details = record.details as Record | undefined; + const p = normalizeOptionalString(details?.path) ?? ""; + if (p) { + return { mediaUrls: [p] }; + } + } + + return undefined; +} diff --git a/src/agents/embedded-agent-tool-results.ts b/src/agents/embedded-agent-tool-results.ts new file mode 100644 index 000000000000..0405c1aa610d --- /dev/null +++ b/src/agents/embedded-agent-tool-results.ts @@ -0,0 +1,496 @@ +/** Sanitizes, extracts, and classifies embedded-agent tool execution results. */ +import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64"; +import { asOptionalRecord as readRecord } from "@openclaw/normalization-core/record-coerce"; +import { + normalizeOptionalLowercaseString, + normalizeOptionalString, + readStringValue, +} from "@openclaw/normalization-core/string-coerce"; +import { + redactSecrets, + redactSensitiveFieldValue, + redactToolPayloadText, +} from "../logging/redact.js"; +import { truncateUtf16Safe } from "../utils.js"; +import { collectTextContentBlocks } from "./content-blocks.js"; +import { + isToolResultError, + readToolResultDetails, + readToolResultStatus, +} from "./tool-result-error.js"; + +const TOOL_RESULT_MAX_CHARS = 8000; +const TOOL_ERROR_MAX_CHARS = 400; +const LIVE_EXEC_OUTPUT_MAX_CHARS = 8000; +const TOOL_DENIAL_ERROR_CODES = ["SYSTEM_RUN_DENIED", "INVALID_REQUEST"] as const; +const OPAQUE_STRUCTURED_RESULT_FIELDS = new Set(["encrypted_content", "encrypted_stdout"]); +const SENSITIVE_STRUCTURED_HEADER_FIELDS = new Set([ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "x-api-key", + "x-auth-token", +]); + +function truncateToolText(text: string): string { + if (text.length <= TOOL_RESULT_MAX_CHARS) { + return text; + } + return `${truncateUtf16Safe(text, TOOL_RESULT_MAX_CHARS)}\n…(truncated)…`; +} + +export function truncateLiveExecOutput(text: string): string { + if (text.length <= LIVE_EXEC_OUTPUT_MAX_CHARS) { + return text; + } + return `${truncateUtf16Safe(text, LIVE_EXEC_OUTPUT_MAX_CHARS)}\n...(live output truncated)...`; +} + +export function capLiveExecResult(result: unknown): unknown { + const details = readToolResultDetails(result); + if (!details || typeof details.status !== "string" || typeof details.aggregated !== "string") { + return result; + } + const aggregated = truncateLiveExecOutput(details.aggregated); + if (aggregated === details.aggregated) { + return result; + } + if (!result || typeof result !== "object" || Array.isArray(result)) { + return result; + } + return { + ...(result as Record), + details: { + ...details, + aggregated, + }, + }; +} + +function normalizeToolErrorText(text: string): string | undefined { + const trimmed = text.trim(); + if (!trimmed) { + return undefined; + } + const firstLine = trimmed.split(/\r?\n/)[0]?.trim() ?? ""; + if (!firstLine) { + return undefined; + } + return firstLine.length > TOOL_ERROR_MAX_CHARS + ? `${truncateUtf16Safe(firstLine, TOOL_ERROR_MAX_CHARS)}…` + : firstLine; +} + +function isErrorLikeStatus(status: string): boolean { + const normalized = normalizeOptionalLowercaseString(status); + if (!normalized) { + return false; + } + if ( + normalized === "0" || + normalized === "ok" || + normalized === "success" || + normalized === "completed" || + normalized === "running" + ) { + return false; + } + return /error|fail|timeout|timed[_\s-]?out|denied|cancel|invalid|forbidden/.test(normalized); +} + +function readErrorCandidate(value: unknown): string | undefined { + if (typeof value === "string") { + return normalizeToolErrorText(value); + } + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + if (typeof record.message === "string") { + return normalizeToolErrorText(record.message); + } + if (typeof record.error === "string") { + return normalizeToolErrorText(record.error); + } + return undefined; +} + +function extractErrorField(value: unknown): string | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + const direct = extractDirectErrorField(record); + if (direct) { + return direct; + } + const status = normalizeOptionalString(record.status) ?? ""; + if (!status || !isErrorLikeStatus(status)) { + return undefined; + } + return normalizeToolErrorText(status); +} + +function extractDirectErrorField(value: unknown): string | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + return ( + readErrorCandidate(record.error) ?? + readErrorCandidate(record.message) ?? + readErrorCandidate(record.reason) + ); +} + +function readErrorCodeField(value: unknown): string | undefined { + return typeof value === "string" ? normalizeOptionalString(value) : undefined; +} + +function readDenialErrorCodeFromMessage(value: unknown): string | undefined { + const message = typeof value === "string" ? normalizeOptionalString(value) : undefined; + if (!message) { + return undefined; + } + for (const code of TOOL_DENIAL_ERROR_CODES) { + if (message === code || message.startsWith(`${code}:`)) { + return code; + } + } + return undefined; +} + +function readNestedErrorCodeField(value: unknown): string | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + return ( + readDenialErrorCodeFromMessage(record.message) ?? + readDenialErrorCodeFromMessage(record.error) ?? + readErrorCodeField(record.code) ?? + readErrorCodeField(record.gatewayCode) + ); +} + +function extractDirectErrorCodeField(value: unknown): string | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + return ( + readNestedErrorCodeField(record.error) ?? + readNestedErrorCodeField(record.nodeError) ?? + readErrorCodeField(record.code) ?? + readErrorCodeField(record.gatewayCode) + ); +} + +export function buildToolLifecycleErrorResult(error: unknown): { + details: Record; +} { + const errorRecord = readRecord(error); + const rawDetails = readRecord(errorRecord?.details); + const nodeError = readRecord(rawDetails?.nodeError); + const gatewayCode = + readErrorCodeField(errorRecord?.gatewayCode) ?? readErrorCodeField(errorRecord?.code); + const message = error instanceof Error ? error.message : String(error); + return { + details: { + status: "error", + error: message, + ...(gatewayCode ? { gatewayCode } : {}), + ...(nodeError ? { nodeError } : {}), + }, + }; +} + +function extractAggregatedErrorField(value: unknown): string | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + return readErrorCandidate(record.aggregated); +} + +function redactStringsDeep(value: unknown, seen = new WeakSet()): unknown { + if (typeof value === "string") { + return redactToolPayloadText(value); + } + if (Array.isArray(value)) { + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + return value.map((item) => redactStringsDeep(item, seen)); + } + if (value && typeof value === "object") { + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + const out: Record = {}; + for (const [key, child] of Object.entries(value as Record)) { + out[key] = + typeof child === "string" + ? redactSensitiveFieldValue(key, child) + : redactStringsDeep(child, seen); + } + return out; + } + return value; +} + +export function sanitizeToolArgs(args: unknown): unknown { + return redactStringsDeep(args); +} + +export function sanitizeToolResult(result: unknown): unknown { + if (typeof result === "string") { + return redactToolPayloadText(result); + } + if (Array.isArray(result)) { + return redactSecrets(result); + } + if (!result || typeof result !== "object") { + return result; + } + const record = result as Record; + // Strip image data first so the deep redaction pass doesn't waste work + // scanning base64 payloads (and so we capture the original byte counts). + const preCleaned: Record = { ...record }; + const originalContent = Array.isArray(record.content) ? record.content : null; + if (originalContent) { + preCleaned.content = originalContent.map((item) => { + if (!item || typeof item !== "object") { + return item; + } + const entry = item as Record; + if (readStringValue(entry.type) === "image") { + const data = readStringValue(entry.data); + const existingBytes = typeof entry.bytes === "number" ? entry.bytes : undefined; + const bytes = data === undefined ? existingBytes : estimateBase64DecodedBytes(data); + const cleaned = { ...entry }; + delete cleaned.data; + return Object.assign({}, cleaned, { bytes, omitted: true }); + } + return entry; + }); + } + // Deep-redact the entire result so any top-level or nested string is + // protected, not just `details` and text content blocks. + const baseline = redactSecrets(preCleaned); + const out: Record = { ...baseline }; + const content = Array.isArray(baseline.content) ? baseline.content : null; + if (content) { + out.content = content.map((item) => { + if (!item || typeof item !== "object") { + return item; + } + const entry = item as Record; + if (readStringValue(entry.type) === "text" && typeof entry.text === "string") { + return Object.assign({}, entry, { text: truncateToolText(entry.text) }); + } + return entry; + }); + } + return out; +} + +const INLINE_DATA_URI_VALUE_PATTERN = + /^data:(?:[a-z][a-z0-9.+-]*\/[a-z0-9.+-]+)?(?:;[a-z0-9.+-]+(?:=[^,;"'\s]+)?)*,/i; + +function redactInlineDataUriValue(value: string): string { + const trimmed = value.trimStart(); + if (!INLINE_DATA_URI_VALUE_PATTERN.test(trimmed)) { + return value; + } + return `[inline data URI: ${value.length} chars]`; +} + +function carriesBinaryData(record: Record): boolean { + const type = normalizeOptionalLowercaseString(record.type); + if (type === "audio" || type === "image" || type === "base64") { + return true; + } + const mediaType = normalizeOptionalLowercaseString(record.media_type ?? record.mimeType); + return ( + mediaType?.startsWith("image/") === true || + mediaType?.startsWith("audio/") === true || + mediaType?.startsWith("video/") === true || + mediaType === "application/pdf" + ); +} + +function sanitizeStructuredToolResultValue( + value: unknown, + key = "", + parentCarriesBinaryData = false, + seen = new WeakSet(), +): unknown { + if (typeof value === "string") { + if (SENSITIVE_STRUCTURED_HEADER_FIELDS.has(key.toLowerCase())) { + return "***"; + } + if (key === "blob" || (key === "data" && parentCarriesBinaryData)) { + return `[binary omitted: ${value.length} chars]`; + } + // Claude CLI result blocks carry replay-only ciphertext that is not useful display text. + if (OPAQUE_STRUCTURED_RESULT_FIELDS.has(key)) { + return `[opaque data omitted: ${value.length} chars]`; + } + return truncateToolText(redactInlineDataUriValue(redactSensitiveFieldValue(key, value))); + } + if (typeof value === "bigint") { + return value.toString(); + } + if (!value || typeof value !== "object") { + return value; + } + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + if (Array.isArray(value)) { + // Keep the owning key so arrays of credentials inherit the same redaction policy. + return value.map((item) => + sanitizeStructuredToolResultValue(item, key, parentCarriesBinaryData, seen), + ); + } + const record = value as Record; + const hasBinaryData = carriesBinaryData(record); + return Object.fromEntries( + Object.entries(record).map(([childKey, child]) => [ + childKey, + sanitizeStructuredToolResultValue(child, childKey, hasBinaryData, seen), + ]), + ); +} + +function stringifyStructuredToolResultContent(block: unknown): string | undefined { + if (!block || typeof block !== "object") { + return undefined; + } + const record = block as Record; + const type = readStringValue(record.type); + if (type === "text" || type === "image" || type === "image_url" || type === "audio") { + return undefined; + } + try { + const serialized = JSON.stringify(sanitizeStructuredToolResultValue(record)); + const redacted = serialized ? redactToolPayloadText(serialized) : serialized; + return redacted && redacted !== "{}" ? redacted : undefined; + } catch { + return undefined; + } +} + +function resolveToolResultContentBlocks(result: object): unknown[] { + if (Array.isArray(result)) { + return result; + } + const record = result as Record; + // Typed provider blocks own their `content`; only untyped tool-result envelopes unwrap it. + if (readStringValue(record.type)) { + return [record]; + } + if (Array.isArray(record.content)) { + return record.content; + } + if (record.content && typeof record.content === "object") { + return [record.content]; + } + return [record]; +} + +export function extractToolResultText(result: unknown): string | undefined { + if (typeof result === "string") { + const trimmed = redactToolPayloadText(redactInlineDataUriValue(result)).trim(); + return trimmed ? truncateToolText(trimmed) : undefined; + } + if (!result || typeof result !== "object") { + return undefined; + } + const content = resolveToolResultContentBlocks(result); + const texts = collectTextContentBlocks(content) + .map((item) => { + const trimmed = item.trim(); + return trimmed ? trimmed : undefined; + }) + .filter((value): value is string => Boolean(value)); + if (texts.length > 0) { + return truncateToolText(texts.join("\n")); + } + const structuredTexts: string[] = []; + for (const item of content) { + const structured = stringifyStructuredToolResultContent(item); + if (structured) { + structuredTexts.push(structured); + } + } + if (structuredTexts.length === 0) { + return undefined; + } + return truncateToolText(structuredTexts.join("\n")); +} + +export function extractToolErrorCode(result: unknown): string | undefined { + if (!result || typeof result !== "object") { + return undefined; + } + const record = result as Record; + return extractDirectErrorCodeField(record.details) ?? extractDirectErrorCodeField(record); +} + +export function isToolResultTimedOut(result: unknown): boolean { + const normalizedStatus = readToolResultStatus(result); + if (normalizedStatus === "timeout") { + return true; + } + return readToolResultDetails(result)?.timedOut === true; +} + +export function extractToolErrorMessage(result: unknown): string | undefined { + if (!result || typeof result !== "object") { + return undefined; + } + const record = result as Record; + const fromDetails = extractDirectErrorField(record.details); + if (fromDetails) { + return fromDetails; + } + const fromDetailsAggregated = extractAggregatedErrorField(record.details); + if (fromDetailsAggregated) { + return fromDetailsAggregated; + } + const fromRoot = extractDirectErrorField(record); + if (fromRoot) { + return fromRoot; + } + const text = extractToolResultText(result); + if (text) { + try { + const parsed = JSON.parse(text) as unknown; + const fromJson = extractErrorField(parsed); + if (fromJson) { + return fromJson; + } + } catch { + // Fall through to status/text fallback. + } + } + const fromDetailsStatus = extractErrorField(record.details); + if (fromDetailsStatus) { + return fromDetailsStatus; + } + const fromRootStatus = extractErrorField(record); + if (fromRootStatus) { + return fromRootStatus; + } + const status = readToolResultStatus(result); + if (status && !isToolResultError(result)) { + return undefined; + } + return text ? normalizeToolErrorText(text) : undefined; +} diff --git a/src/agents/tools/message-tool.internal-source-reply.integration.test.ts b/src/agents/tools/message-tool.internal-source-reply.integration.test.ts index a5b9fe4c32c4..d3ba08c3a20f 100644 --- a/src/agents/tools/message-tool.internal-source-reply.integration.test.ts +++ b/src/agents/tools/message-tool.internal-source-reply.integration.test.ts @@ -3,8 +3,8 @@ import { describe, expect, it } from "vitest"; import { getReplyPayloadMetadata } from "../../auto-reply/reply-payload.js"; import { buildReplyPayloads } from "../../auto-reply/reply/agent-runner-payloads.js"; +import { extractMessagingToolSourceReplyPayload } from "../embedded-agent-messaging-extraction.js"; import { buildEmbeddedRunPayloads } from "../embedded-agent-runner/run/payloads.js"; -import { extractMessagingToolSourceReplyPayload } from "../embedded-agent-subscribe.tools.js"; import { createMessageTool } from "./message-tool-execution.js"; describe("WebChat message tool internal source reply", () => { diff --git a/src/auto-reply/reply/agent-runner-cli-dispatch.ts b/src/auto-reply/reply/agent-runner-cli-dispatch.ts index 0ed351546f1b..48e1b4f6d696 100644 --- a/src/auto-reply/reply/agent-runner-cli-dispatch.ts +++ b/src/auto-reply/reply/agent-runner-cli-dispatch.ts @@ -4,7 +4,7 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe import { runCliAgent } from "../../agents/cli-runner.js"; import type { RunCliAgentParams } from "../../agents/cli-runner/types.js"; import { clearCliSession, getCliSessionBinding } from "../../agents/cli-session.js"; -import { extractToolResultText } from "../../agents/embedded-agent-subscribe.tools.js"; +import { extractToolResultText } from "../../agents/embedded-agent-tool-results.js"; import type { EmbeddedAgentRunResult } from "../../agents/embedded-agent.js"; import { DEFAULT_FAST_MODE_AUTO_ON_SECONDS, diff --git a/src/gateway/server-methods/chat-reply-media.test.ts b/src/gateway/server-methods/chat-reply-media.test.ts index 1b0178341449..ff6ab0b15a51 100644 --- a/src/gateway/server-methods/chat-reply-media.test.ts +++ b/src/gateway/server-methods/chat-reply-media.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { consumePendingToolMediaIntoReply } from "../../agents/embedded-agent-subscribe.handlers.messages.js"; +import { consumePendingToolMediaIntoReply } from "../../agents/embedded-agent-subscribe.handlers.messages.replies.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { getAgentScopedMediaLocalRoots } from "../../media/local-roots.js"; import { diff --git a/src/gateway/worker-environments/live-event-projection.ts b/src/gateway/worker-environments/live-event-projection.ts index 9eeed7fe1f9b..12f793cda567 100644 --- a/src/gateway/worker-environments/live-event-projection.ts +++ b/src/gateway/worker-environments/live-event-projection.ts @@ -3,7 +3,7 @@ import { capLiveExecResult, sanitizeToolArgs, sanitizeToolResult, -} from "../../agents/embedded-agent-subscribe.tools.js"; +} from "../../agents/embedded-agent-tool-results.js"; import { normalizeToolPolicyName } from "../../agents/tool-policy.js"; import { createTrajectoryRuntimeRecorder } from "../../trajectory/runtime.js"; diff --git a/src/plugin-sdk/agent-harness-runtime.ts b/src/plugin-sdk/agent-harness-runtime.ts index 3fa5ec42e242..d6fbfc62e19b 100644 --- a/src/plugin-sdk/agent-harness-runtime.ts +++ b/src/plugin-sdk/agent-harness-runtime.ts @@ -216,14 +216,18 @@ export { isMessagingTool, isMessagingToolSendAction } from "../agents/embedded-a export { extractMessagingToolSend, extractMessagingToolSendResult, - extractToolErrorMessage, +} from "../agents/embedded-agent-messaging-extraction.js"; +export { extractToolResultMediaArtifact, filterToolResultMediaUrls, - isToolResultError, +} from "../agents/embedded-agent-tool-media.js"; +export { + extractToolErrorMessage, sanitizeToolResult, -} from "../agents/embedded-agent-subscribe.tools.js"; +} from "../agents/embedded-agent-tool-results.js"; export { formatToolExecutionErrorMessage, + isToolResultError, resolveToolExecutionErrorKind, resolveToolResultFailureKind, type ToolResultFailureKind, diff --git a/src/system-agent/agent-turn.ts b/src/system-agent/agent-turn.ts index fc9c6c00f507..b1e49add68c1 100644 --- a/src/system-agent/agent-turn.ts +++ b/src/system-agent/agent-turn.ts @@ -241,7 +241,7 @@ async function mirrorSystemAgentToolStateFromEvents(params: { { resolveSystemAgentProposalTransition, resolveSystemAgentDirectiveTransition }, ] = await Promise.all([ import("../infra/agent-events.js"), - import("../agents/embedded-agent-subscribe.tools.js"), + import("../agents/embedded-agent-tool-results.js"), import("../agents/tools/system-agent-tool.js"), ]); return onAgentEvent((evt) => {