From 66081c09ee501d5edd8b0d2f0fa46218f8934833 Mon Sep 17 00:00:00 2001 From: Peter Lee Date: Sun, 5 Jul 2026 07:48:35 -0500 Subject: [PATCH] fix(core): keep backend truncation UTF-16 safe (#100244) * fix(core): use UTF-16-safe truncation for chat display, ACP stream relay, and native hook relay * fix(core): narrow UTF-16 truncation repair Co-authored-by: xialonglee Co-authored-by: ZengWen-DT --------- Co-authored-by: Vincent Koc Co-authored-by: ZengWen-DT --- extensions/active-memory/index.test.ts | 74 ++++++++++++++++------ extensions/active-memory/index.ts | 8 ++- src/agents/acp-spawn-parent-stream.test.ts | 31 +++++++++ src/agents/acp-spawn-parent-stream.ts | 7 +- 4 files changed, 93 insertions(+), 27 deletions(-) diff --git a/extensions/active-memory/index.test.ts b/extensions/active-memory/index.test.ts index b0e64d6230e7..8531f452c3d0 100644 --- a/extensions/active-memory/index.test.ts +++ b/extensions/active-memory/index.test.ts @@ -290,6 +290,27 @@ describe("active-memory plugin", () => { (result as { prependContext?: unknown } | undefined)?.prependContext, "expected prependContext", ); + const runRecallWithSummary = async (params: { + prompt: string; + summary: string; + memoryText?: string; + }): Promise => { + runEmbeddedAgent.mockImplementationOnce(async (runParams: { sessionFile: string }) => { + await writeUsableMemoryTranscript(runParams.sessionFile, params.memoryText ?? params.summary); + return { payloads: [{ text: params.summary }] }; + }); + return requirePrependContext( + await hooks.before_prompt_build( + { prompt: params.prompt, messages: [] }, + { + agentId: "main", + trigger: "user", + sessionKey: "agent:main:main", + messageProvider: "webchat", + }, + ), + ); + }; const expectPrependContextContains = (result: unknown, text: string) => { expect(requirePrependContext(result)).toContain(text); }; @@ -5335,34 +5356,45 @@ describe("active-memory plugin", () => { maxSummaryChars: 40, }; plugin.register(api as unknown as OpenClawPluginApi); - runEmbeddedAgent.mockImplementationOnce(async (params: { sessionFile: string }) => { - await writeUsableMemoryTranscript(params.sessionFile, "alpha beta gamma"); - return { - payloads: [ - { - text: "alpha beta gamma delta epsilon zetalongword", - }, - ], - }; + const prependContext = await runRecallWithSummary({ + prompt: "what wings should i order? word-boundary-truncation-40", + summary: "alpha beta gamma delta epsilon zetalongword", + memoryText: "alpha beta gamma", }); - - const result = await hooks.before_prompt_build( - { prompt: "what wings should i order? word-boundary-truncation-40", messages: [] }, - { - agentId: "main", - trigger: "user", - sessionKey: "agent:main:main", - messageProvider: "webchat", - }, - ); - - const prependContext = requirePrependContext(result); expect(prependContext).toContain("alpha beta gamma"); expect(prependContext).toContain("alpha beta gamma delta epsilon…"); expect(prependContext).not.toContain("zetalo"); expect(prependContext).not.toContain("zetalongword"); }); + it.each([ + { + name: "split surrogate", + summary: `${"a".repeat(38)}🎉TAILWORD`, + expected: `${"a".repeat(38)}…`, + }, + { + name: "whitespace before a split surrogate", + summary: `alpha beta ${"c".repeat(26)} 🎉TAILWORD`, + expected: `alpha beta ${"c".repeat(26)}…`, + }, + ])("keeps $name truncation UTF-16 safe", async ({ name, summary, expected }) => { + api.pluginConfig = { + agents: ["main"], + maxSummaryChars: 40, + }; + plugin.register(api as unknown as OpenClawPluginApi); + + const prependContext = await runRecallWithSummary({ + prompt: `recall summary boundary: ${name}`, + summary, + memoryText: expected, + }); + + expect(prependContext).toContain(expected); + expect(prependContext).not.toContain("TAILWORD"); + }); + it("asks recall subagents to mark mutable operational facts stale unless source status is current", async () => { await hooks.before_prompt_build( { prompt: "is autonomous pickup running?", messages: [] }, diff --git a/extensions/active-memory/index.ts b/extensions/active-memory/index.ts index ffa7a927567c..13dd3f62dfa9 100644 --- a/extensions/active-memory/index.ts +++ b/extensions/active-memory/index.ts @@ -37,6 +37,7 @@ import { uniqueStrings, } from "openclaw/plugin-sdk/string-coerce-runtime"; import { tempWorkspace, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; const DEFAULT_TIMEOUT_MS = 15_000; const DEFAULT_AGENT_ID = "main"; @@ -2570,15 +2571,16 @@ function truncateSummary(summary: string, maxSummaryChars: number): string { return ellipsis.slice(0, Math.max(0, maxSummaryChars)); } const contentMaxChars = maxSummaryChars - ellipsis.length; - const bounded = trimmed.slice(0, contentMaxChars).trimEnd(); + const rawBounded = trimmed.slice(0, contentMaxChars).trimEnd(); + const bounded = truncateUtf16Safe(trimmed, contentMaxChars).trimEnd(); const nextChar = trimmed.charAt(contentMaxChars); if (!nextChar || /\s/.test(nextChar)) { return `${bounded}${ellipsis}`; } - const lastBoundary = bounded.search(/\s\S*$/); + const lastBoundary = rawBounded.search(/\s\S*$/); if (lastBoundary > 0) { - return `${bounded.slice(0, lastBoundary).trimEnd()}${ellipsis}`; + return `${truncateUtf16Safe(trimmed, lastBoundary).trimEnd()}${ellipsis}`; } return `${bounded}${ellipsis}`; diff --git a/src/agents/acp-spawn-parent-stream.test.ts b/src/agents/acp-spawn-parent-stream.test.ts index 377baffab269..62581a275abf 100644 --- a/src/agents/acp-spawn-parent-stream.test.ts +++ b/src/agents/acp-spawn-parent-stream.test.ts @@ -1406,6 +1406,37 @@ describe("startAcpSpawnParentStreamRelay", () => { relay.dispose(); }); + it.each([ + { + name: "preview cutoff", + delta: `${"a".repeat(218)}😀tail`, + expected: `${"a".repeat(218)}…`, + }, + { + name: "retained buffer start", + delta: `😀${"b".repeat(3_999)}`, + expected: `${"b".repeat(219)}…`, + }, + ])("keeps $name on UTF-16 boundaries", ({ delta, expected }) => { + const relay = startAcpSpawnParentStreamRelay({ + runId: "run-utf16-safe", + parentSessionKey: "agent:main:main", + childSessionKey: "agent:codex:acp:utf16-safe", + agentId: "codex", + streamFlushMs: 0, + noOutputNoticeMs: 120_000, + }); + + emitAgentEvent({ + runId: "run-utf16-safe", + stream: "assistant", + data: { delta }, + }); + + expect(collectedTexts()[1]).toBe(`codex: ${expected}`); + relay.dispose(); + }); + it("resolves ACP spawn stream log path from session metadata", () => { readAcpSessionEntryMock.mockReturnValue({ storePath: "/tmp/openclaw/agents/codex/sessions/sessions.json", diff --git a/src/agents/acp-spawn-parent-stream.ts b/src/agents/acp-spawn-parent-stream.ts index 3589350d0521..6595c70aea4d 100644 --- a/src/agents/acp-spawn-parent-stream.ts +++ b/src/agents/acp-spawn-parent-stream.ts @@ -3,6 +3,7 @@ import { mkdir } from "node:fs/promises"; import path from "node:path"; import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { sliceUtf16Safe, truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { readAcpSessionEntry } from "../acp/runtime/session-meta.js"; import { isAcpTagVisible, @@ -50,9 +51,9 @@ function truncate(value: string, maxChars: number): string { return value; } if (maxChars <= 1) { - return value.slice(0, maxChars); + return truncateUtf16Safe(value, maxChars); } - return `${value.slice(0, maxChars - 1)}…`; + return `${truncateUtf16Safe(value, maxChars - 1)}…`; } function normalizeStringArray(value: unknown): string[] { @@ -503,7 +504,7 @@ export function startAcpSpawnParentStreamRelay(params: { pendingProgressKind = kind; pendingText += delta; if (pendingText.length > STREAM_BUFFER_MAX_CHARS) { - pendingText = pendingText.slice(-STREAM_BUFFER_MAX_CHARS); + pendingText = sliceUtf16Safe(pendingText, -STREAM_BUFFER_MAX_CHARS); } if (pendingText.length >= STREAM_SNIPPET_MAX_CHARS || delta.includes("\n\n")) { flushPending();