From 45f261ff7ad13de99452f94cded72d8384f4775b Mon Sep 17 00:00:00 2001 From: mushuiyu886 Date: Mon, 29 Jun 2026 02:19:55 +0800 Subject: [PATCH] fix(memory-wiki): truncate import insights safely Use the shared UTF-16 safe truncation helper for Memory Wiki import insight summaries and add regression coverage for surrogate-boundary summaries. --- .../memory-wiki/src/import-insights.test.ts | 71 +++++++++++++++++++ extensions/memory-wiki/src/import-insights.ts | 3 +- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/extensions/memory-wiki/src/import-insights.test.ts b/extensions/memory-wiki/src/import-insights.test.ts index fba91a567cc1..2b05a138ab0d 100644 --- a/extensions/memory-wiki/src/import-insights.test.ts +++ b/extensions/memory-wiki/src/import-insights.test.ts @@ -8,6 +8,24 @@ import { createMemoryWikiTestHarness } from "./test-helpers.js"; const { createVault } = createMemoryWikiTestHarness(); +function hasLoneSurrogate(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) { + return true; + } + index += 1; + continue; + } + if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +} + describe("listMemoryWikiImportInsights", () => { it("clusters ChatGPT import pages by topic and extracts digest fields", async () => { const { rootDir, config } = await createVault({ @@ -139,4 +157,57 @@ describe("listMemoryWikiImportInsights", () => { expect(healthItem?.lastUserLine).toBeUndefined(); expect(healthItem?.assistantOpener).toBeUndefined(); }); + + it("truncates import insight summaries without leaving lone surrogates", async () => { + const { rootDir, config } = await createVault({ + prefix: "memory-wiki-import-insights-surrogate-", + initialize: true, + }); + await fs.mkdir(path.join(rootDir, "sources"), { recursive: true }); + const assistantOpener = `${"a".repeat(178)}😀${"b".repeat(20)}`; + await fs.writeFile( + path.join(rootDir, "sources", "chatgpt-emoji.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "source", + id: "source.chatgpt.emoji", + title: "ChatGPT Export: Emoji truncation", + sourceType: "chatgpt-export", + riskLevel: "low", + riskReasons: [], + labels: ["domain/work", "area/memory", "topic/memory"], + updatedAt: "2026-02-01T12:00:00.000Z", + }, + body: [ + "# ChatGPT Export: Emoji truncation", + "", + "## Auto Digest", + "- User messages: 1", + "- Assistant messages: 1", + "- First user line: summarize this", + "- Last user line: summarize this", + "- Preference signals:", + " - prefers emoji-safe summaries", + "", + "## Active Branch Transcript", + "### User", + "", + "summarize this", + "", + "### Assistant", + "", + assistantOpener, + "", + ].join("\n"), + }), + "utf8", + ); + + const result = await listMemoryWikiImportInsights(config); + + const item = result.clusters[0]?.items[0]; + expect(item?.summary).toBe(`${"a".repeat(178)}…`); + expect(hasLoneSurrogate(item?.summary ?? "")).toBe(false); + expect(item?.summary).not.toContain("�"); + }); }); diff --git a/extensions/memory-wiki/src/import-insights.ts b/extensions/memory-wiki/src/import-insights.ts index bf60964032fb..6ff8a89526bc 100644 --- a/extensions/memory-wiki/src/import-insights.ts +++ b/extensions/memory-wiki/src/import-insights.ts @@ -1,3 +1,4 @@ +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; // Memory Wiki plugin module implements import insights behavior. import type { ResolvedMemoryWikiConfig } from "./config.js"; import { parseWikiMarkdown } from "./markdown.js"; @@ -208,7 +209,7 @@ function shortenSentence(value: string, maxLength = 180): string { if (compact.length <= maxLength) { return compact; } - return `${compact.slice(0, maxLength - 1).trimEnd()}…`; + return `${truncateUtf16Safe(compact, maxLength - 1).trimEnd()}…`; } function extractCorrectionSignals(turns: TranscriptTurn[]): string[] {