fix(memory-wiki): truncate import insights safely

Use the shared UTF-16 safe truncation helper for Memory Wiki import insight summaries and add regression coverage for surrogate-boundary summaries.
This commit is contained in:
mushuiyu886
2026-06-29 02:19:55 +08:00
committed by GitHub
parent 48f34b1d4d
commit 45f261ff7a
2 changed files with 73 additions and 1 deletions
@@ -8,6 +8,24 @@ import { createMemoryWikiTestHarness } from "./test-helpers.js";
const { createVault } = createMemoryWikiTestHarness();
function hasLoneSurrogate(value: string): boolean {
for (let index = 0; index < value.length; index += 1) {
const code = value.charCodeAt(index);
if (code >= 0xd800 && code <= 0xdbff) {
const next = value.charCodeAt(index + 1);
if (!(next >= 0xdc00 && next <= 0xdfff)) {
return true;
}
index += 1;
continue;
}
if (code >= 0xdc00 && code <= 0xdfff) {
return true;
}
}
return false;
}
describe("listMemoryWikiImportInsights", () => {
it("clusters ChatGPT import pages by topic and extracts digest fields", async () => {
const { rootDir, config } = await createVault({
@@ -139,4 +157,57 @@ describe("listMemoryWikiImportInsights", () => {
expect(healthItem?.lastUserLine).toBeUndefined();
expect(healthItem?.assistantOpener).toBeUndefined();
});
it("truncates import insight summaries without leaving lone surrogates", async () => {
const { rootDir, config } = await createVault({
prefix: "memory-wiki-import-insights-surrogate-",
initialize: true,
});
await fs.mkdir(path.join(rootDir, "sources"), { recursive: true });
const assistantOpener = `${"a".repeat(178)}😀${"b".repeat(20)}`;
await fs.writeFile(
path.join(rootDir, "sources", "chatgpt-emoji.md"),
renderWikiMarkdown({
frontmatter: {
pageType: "source",
id: "source.chatgpt.emoji",
title: "ChatGPT Export: Emoji truncation",
sourceType: "chatgpt-export",
riskLevel: "low",
riskReasons: [],
labels: ["domain/work", "area/memory", "topic/memory"],
updatedAt: "2026-02-01T12:00:00.000Z",
},
body: [
"# ChatGPT Export: Emoji truncation",
"",
"## Auto Digest",
"- User messages: 1",
"- Assistant messages: 1",
"- First user line: summarize this",
"- Last user line: summarize this",
"- Preference signals:",
" - prefers emoji-safe summaries",
"",
"## Active Branch Transcript",
"### User",
"",
"summarize this",
"",
"### Assistant",
"",
assistantOpener,
"",
].join("\n"),
}),
"utf8",
);
const result = await listMemoryWikiImportInsights(config);
const item = result.clusters[0]?.items[0];
expect(item?.summary).toBe(`${"a".repeat(178)}`);
expect(hasLoneSurrogate(item?.summary ?? "")).toBe(false);
expect(item?.summary).not.toContain("");
});
});
@@ -1,3 +1,4 @@
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
// Memory Wiki plugin module implements import insights behavior.
import type { ResolvedMemoryWikiConfig } from "./config.js";
import { parseWikiMarkdown } from "./markdown.js";
@@ -208,7 +209,7 @@ function shortenSentence(value: string, maxLength = 180): string {
if (compact.length <= maxLength) {
return compact;
}
return `${compact.slice(0, maxLength - 1).trimEnd()}`;
return `${truncateUtf16Safe(compact, maxLength - 1).trimEnd()}`;
}
function extractCorrectionSignals(turns: TranscriptTurn[]): string[] {