mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
fix(memory-wiki): truncate import insights safely
Use the shared UTF-16 safe truncation helper for Memory Wiki import insight summaries and add regression coverage for surrogate-boundary summaries.
This commit is contained in:
@@ -8,6 +8,24 @@ import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
function hasLoneSurrogate(value: string): boolean {
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const code = value.charCodeAt(index);
|
||||
if (code >= 0xd800 && code <= 0xdbff) {
|
||||
const next = value.charCodeAt(index + 1);
|
||||
if (!(next >= 0xdc00 && next <= 0xdfff)) {
|
||||
return true;
|
||||
}
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (code >= 0xdc00 && code <= 0xdfff) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
describe("listMemoryWikiImportInsights", () => {
|
||||
it("clusters ChatGPT import pages by topic and extracts digest fields", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
@@ -139,4 +157,57 @@ describe("listMemoryWikiImportInsights", () => {
|
||||
expect(healthItem?.lastUserLine).toBeUndefined();
|
||||
expect(healthItem?.assistantOpener).toBeUndefined();
|
||||
});
|
||||
|
||||
it("truncates import insight summaries without leaving lone surrogates", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-import-insights-surrogate-",
|
||||
initialize: true,
|
||||
});
|
||||
await fs.mkdir(path.join(rootDir, "sources"), { recursive: true });
|
||||
const assistantOpener = `${"a".repeat(178)}😀${"b".repeat(20)}`;
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "chatgpt-emoji.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.chatgpt.emoji",
|
||||
title: "ChatGPT Export: Emoji truncation",
|
||||
sourceType: "chatgpt-export",
|
||||
riskLevel: "low",
|
||||
riskReasons: [],
|
||||
labels: ["domain/work", "area/memory", "topic/memory"],
|
||||
updatedAt: "2026-02-01T12:00:00.000Z",
|
||||
},
|
||||
body: [
|
||||
"# ChatGPT Export: Emoji truncation",
|
||||
"",
|
||||
"## Auto Digest",
|
||||
"- User messages: 1",
|
||||
"- Assistant messages: 1",
|
||||
"- First user line: summarize this",
|
||||
"- Last user line: summarize this",
|
||||
"- Preference signals:",
|
||||
" - prefers emoji-safe summaries",
|
||||
"",
|
||||
"## Active Branch Transcript",
|
||||
"### User",
|
||||
"",
|
||||
"summarize this",
|
||||
"",
|
||||
"### Assistant",
|
||||
"",
|
||||
assistantOpener,
|
||||
"",
|
||||
].join("\n"),
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await listMemoryWikiImportInsights(config);
|
||||
|
||||
const item = result.clusters[0]?.items[0];
|
||||
expect(item?.summary).toBe(`${"a".repeat(178)}…`);
|
||||
expect(hasLoneSurrogate(item?.summary ?? "")).toBe(false);
|
||||
expect(item?.summary).not.toContain("�");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
// Memory Wiki plugin module implements import insights behavior.
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { parseWikiMarkdown } from "./markdown.js";
|
||||
@@ -208,7 +209,7 @@ function shortenSentence(value: string, maxLength = 180): string {
|
||||
if (compact.length <= maxLength) {
|
||||
return compact;
|
||||
}
|
||||
return `${compact.slice(0, maxLength - 1).trimEnd()}…`;
|
||||
return `${truncateUtf16Safe(compact, maxLength - 1).trimEnd()}…`;
|
||||
}
|
||||
|
||||
function extractCorrectionSignals(turns: TranscriptTurn[]): string[] {
|
||||
|
||||
Reference in New Issue
Block a user