From f267a4f99b09cfb1d40aa1c63d5b815e488a6c6f Mon Sep 17 00:00:00 2001 From: lzyyzznl Date: Tue, 21 Jul 2026 09:50:03 +0800 Subject: [PATCH] fix(scripts): preserve UTF-16 surrogate pairs in firecrawl truncation (#109561) * fix(scripts): preserve UTF-16 surrogate pairs in firecrawl truncation * test(scripts): cover UTF-16-safe firecrawl truncation --------- Co-authored-by: Peter Steinberger --- scripts/firecrawl-compare.ts | 4 +++- test/scripts/firecrawl-compare.test.ts | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/firecrawl-compare.ts b/scripts/firecrawl-compare.ts index 76e3c6eaa18a..8d03d845f121 100644 --- a/scripts/firecrawl-compare.ts +++ b/scripts/firecrawl-compare.ts @@ -1,5 +1,6 @@ // Firecrawl Compare script supports OpenClaw repository automation. import { pathToFileURL } from "node:url"; +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { fetchFirecrawlContent } from "../extensions/firecrawl/api.ts"; import { formatErrorMessage } from "../src/infra/errors.ts"; import { extractReadableContent } from "../src/web-fetch/content-extractors.runtime.js"; @@ -32,7 +33,7 @@ function truncate(value: string, max = 180): string { if (!value) { return ""; } - return value.length > max ? `${value.slice(0, max)}…` : value; + return value.length > max ? `${truncateUtf16Safe(value, max)}…` : value; } function readBoundedResponseText( @@ -176,4 +177,5 @@ export const testing = { FETCH_HTML_MAX_BYTES, fetchHtml, readBoundedResponseText, + truncate, }; diff --git a/test/scripts/firecrawl-compare.test.ts b/test/scripts/firecrawl-compare.test.ts index 707d1c4f65a7..2ea6168025cb 100644 --- a/test/scripts/firecrawl-compare.test.ts +++ b/test/scripts/firecrawl-compare.test.ts @@ -3,6 +3,10 @@ import { describe, expect, it } from "vitest"; import { testing as firecrawlCompareTesting } from "../../scripts/firecrawl-compare.ts"; describe("firecrawl-compare", () => { + it("does not split surrogate pairs when truncating output", () => { + expect(firecrawlCompareTesting.truncate(`ab🤖cd`, 3)).toBe("ab…"); + }); + it("fetches local HTML under the byte cap", async () => { const result = await firecrawlCompareTesting.fetchHtml("https://example.test/page", { fetchImpl: (() =>