diff --git a/scripts/firecrawl-compare.ts b/scripts/firecrawl-compare.ts index 76e3c6eaa18a..8d03d845f121 100644 --- a/scripts/firecrawl-compare.ts +++ b/scripts/firecrawl-compare.ts @@ -1,5 +1,6 @@ // Firecrawl Compare script supports OpenClaw repository automation. import { pathToFileURL } from "node:url"; +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { fetchFirecrawlContent } from "../extensions/firecrawl/api.ts"; import { formatErrorMessage } from "../src/infra/errors.ts"; import { extractReadableContent } from "../src/web-fetch/content-extractors.runtime.js"; @@ -32,7 +33,7 @@ function truncate(value: string, max = 180): string { if (!value) { return ""; } - return value.length > max ? `${value.slice(0, max)}…` : value; + return value.length > max ? `${truncateUtf16Safe(value, max)}…` : value; } function readBoundedResponseText( @@ -176,4 +177,5 @@ export const testing = { FETCH_HTML_MAX_BYTES, fetchHtml, readBoundedResponseText, + truncate, }; diff --git a/test/scripts/firecrawl-compare.test.ts b/test/scripts/firecrawl-compare.test.ts index 707d1c4f65a7..2ea6168025cb 100644 --- a/test/scripts/firecrawl-compare.test.ts +++ b/test/scripts/firecrawl-compare.test.ts @@ -3,6 +3,10 @@ import { describe, expect, it } from "vitest"; import { testing as firecrawlCompareTesting } from "../../scripts/firecrawl-compare.ts"; describe("firecrawl-compare", () => { + it("does not split surrogate pairs when truncating output", () => { + expect(firecrawlCompareTesting.truncate(`ab🤖cd`, 3)).toBe("ab…"); + }); + it("fetches local HTML under the byte cap", async () => { const result = await firecrawlCompareTesting.fetchHtml("https://example.test/page", { fetchImpl: (() =>