refactor(zalouser): share offset-preserving text chunk ranges (#105842)

* refactor(zalouser): share text chunk ranges

* chore(plugin-sdk): refresh combined API baseline
This commit is contained in:
Peter Steinberger
2026-07-12 23:46:16 -07:00
committed by GitHub
parent be7241b656
commit 4cbbb86d5e
10 changed files with 160 additions and 94 deletions
+42 -1
View File
@@ -2,7 +2,7 @@
* Tests text and Markdown chunking helpers exported by the plugin SDK.
*/
import { describe, expect, it } from "vitest";
import { chunkTextForOutbound } from "./text-chunking.js";
import { chunkTextForOutbound, chunkTextRanges } from "./text-chunking.js";
describe("chunkTextForOutbound", () => {
it.each([
@@ -28,3 +28,44 @@ describe("chunkTextForOutbound", () => {
expect(chunkTextForOutbound(text, maxLen)).toEqual(expected);
});
});
describe("chunkTextRanges", () => {
it("returns contiguous hard ranges without dropping whitespace", () => {
const text = "alpha beta\n\ngamma delta";
const ranges = chunkTextRanges(text, { limit: 12, mode: "hard" });
expect(ranges).toEqual([
{ start: 0, end: 12 },
{ start: 12, end: 24 },
]);
expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual([
"alpha beta\n",
"\ngamma delta",
]);
});
it("prefers paragraph, newline, then whitespace boundaries", () => {
const text = "a\n\nb\nc xyz";
const ranges = chunkTextRanges(text, { limit: 8, mode: "preferred" });
expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual(["a\n\n", "b\nc xyz"]);
});
it("falls back to hard ranges and handles empty or non-positive limits", () => {
expect(chunkTextRanges("abcdefgh", { limit: 3, mode: "preferred" })).toEqual([
{ start: 0, end: 3 },
{ start: 3, end: 6 },
{ start: 6, end: 8 },
]);
expect(chunkTextRanges("", { limit: 3 })).toEqual([]);
expect(chunkTextRanges("abc", { limit: 0 })).toEqual([{ start: 0, end: 3 }]);
});
it.each(["hard", "preferred"] as const)("keeps surrogate pairs intact in %s mode", (mode) => {
expect(chunkTextRanges("a😀b", { limit: 2, mode })).toEqual([
{ start: 0, end: 1 },
{ start: 1, end: 3 },
{ start: 3, end: 4 },
]);
});
});
+7
View File
@@ -1,6 +1,13 @@
// Text chunking helpers split long outbound text while preserving readable line boundaries.
import { chunkTextByBreakResolver } from "../shared/text-chunking.js";
/** Offset-preserving text ranges for transports with native style metadata. */
export {
chunkTextRanges,
type ChunkTextRangesOptions,
type TextChunkRange,
} from "../../packages/markdown-core/src/chunk-text.js";
/**
* Splits outbound channel text into chunks no longer than the requested limit.
* Newline boundaries win over spaces; text without usable separators falls back