mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
refactor(zalouser): share offset-preserving text chunk ranges (#105842)
* refactor(zalouser): share text chunk ranges * chore(plugin-sdk): refresh combined API baseline
This commit is contained in:
committed by
GitHub
parent
be7241b656
commit
4cbbb86d5e
@@ -2,7 +2,7 @@
|
||||
* Tests text and Markdown chunking helpers exported by the plugin SDK.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { chunkTextForOutbound } from "./text-chunking.js";
|
||||
import { chunkTextForOutbound, chunkTextRanges } from "./text-chunking.js";
|
||||
|
||||
describe("chunkTextForOutbound", () => {
|
||||
it.each([
|
||||
@@ -28,3 +28,44 @@ describe("chunkTextForOutbound", () => {
|
||||
expect(chunkTextForOutbound(text, maxLen)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunkTextRanges", () => {
|
||||
it("returns contiguous hard ranges without dropping whitespace", () => {
|
||||
const text = "alpha beta\n\ngamma delta";
|
||||
const ranges = chunkTextRanges(text, { limit: 12, mode: "hard" });
|
||||
|
||||
expect(ranges).toEqual([
|
||||
{ start: 0, end: 12 },
|
||||
{ start: 12, end: 24 },
|
||||
]);
|
||||
expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual([
|
||||
"alpha beta\n",
|
||||
"\ngamma delta",
|
||||
]);
|
||||
});
|
||||
|
||||
it("prefers paragraph, newline, then whitespace boundaries", () => {
|
||||
const text = "a\n\nb\nc xyz";
|
||||
const ranges = chunkTextRanges(text, { limit: 8, mode: "preferred" });
|
||||
|
||||
expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual(["a\n\n", "b\nc xyz"]);
|
||||
});
|
||||
|
||||
it("falls back to hard ranges and handles empty or non-positive limits", () => {
|
||||
expect(chunkTextRanges("abcdefgh", { limit: 3, mode: "preferred" })).toEqual([
|
||||
{ start: 0, end: 3 },
|
||||
{ start: 3, end: 6 },
|
||||
{ start: 6, end: 8 },
|
||||
]);
|
||||
expect(chunkTextRanges("", { limit: 3 })).toEqual([]);
|
||||
expect(chunkTextRanges("abc", { limit: 0 })).toEqual([{ start: 0, end: 3 }]);
|
||||
});
|
||||
|
||||
it.each(["hard", "preferred"] as const)("keeps surrogate pairs intact in %s mode", (mode) => {
|
||||
expect(chunkTextRanges("a😀b", { limit: 2, mode })).toEqual([
|
||||
{ start: 0, end: 1 },
|
||||
{ start: 1, end: 3 },
|
||||
{ start: 3, end: 4 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
// Text chunking helpers split long outbound text while preserving readable line boundaries.
|
||||
import { chunkTextByBreakResolver } from "../shared/text-chunking.js";
|
||||
|
||||
/** Offset-preserving text ranges for transports with native style metadata. */
|
||||
export {
|
||||
chunkTextRanges,
|
||||
type ChunkTextRangesOptions,
|
||||
type TextChunkRange,
|
||||
} from "../../packages/markdown-core/src/chunk-text.js";
|
||||
|
||||
/**
|
||||
* Splits outbound channel text into chunks no longer than the requested limit.
|
||||
* Newline boundaries win over spaces; text without usable separators falls back
|
||||
|
||||
Reference in New Issue
Block a user