mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(terminal): measure unicode display width consistently (#117062)
Co-authored-by: Peter Steinberger <steipete@macos.shared>
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
d8ba00f8a2
commit
95e50d3a7f
@@ -103,6 +103,7 @@
|
||||
"dependencies": {
|
||||
"@clack/prompts": "1.7.0",
|
||||
"@openclaw/normalization-core": "workspace:*",
|
||||
"chalk": "6.0.0"
|
||||
"chalk": "6.0.0",
|
||||
"string-width": "8.2.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,6 +193,59 @@ describe("terminal ansi helpers", () => {
|
||||
expect(visibleWidth("a\u001B[31\u0001mb")).toBe(2);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["halfwidth voiced kana", "ﾊﾞ", 2],
|
||||
["halfwidth semi-voiced kana", "ﾊﾟ", 2],
|
||||
["halfwidth kana with a prolonged sound", "ｳﾞｰ", 3],
|
||||
["zero-width space", "\u200B", 0],
|
||||
["zero-width non-joiner", "\u200C", 0],
|
||||
["word joiner", "\u2060", 0],
|
||||
["function application", "\u2061", 0],
|
||||
["soft hyphen", "\u00AD", 0],
|
||||
["zero-width no-break space", "\uFEFF", 0],
|
||||
["Hindi spacing mark", "का", 2],
|
||||
["repeated leading Hangul jamo", "ᄀᄀ", 4],
|
||||
["repeated Hangul jamo with a vowel", "ᄀ가", 4],
|
||||
["Hangul leading filler", "\u115F", 2],
|
||||
["Hangul vowel filler", "\u1160", 0],
|
||||
["Hangul compatibility filler", "\u3164", 2],
|
||||
["halfwidth Hangul filler", "\uFFA0", 1],
|
||||
["Hangul filler with a vowel", "\u115F\u1161", 2],
|
||||
["Hangul leading and vowel fillers", "\u115F\u1160", 2],
|
||||
["Hangul compatibility filler with a combining mark", "\u3164\u0301", 2],
|
||||
["halfwidth Hangul filler with a voiced mark", "\uFFA0\uFF9E", 2],
|
||||
["lone high surrogate", "\uD800", 1],
|
||||
["lone low surrogate", "\uDC00", 1],
|
||||
["well-formed emoji surrogate pair", "\uD83D\uDE00", 2],
|
||||
])("measures %s with Unicode terminal-width rules", (_label, text, width) => {
|
||||
expect(visibleWidth(text)).toBe(width);
|
||||
});
|
||||
|
||||
it("measures malformed UTF-16 by Node's rendered replacement cells", () => {
|
||||
for (const input of ["\uD800", "\uDC00", "表\uD800\t", "\u001B[31m\uDC00\u001B[0m"]) {
|
||||
const rendered = Buffer.from(input, "utf8").toString("utf8");
|
||||
expect(visibleWidth(input)).toBe(visibleWidth(rendered));
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves the executed width of tabs", () => {
|
||||
expect(visibleWidth("\t")).toBe(1);
|
||||
expect(visibleWidth("a\tb")).toBe(3);
|
||||
expect(visibleWidth("a\u001B[31\tmb")).toBe(3);
|
||||
expect(visibleWidth("a\u009B31\tmb")).toBe(3);
|
||||
});
|
||||
|
||||
it("keeps malformed ANSI ownership with OpenClaw's canonical scanner", () => {
|
||||
expect(visibleWidth("\u001B")).toBe(0);
|
||||
expect(visibleWidth("\u009B")).toBe(0);
|
||||
expect(visibleWidth("\u009D")).toBe(0);
|
||||
expect(visibleWidth("\u001B[")).toBe(0);
|
||||
expect(visibleWidth("\u001B]visible")).toBe("]visible".length);
|
||||
expect(visibleWidth("\u009Dvisible")).toBe("visible".length);
|
||||
// Removing CSI must not let another ANSI grammar reinterpret joined bytes as OSC.
|
||||
expect(visibleWidth("\u001B\u001B[0m]visible\u0007after")).toBe("]visibleafter".length);
|
||||
});
|
||||
|
||||
it("keeps emoji zwj sequences as single graphemes", () => {
|
||||
expect(splitGraphemes("👨👩👧👦")).toEqual(["👨👩👧👦"]);
|
||||
expect(visibleWidth("👨👩👧👦")).toBe(2);
|
||||
@@ -229,9 +282,32 @@ describe("terminal ansi helpers", () => {
|
||||
// never emitted half-width, so the result never exceeds the budget.
|
||||
expect(truncateToVisibleWidth("表文", 2)).toBe("表");
|
||||
expect(truncateToVisibleWidth("表", 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("ﾊﾞ", 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("ﾊﾞ", 2)).toBe("ﾊﾞ");
|
||||
expect(truncateToVisibleWidth("का", 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("ᄀᄀ", 3)).toBe("");
|
||||
expect(truncateToVisibleWidth("👨👩👧👦", 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("🇬🇧", 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("\u200B表", 1)).toBe("\u200B");
|
||||
expect(truncateToVisibleWidth("a\tb", 2)).toBe("a\t");
|
||||
expect(truncateToVisibleWidth("\u115F\u1161", 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("\u3164".repeat(30), 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("\u3164".repeat(30), 2)).toBe("\u3164");
|
||||
expect(truncateToVisibleWidth("\uFFA0\uFF9E", 1)).toBe("");
|
||||
expect(truncateToVisibleWidth("\uD800".repeat(30), 5)).toBe("\uD800".repeat(5));
|
||||
expect(truncateToVisibleWidth("表\uDC00\t", 3)).toBe("表\uDC00");
|
||||
expect(visibleWidth(truncateToVisibleWidth("📸📸", 1))).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("keeps complete graphemes and zero-width prefixes at either truncation edge", () => {
|
||||
expect(truncateToVisibleWidth("abcd", 3)).toBe("abc");
|
||||
expect(truncateToVisibleWidth("表文字", 5)).toBe("表文");
|
||||
expect(truncateToVisibleWidth("👨👩👧👦📸", 3)).toBe("👨👩👧👦");
|
||||
expect(truncateToVisibleWidth("a\u200Bb", 1)).toBe("a\u200B");
|
||||
expect(truncateToVisibleWidth("abc\u200B", 2)).toBe("ab");
|
||||
expect(truncateToVisibleWidth("\u200Babc", 2)).toBe("\u200Bab");
|
||||
});
|
||||
|
||||
it("preserves ANSI sequences when truncating styled text", () => {
|
||||
// Trailing reset is retained even when its grapheme is dropped, so the cell
|
||||
// does not bleed styling into surrounding padding.
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import stringWidth from "string-width";
|
||||
import {
|
||||
ANSI_COMPAT_CONTROL_SEQUENCE_PATTERN,
|
||||
ANSI_OSC_INTRODUCER_PATTERN,
|
||||
@@ -179,102 +180,27 @@ export function sanitizeForLog(v: string): string {
|
||||
return stripAnsi(v).replace(controlCharsRegex, "");
|
||||
}
|
||||
|
||||
function isZeroWidthCodePoint(codePoint: number): boolean {
|
||||
return (
|
||||
(codePoint <= 0x1f && codePoint !== 0x09) ||
|
||||
(codePoint >= 0x7f && codePoint <= 0x9f) ||
|
||||
(codePoint >= 0x0300 && codePoint <= 0x036f) ||
|
||||
(codePoint >= 0x1ab0 && codePoint <= 0x1aff) ||
|
||||
(codePoint >= 0x1dc0 && codePoint <= 0x1dff) ||
|
||||
(codePoint >= 0x20d0 && codePoint <= 0x20ff) ||
|
||||
(codePoint >= 0xfe20 && codePoint <= 0xfe2f) ||
|
||||
(codePoint >= 0xfe00 && codePoint <= 0xfe0f) ||
|
||||
codePoint === 0x200d
|
||||
);
|
||||
}
|
||||
|
||||
function isFullWidthCodePoint(codePoint: number): boolean {
|
||||
if (codePoint < 0x1100) {
|
||||
return false;
|
||||
function textWidth(text: string): number {
|
||||
// POSIX renders these default-ignorable Hangul fillers as wide/halfwidth cells;
|
||||
// same-shaping representatives and well-formed surrogates preserve terminal output.
|
||||
const printable = /[\u115F\u3164\uFFA0\uD800-\uDFFF]/u.test(text)
|
||||
? text
|
||||
.replace(/[\uD800-\uDFFF]/gu, "\uFFFD")
|
||||
.replaceAll("\u115F", "\u1100")
|
||||
.replaceAll("\u3164", "\u3131")
|
||||
.replaceAll("\uFFA0", "\uFF8A")
|
||||
: text;
|
||||
// OpenClaw owns ANSI parsing; upstream must not reinterpret malformed sequences.
|
||||
let width = stringWidth(printable, { countAnsiEscapeCodes: true });
|
||||
// Tabs execute inside CSI too; string-width intentionally treats them as zero-width.
|
||||
for (let index = text.indexOf("\t"); index !== -1; index = text.indexOf("\t", index + 1)) {
|
||||
width += 1;
|
||||
}
|
||||
return (
|
||||
codePoint <= 0x115f ||
|
||||
codePoint === 0x2329 ||
|
||||
codePoint === 0x232a ||
|
||||
(codePoint >= 0x2e80 && codePoint <= 0x3247 && codePoint !== 0x303f) ||
|
||||
(codePoint >= 0x3250 && codePoint <= 0x4dbf) ||
|
||||
(codePoint >= 0x4e00 && codePoint <= 0xa4c6) ||
|
||||
(codePoint >= 0xa960 && codePoint <= 0xa97c) ||
|
||||
(codePoint >= 0xac00 && codePoint <= 0xd7a3) ||
|
||||
(codePoint >= 0xf900 && codePoint <= 0xfaff) ||
|
||||
(codePoint >= 0xfe10 && codePoint <= 0xfe19) ||
|
||||
(codePoint >= 0xfe30 && codePoint <= 0xfe6b) ||
|
||||
(codePoint >= 0xff01 && codePoint <= 0xff60) ||
|
||||
(codePoint >= 0xffe0 && codePoint <= 0xffe6) ||
|
||||
(codePoint >= 0x1aff0 && codePoint <= 0x1aff3) ||
|
||||
(codePoint >= 0x1aff5 && codePoint <= 0x1affb) ||
|
||||
(codePoint >= 0x1affd && codePoint <= 0x1affe) ||
|
||||
(codePoint >= 0x1b000 && codePoint <= 0x1b2ff) ||
|
||||
(codePoint >= 0x1f200 && codePoint <= 0x1f251) ||
|
||||
(codePoint >= 0x20000 && codePoint <= 0x3fffd)
|
||||
);
|
||||
}
|
||||
|
||||
const rgiEmojiPattern = new RegExp("^\\p{RGI_Emoji}$", "v");
|
||||
const emojiPresentationPattern = /\p{Emoji_Presentation}/u;
|
||||
const regionalIndicatorPattern = /\p{Regional_Indicator}/u;
|
||||
const unqualifiedKeycapPattern = /^[#*0-9]\u20E3$/u;
|
||||
const extendedPictographicPattern = /\p{Extended_Pictographic}/gu;
|
||||
|
||||
function isWideEmojiGrapheme(grapheme: string): boolean {
|
||||
const isRgiEmoji = rgiEmojiPattern.test(grapheme);
|
||||
// RGI recognizes paired flags while keeping a lone regional indicator narrow.
|
||||
if (regionalIndicatorPattern.test(grapheme)) {
|
||||
return isRgiEmoji;
|
||||
}
|
||||
if (
|
||||
emojiPresentationPattern.test(grapheme) ||
|
||||
isRgiEmoji ||
|
||||
unqualifiedKeycapPattern.test(grapheme)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
// Minimally qualified ZWJ sequences still shape as one wide emoji in terminals.
|
||||
return (
|
||||
grapheme.includes("\u200D") && (grapheme.match(extendedPictographicPattern)?.length ?? 0) >= 2
|
||||
);
|
||||
}
|
||||
|
||||
function graphemeWidth(grapheme: string): number {
|
||||
if (!grapheme) {
|
||||
return 0;
|
||||
}
|
||||
if (isWideEmojiGrapheme(grapheme)) {
|
||||
return 2;
|
||||
}
|
||||
|
||||
let sawPrintable = false;
|
||||
for (const char of grapheme) {
|
||||
const codePoint = char.codePointAt(0);
|
||||
if (codePoint == null) {
|
||||
continue;
|
||||
}
|
||||
if (isZeroWidthCodePoint(codePoint)) {
|
||||
continue;
|
||||
}
|
||||
if (isFullWidthCodePoint(codePoint)) {
|
||||
return 2;
|
||||
}
|
||||
sawPrintable = true;
|
||||
}
|
||||
return sawPrintable ? 1 : 0;
|
||||
return width;
|
||||
}
|
||||
|
||||
export function visibleWidth(input: string): number {
|
||||
return splitGraphemes(stripAnsi(input)).reduce(
|
||||
(sum, grapheme) => sum + graphemeWidth(grapheme),
|
||||
0,
|
||||
);
|
||||
return textWidth(stripAnsi(input));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -288,7 +214,9 @@ export function truncateToVisibleWidth(input: string, maxWidth: number): string
|
||||
if (maxWidth <= 0) {
|
||||
return "";
|
||||
}
|
||||
if (visibleWidth(input) <= maxWidth) {
|
||||
const plainInput = stripAnsi(input);
|
||||
const inputWidth = textWidth(plainInput);
|
||||
if (inputWidth <= maxWidth) {
|
||||
return input;
|
||||
}
|
||||
let out = "";
|
||||
@@ -301,20 +229,89 @@ export function truncateToVisibleWidth(input: string, maxWidth: number): string
|
||||
if (budgetSpent) {
|
||||
return;
|
||||
}
|
||||
for (const grapheme of splitGraphemes(segment)) {
|
||||
const width = graphemeWidth(grapheme);
|
||||
if (used + width > maxWidth) {
|
||||
budgetSpent = true;
|
||||
return;
|
||||
}
|
||||
out += grapheme;
|
||||
const remaining = maxWidth - used;
|
||||
const width = segment === plainInput ? inputWidth : textWidth(segment);
|
||||
if (width <= remaining) {
|
||||
out += segment;
|
||||
used += width;
|
||||
return;
|
||||
}
|
||||
|
||||
const graphemes = splitGraphemes(segment);
|
||||
let offset = 0;
|
||||
const offsets = [offset];
|
||||
for (const grapheme of graphemes) {
|
||||
offset += grapheme.length;
|
||||
offsets.push(offset);
|
||||
}
|
||||
let start = 0;
|
||||
let fittedWidth = 0;
|
||||
if (remaining <= width / 2) {
|
||||
let end = Math.max(
|
||||
1,
|
||||
Math.min(graphemes.length - 1, Math.floor((remaining * graphemes.length) / width)),
|
||||
);
|
||||
let stride = 1;
|
||||
// Estimate the cell boundary first; gallop handles uneven/zero-width clusters.
|
||||
while (end < graphemes.length) {
|
||||
const candidateWidth = textWidth(segment.slice(0, offsets[end]));
|
||||
if (candidateWidth > remaining) {
|
||||
break;
|
||||
}
|
||||
start = end;
|
||||
fittedWidth = candidateWidth;
|
||||
end = Math.min(graphemes.length, end + stride);
|
||||
stride *= 2;
|
||||
}
|
||||
while (start + 1 < end) {
|
||||
const middle = Math.floor((start + end) / 2);
|
||||
const candidateWidth = textWidth(segment.slice(0, offsets[middle]));
|
||||
if (candidateWidth <= remaining) {
|
||||
start = middle;
|
||||
fittedWidth = candidateWidth;
|
||||
} else {
|
||||
end = middle;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const overflow = width - remaining;
|
||||
let tooShort = 0;
|
||||
let removed = Math.min(graphemes.length, 1);
|
||||
let removedWidth = width;
|
||||
// Near-end cuts search short complete-grapheme suffixes, not repeated full prefixes.
|
||||
while (removed < graphemes.length) {
|
||||
removedWidth = textWidth(segment.slice(offsets[graphemes.length - removed]));
|
||||
if (removedWidth >= overflow) {
|
||||
break;
|
||||
}
|
||||
tooShort = removed;
|
||||
removed = Math.min(graphemes.length, removed * 2);
|
||||
}
|
||||
if (removed === graphemes.length) {
|
||||
removedWidth = width;
|
||||
}
|
||||
while (tooShort + 1 < removed) {
|
||||
const middle = Math.floor((tooShort + removed) / 2);
|
||||
const candidateWidth = textWidth(segment.slice(offsets[graphemes.length - middle]));
|
||||
if (candidateWidth >= overflow) {
|
||||
removed = middle;
|
||||
removedWidth = candidateWidth;
|
||||
} else {
|
||||
tooShort = middle;
|
||||
}
|
||||
}
|
||||
start = graphemes.length - removed;
|
||||
fittedWidth = width - removedWidth;
|
||||
}
|
||||
out += segment.slice(0, offsets[start]);
|
||||
used += fittedWidth;
|
||||
budgetSpent = true;
|
||||
};
|
||||
for (const segment of splitAnsiSegments(input)) {
|
||||
if (segment.kind === "ansi") {
|
||||
const widthControls = segment.controls.filter((control) => graphemeWidth(control) > 0);
|
||||
const controlWidth = widthControls.reduce((sum, control) => sum + graphemeWidth(control), 0);
|
||||
// CSI retains only C0/DEL controls; TAB is the sole visible-width member.
|
||||
const widthControls = segment.controls.filter((control) => control === "\t");
|
||||
const controlWidth = widthControls.length;
|
||||
if (!budgetSpent && used + controlWidth <= maxWidth) {
|
||||
out += segment.value;
|
||||
used += controlWidth;
|
||||
|
||||
Generated
+3
@@ -2331,6 +2331,9 @@ importers:
|
||||
chalk:
|
||||
specifier: 6.0.0
|
||||
version: 6.0.0
|
||||
string-width:
|
||||
specifier: 8.2.2
|
||||
version: 8.2.2
|
||||
|
||||
packages/tool-call-repair:
|
||||
dependencies:
|
||||
|
||||
@@ -129,6 +129,33 @@ describe("printCronList", () => {
|
||||
expect(output).not.toContain(injectedMarker);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["halfwidth voiced kana", `${"x".repeat(23)}ﾊﾞ`, `${"x".repeat(21)}...`],
|
||||
["halfwidth semi-voiced kana", `${"x".repeat(23)}ﾊﾟ`, `${"x".repeat(21)}...`],
|
||||
["zero-width space", `${"x".repeat(23)}\u200B`, `${"x".repeat(23)}\u200B `],
|
||||
["word joiner", `${"x".repeat(23)}\u2060`, `${"x".repeat(23)}\u2060 `],
|
||||
["zero-width no-break space", `${"x".repeat(23)}\uFEFF`, `${"x".repeat(23)}\uFEFF `],
|
||||
["leading zero-width non-joiner", `\u200C${"x".repeat(23)}`, `\u200C${"x".repeat(23)} `],
|
||||
["Hindi spacing mark", `${"x".repeat(23)}का`, `${"x".repeat(21)}...`],
|
||||
["repeated Hangul jamo", `${"x".repeat(22)}ᄀ가`, `${"x".repeat(21)}...`],
|
||||
["Hangul leading filler", `${"x".repeat(23)}\u115F`, `${"x".repeat(21)}...`],
|
||||
["Hangul compatibility filler", `${"x".repeat(23)}\u3164`, `${"x".repeat(21)}...`],
|
||||
["halfwidth Hangul filler", `${"x".repeat(23)}\uFFA0`, `${"x".repeat(23)}\uFFA0`],
|
||||
["zero-width Hangul vowel filler", `${"x".repeat(23)}\u1160`, `${"x".repeat(23)}\u1160 `],
|
||||
["lone high surrogate", `${"x".repeat(23)}\uD800`, `${"x".repeat(23)}\uD800`],
|
||||
["lone low surrogate", `${"x".repeat(23)}\uDC00`, `${"x".repeat(23)}\uDC00`],
|
||||
])("aligns the %s name cell without relying on its width helper", (_label, name, expected) => {
|
||||
const { logs, runtime } = createRuntimeLogCapture();
|
||||
printCronList([createBaseJob({ name })], runtime);
|
||||
|
||||
const [header = "", row = ""] = logs;
|
||||
const nameColumn = header.indexOf("Name");
|
||||
const scheduleColumn = row.indexOf("at ");
|
||||
expect(nameColumn).toBeGreaterThan(-1);
|
||||
expect(scheduleColumn).toBeGreaterThan(nameColumn);
|
||||
expect(row.slice(nameColumn, scheduleColumn - 1)).toBe(expected);
|
||||
});
|
||||
|
||||
it("sanitizes and bounds named-session targets", () => {
|
||||
const { logs, runtime } = createRuntimeLogCapture();
|
||||
const injectedMarker = "cron-target-injection";
|
||||
|
||||
Reference in New Issue
Block a user