diff --git a/scripts/e2e/telegram-user-crabbox-proof.ts b/scripts/e2e/telegram-user-crabbox-proof.ts index 6129fbb1cdc2..f305940bd197 100644 --- a/scripts/e2e/telegram-user-crabbox-proof.ts +++ b/scripts/e2e/telegram-user-crabbox-proof.ts @@ -13,10 +13,12 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { clampTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion"; +import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { sleep } from "../lib/sleep.mjs"; import { resolveWindowsTaskkillPath } from "../lib/windows-taskkill.mjs"; import { createPnpmRunnerSpawnSpec } from "../pnpm-runner.mts"; import { readPositiveIntEnv } from "./lib/env-limits.mjs"; +import { readTextFileTail } from "./lib/text-file-utils.mjs"; import { telegramBotApi } from "./telegram-bot-api.ts"; type CommandResult = { @@ -690,21 +692,17 @@ function shellQuote(value: string) { type AppendCommandStdoutResult = { ok: true; value: string } | { ok: false; message: string }; -function appendCommandText(current: string, chunk: Buffer): string { - return current + chunk.toString("utf8"); -} - -function appendCommandTextTail(current: string, chunk: Buffer, maxChars: number): string { - const next = appendCommandText(current, chunk); - return next.length > maxChars ? next.slice(-maxChars) : next; +function appendCommandTextTail(current: string, chunk: string, maxChars: number): string { + const next = current + chunk; + return next.length > maxChars ? sliceUtf16Safe(next, -maxChars) : next; } function appendCommandStdout( current: string, - chunk: Buffer, + chunk: string, maxChars = COMMAND_STDOUT_MAX_CHARS, ): AppendCommandStdoutResult { - const next = appendCommandText(current, chunk); + const next = current + chunk; if (next.length > maxChars) { return { ok: false, message: `command stdout exceeded ${maxChars} characters` }; } @@ -713,7 +711,7 @@ function appendCommandStdout( function appendCommandStderrTail( current: string, - chunk: Buffer, + chunk: string, maxChars = COMMAND_STDERR_TAIL_CHARS, ): string { return appendCommandTextTail(current, chunk, maxChars); @@ -722,9 +720,7 @@ function appendCommandStderrTail( function commandFailureOutput(stdout: string, stderr: string): string { const stdoutTail = stdout.length > COMMAND_FAILURE_STDOUT_TAIL_CHARS - ? `\n[stdout truncated to last ${COMMAND_FAILURE_STDOUT_TAIL_CHARS} characters]\n${stdout.slice( - -COMMAND_FAILURE_STDOUT_TAIL_CHARS, - )}` + ? `\n[stdout truncated to last ${COMMAND_FAILURE_STDOUT_TAIL_CHARS} characters]\n${sliceUtf16Safe(stdout, -COMMAND_FAILURE_STDOUT_TAIL_CHARS)}` : stdout; return `${stdoutTail}${stderr}`; } @@ -917,10 +913,11 @@ export function runCommand(params: { killTimer.unref?.(); }, timeoutMs); timeout.unref?.(); - child.stdout.on("data", (chunk: Buffer) => { - const text = chunk.toString(); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { if (params.outputFile) { - fs.appendFileSync(params.outputFile, text); + fs.appendFileSync(params.outputFile, chunk); stdout = appendCommandTextTail(stdout, chunk, COMMAND_FAILURE_STDOUT_TAIL_CHARS); } else if (params.stdio === "inherit") { stdout = appendCommandTextTail(stdout, chunk, COMMAND_FAILURE_STDOUT_TAIL_CHARS); @@ -934,17 +931,16 @@ export function runCommand(params: { } } if (params.stdio === "inherit") { - process.stdout.write(text); + process.stdout.write(chunk); } }); - child.stderr.on("data", (chunk: Buffer) => { - const text = chunk.toString(); + child.stderr.on("data", (chunk: string) => { if (params.outputFile) { - fs.appendFileSync(params.outputFile, text); + fs.appendFileSync(params.outputFile, chunk); } stderr = appendCommandStderrTail(stderr, chunk); if (params.stdio === "inherit") { - process.stderr.write(text); + process.stderr.write(chunk); } }); child.on("error", (error) => { @@ -1012,7 +1008,7 @@ function spawnLogged(command: string, args: string[], options: SpawnOptionsWitho child.stderr.setEncoding("utf8"); let output = ""; const capture = (chunk: string) => { - output = `${output}${chunk}`.slice(-12000); + output = sliceUtf16Safe(`${output}${chunk}`, -12000); }; child.stdout.on("data", capture); child.stderr.on("data", capture); @@ -1036,7 +1032,7 @@ function waitForOutput( const timeout = setTimeout(() => { reject( new Error( - `${label} did not become ready within ${resolvedTimeoutMs}ms\n${output().slice(-4000)}`, + `${label} did not become ready within ${resolvedTimeoutMs}ms\n${sliceUtf16Safe(output(), -4000)}`, ), ); }, resolvedTimeoutMs); @@ -1050,7 +1046,7 @@ function waitForOutput( cleanup(); reject( new Error( - `${label} exited before ready with code ${code ?? "unknown"}\n${output().slice(-4000)}`, + `${label} exited before ready with code ${code ?? "unknown"}\n${sliceUtf16Safe(output(), -4000)}`, ), ); }; @@ -1176,25 +1172,7 @@ function waitForChildExit(child: ChildProcess) { } export function readLogTail(logPath: string, maxBytes = LOG_READY_TAIL_BYTES): string { - let stat: fs.Stats; - try { - stat = fs.statSync(logPath); - } catch { - return ""; - } - if (!stat.isFile() || stat.size <= 0) { - return ""; - } - const bytesToRead = Math.min(Math.max(1, maxBytes), stat.size); - const buffer = Buffer.alloc(bytesToRead); - const fd = fs.openSync(logPath, "r"); - let bytesRead; - try { - bytesRead = fs.readSync(fd, buffer, 0, bytesToRead, stat.size - bytesToRead); - } finally { - fs.closeSync(fd); - } - return buffer.subarray(0, bytesRead).toString("utf8"); + return readTextFileTail(logPath, Math.max(1, maxBytes)); } export async function waitForLog( @@ -1214,7 +1192,9 @@ export async function waitForLog( }); } const text = readLogTail(logPath); - throw new Error(`${label} did not become ready within ${timeoutMs}ms\n${text.slice(-4000)}`); + throw new Error( + `${label} did not become ready within ${timeoutMs}ms\n${sliceUtf16Safe(text, -4000)}`, + ); } async function telegram(token: string, method: string, body: JsonObject = {}) { diff --git a/test/scripts/telegram-user-crabbox-proof.test.ts b/test/scripts/telegram-user-crabbox-proof.test.ts index 209dd27b6a1e..755351ad9f93 100644 --- a/test/scripts/telegram-user-crabbox-proof.test.ts +++ b/test/scripts/telegram-user-crabbox-proof.test.ts @@ -53,6 +53,24 @@ function isProcessAlive(pid: number): boolean { } } +function hasLoneSurrogate(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const codeUnit = value.charCodeAt(index); + if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (next >= 0xdc00 && next <= 0xdfff) { + index += 1; + continue; + } + return true; + } + if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) { + return true; + } + } + return false; +} + function writeExecutable(pathname: string, content: string): void { fs.writeFileSync(pathname, content, { mode: 0o755 }); } @@ -631,6 +649,34 @@ describe("telegram user Crabbox proof log polling", () => { expect(tail).not.toContain("old\nold\nold\nold\nold\nold\nold\nold\nold"); }); + it("keeps byte-cut log tails UTF-8 safe and reads at least one byte", () => { + const logPath = path.join(makeTempDir(tempDirs, "openclaw-telegram-proof-"), "gateway.log"); + fs.writeFileSync( + logPath, + Buffer.concat([Buffer.from("x".repeat(100)), Buffer.from("😀"), Buffer.from("y".repeat(20))]), + ); + + expect(readLogTail(logPath, 23)).toBe("y".repeat(20)); + expect(readLogTail(logPath, 24)).toBe(`😀${"y".repeat(20)}`); + expect(readLogTail(logPath, 0)).toBe("y"); + }); + + it("keeps readiness timeout tails free of split surrogate pairs", async () => { + const logPath = path.join(makeTempDir(tempDirs, "openclaw-telegram-proof-"), "gateway.log"); + fs.writeFileSync(logPath, `${"a".repeat(9)}😀${"b".repeat(3999)}`, "utf8"); + + let message = ""; + try { + await waitForLog(logPath, /\[gateway\] ready/u, "gateway", 0); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + + const tail = message.split("\n").at(-1) ?? ""; + expect(tail).toBe("b".repeat(3999)); + expect(hasLoneSurrogate(tail)).toBe(false); + }); + it("honors short reads when a log shrinks during tailing", () => { vi.spyOn(fs, "statSync").mockReturnValue({ isFile: () => true, @@ -804,6 +850,68 @@ fs.writeFileSync(process.env.OPENCLAW_TEST_ARGV_PATH, JSON.stringify(process.arg setTimeoutSpy.mockRestore(); }); + it("keeps command failure tails free of split surrogate pairs", async () => { + const root = makeTempDir(tempDirs, "openclaw-telegram-proof-"); + const scriptPath = path.join(root, "unicode-failure.mjs"); + fs.writeFileSync( + scriptPath, + ` +await new Promise((resolve) => { + process.stdout.write("a".repeat(3) + "😀" + "b".repeat(65_535), resolve); +}); +await new Promise((resolve) => { + process.stderr.write("😀" + "c".repeat(262_143), resolve); +}); +process.exitCode = 2; +`, + ); + let message = ""; + try { + await runCommand({ + args: [scriptPath], + command: process.execPath, + cwd: root, + }); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + + const marker = "[stdout truncated to last 65536 characters]\n"; + const tail = message.split(marker).at(-1) ?? ""; + expect(message).toContain(marker); + expect(tail.startsWith("b".repeat(100))).toBe(true); + expect(tail.endsWith("c".repeat(100))).toBe(true); + expect(tail).not.toContain("😀"); + expect(hasLoneSurrogate(tail)).toBe(false); + }); + + it("decodes command output statefully across split stream chunks", async () => { + const script = [ + 'const emoji = Buffer.from("😀", "utf8");', + "process.stdout.write(emoji.subarray(0, 2));", + "process.stderr.write(emoji.subarray(0, 2));", + "setTimeout(() => {", + " process.stdout.write(emoji.subarray(2));", + " process.stderr.write(emoji.subarray(2));", + " process.exit(2);", + "}, 100);", + ].join("\n"); + let message = ""; + try { + await runCommand({ + args: ["-e", script], + command: process.execPath, + cwd: makeTempDir(tempDirs, "openclaw-telegram-proof-"), + }); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + + const output = message.split("failed with exit code 2\n").at(-1) ?? ""; + expect(output.match(/😀/gu)).toHaveLength(2); + expect(output).not.toContain("�"); + }); + posixIt("kills timed-out command process groups when the leader exits first", async () => { const root = makeTempDir(tempDirs, "openclaw-telegram-proof-"); const scriptPath = path.join(root, "trap-term.mjs"); @@ -1069,7 +1177,7 @@ setInterval(() => {}, 1000); } }); - posixIt("cleans local SUT children when gateway startup fails", async () => { + posixIt("keeps local SUT startup tails Unicode-safe and cleans child processes", async () => { const root = makeTempDir(tempDirs, "openclaw-telegram-proof-"); const outputDir = makeTempDir(tempDirs, "openclaw-telegram-proof-"); const mockScript = path.join(root, "scripts/e2e/mock-openai-server.mjs"); @@ -1096,13 +1204,15 @@ setInterval(() => {}, 1000); writeExecutable( gatewayScript, ` -process.stderr.write("gateway startup failed\\n"); +const output = "😀" + "x".repeat(7998) + "😀" + "y".repeat(3999); +process.stderr.write(output); process.exit(2); `, ); - await expect( - startLocalSut( + let message = ""; + try { + await startLocalSut( { gatewayPort: 19042, groupId: "group", @@ -1126,9 +1236,14 @@ process.exit(2); webhookUrlSet: false, }), }, - ), - ).rejects.toThrow("gateway exited before ready"); + ); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + expect(message).toContain("gateway exited before ready"); + expect(message.endsWith("y".repeat(3999))).toBe(true); + expect(hasLoneSurrogate(message)).toBe(false); await waitFor(() => fs.existsSync(mockTermPath)); const mockPid = Number.parseInt(fs.readFileSync(mockPidPath, "utf8"), 10); await waitFor(() => !isProcessAlive(mockPid));