mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(e2e): preserve Unicode in Telegram proof tails (#109940)
Punchcard-Session: ember-brook-workshop-qc Co-authored-by: wangmiao0668000666 <290215524+wangmiao0668000666@users.noreply.github.com>
This commit is contained in:
wangmiao0668000666
committed by
GitHub
GitHub
parent
226c699a23
commit
c719cbbfe9
@@ -13,10 +13,12 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { clampTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { sleep } from "../lib/sleep.mjs";
|
||||
import { resolveWindowsTaskkillPath } from "../lib/windows-taskkill.mjs";
|
||||
import { createPnpmRunnerSpawnSpec } from "../pnpm-runner.mts";
|
||||
import { readPositiveIntEnv } from "./lib/env-limits.mjs";
|
||||
import { readTextFileTail } from "./lib/text-file-utils.mjs";
|
||||
import { telegramBotApi } from "./telegram-bot-api.ts";
|
||||
|
||||
type CommandResult = {
|
||||
@@ -690,21 +692,17 @@ function shellQuote(value: string) {
|
||||
|
||||
type AppendCommandStdoutResult = { ok: true; value: string } | { ok: false; message: string };
|
||||
|
||||
function appendCommandText(current: string, chunk: Buffer): string {
|
||||
return current + chunk.toString("utf8");
|
||||
}
|
||||
|
||||
function appendCommandTextTail(current: string, chunk: Buffer, maxChars: number): string {
|
||||
const next = appendCommandText(current, chunk);
|
||||
return next.length > maxChars ? next.slice(-maxChars) : next;
|
||||
function appendCommandTextTail(current: string, chunk: string, maxChars: number): string {
|
||||
const next = current + chunk;
|
||||
return next.length > maxChars ? sliceUtf16Safe(next, -maxChars) : next;
|
||||
}
|
||||
|
||||
function appendCommandStdout(
|
||||
current: string,
|
||||
chunk: Buffer,
|
||||
chunk: string,
|
||||
maxChars = COMMAND_STDOUT_MAX_CHARS,
|
||||
): AppendCommandStdoutResult {
|
||||
const next = appendCommandText(current, chunk);
|
||||
const next = current + chunk;
|
||||
if (next.length > maxChars) {
|
||||
return { ok: false, message: `command stdout exceeded ${maxChars} characters` };
|
||||
}
|
||||
@@ -713,7 +711,7 @@ function appendCommandStdout(
|
||||
|
||||
function appendCommandStderrTail(
|
||||
current: string,
|
||||
chunk: Buffer,
|
||||
chunk: string,
|
||||
maxChars = COMMAND_STDERR_TAIL_CHARS,
|
||||
): string {
|
||||
return appendCommandTextTail(current, chunk, maxChars);
|
||||
@@ -722,9 +720,7 @@ function appendCommandStderrTail(
|
||||
function commandFailureOutput(stdout: string, stderr: string): string {
|
||||
const stdoutTail =
|
||||
stdout.length > COMMAND_FAILURE_STDOUT_TAIL_CHARS
|
||||
? `\n[stdout truncated to last ${COMMAND_FAILURE_STDOUT_TAIL_CHARS} characters]\n${stdout.slice(
|
||||
-COMMAND_FAILURE_STDOUT_TAIL_CHARS,
|
||||
)}`
|
||||
? `\n[stdout truncated to last ${COMMAND_FAILURE_STDOUT_TAIL_CHARS} characters]\n${sliceUtf16Safe(stdout, -COMMAND_FAILURE_STDOUT_TAIL_CHARS)}`
|
||||
: stdout;
|
||||
return `${stdoutTail}${stderr}`;
|
||||
}
|
||||
@@ -917,10 +913,11 @@ export function runCommand(params: {
|
||||
killTimer.unref?.();
|
||||
}, timeoutMs);
|
||||
timeout.unref?.();
|
||||
child.stdout.on("data", (chunk: Buffer) => {
|
||||
const text = chunk.toString();
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (params.outputFile) {
|
||||
fs.appendFileSync(params.outputFile, text);
|
||||
fs.appendFileSync(params.outputFile, chunk);
|
||||
stdout = appendCommandTextTail(stdout, chunk, COMMAND_FAILURE_STDOUT_TAIL_CHARS);
|
||||
} else if (params.stdio === "inherit") {
|
||||
stdout = appendCommandTextTail(stdout, chunk, COMMAND_FAILURE_STDOUT_TAIL_CHARS);
|
||||
@@ -934,17 +931,16 @@ export function runCommand(params: {
|
||||
}
|
||||
}
|
||||
if (params.stdio === "inherit") {
|
||||
process.stdout.write(text);
|
||||
process.stdout.write(chunk);
|
||||
}
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer) => {
|
||||
const text = chunk.toString();
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
if (params.outputFile) {
|
||||
fs.appendFileSync(params.outputFile, text);
|
||||
fs.appendFileSync(params.outputFile, chunk);
|
||||
}
|
||||
stderr = appendCommandStderrTail(stderr, chunk);
|
||||
if (params.stdio === "inherit") {
|
||||
process.stderr.write(text);
|
||||
process.stderr.write(chunk);
|
||||
}
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
@@ -1012,7 +1008,7 @@ function spawnLogged(command: string, args: string[], options: SpawnOptionsWitho
|
||||
child.stderr.setEncoding("utf8");
|
||||
let output = "";
|
||||
const capture = (chunk: string) => {
|
||||
output = `${output}${chunk}`.slice(-12000);
|
||||
output = sliceUtf16Safe(`${output}${chunk}`, -12000);
|
||||
};
|
||||
child.stdout.on("data", capture);
|
||||
child.stderr.on("data", capture);
|
||||
@@ -1036,7 +1032,7 @@ function waitForOutput(
|
||||
const timeout = setTimeout(() => {
|
||||
reject(
|
||||
new Error(
|
||||
`${label} did not become ready within ${resolvedTimeoutMs}ms\n${output().slice(-4000)}`,
|
||||
`${label} did not become ready within ${resolvedTimeoutMs}ms\n${sliceUtf16Safe(output(), -4000)}`,
|
||||
),
|
||||
);
|
||||
}, resolvedTimeoutMs);
|
||||
@@ -1050,7 +1046,7 @@ function waitForOutput(
|
||||
cleanup();
|
||||
reject(
|
||||
new Error(
|
||||
`${label} exited before ready with code ${code ?? "unknown"}\n${output().slice(-4000)}`,
|
||||
`${label} exited before ready with code ${code ?? "unknown"}\n${sliceUtf16Safe(output(), -4000)}`,
|
||||
),
|
||||
);
|
||||
};
|
||||
@@ -1176,25 +1172,7 @@ function waitForChildExit(child: ChildProcess) {
|
||||
}
|
||||
|
||||
export function readLogTail(logPath: string, maxBytes = LOG_READY_TAIL_BYTES): string {
|
||||
let stat: fs.Stats;
|
||||
try {
|
||||
stat = fs.statSync(logPath);
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
if (!stat.isFile() || stat.size <= 0) {
|
||||
return "";
|
||||
}
|
||||
const bytesToRead = Math.min(Math.max(1, maxBytes), stat.size);
|
||||
const buffer = Buffer.alloc(bytesToRead);
|
||||
const fd = fs.openSync(logPath, "r");
|
||||
let bytesRead;
|
||||
try {
|
||||
bytesRead = fs.readSync(fd, buffer, 0, bytesToRead, stat.size - bytesToRead);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
return buffer.subarray(0, bytesRead).toString("utf8");
|
||||
return readTextFileTail(logPath, Math.max(1, maxBytes));
|
||||
}
|
||||
|
||||
export async function waitForLog(
|
||||
@@ -1214,7 +1192,9 @@ export async function waitForLog(
|
||||
});
|
||||
}
|
||||
const text = readLogTail(logPath);
|
||||
throw new Error(`${label} did not become ready within ${timeoutMs}ms\n${text.slice(-4000)}`);
|
||||
throw new Error(
|
||||
`${label} did not become ready within ${timeoutMs}ms\n${sliceUtf16Safe(text, -4000)}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function telegram(token: string, method: string, body: JsonObject = {}) {
|
||||
|
||||
@@ -53,6 +53,24 @@ function isProcessAlive(pid: number): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
function hasLoneSurrogate(value: string): boolean {
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const codeUnit = value.charCodeAt(index);
|
||||
if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) {
|
||||
const next = value.charCodeAt(index + 1);
|
||||
if (next >= 0xdc00 && next <= 0xdfff) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function writeExecutable(pathname: string, content: string): void {
|
||||
fs.writeFileSync(pathname, content, { mode: 0o755 });
|
||||
}
|
||||
@@ -631,6 +649,34 @@ describe("telegram user Crabbox proof log polling", () => {
|
||||
expect(tail).not.toContain("old\nold\nold\nold\nold\nold\nold\nold\nold");
|
||||
});
|
||||
|
||||
it("keeps byte-cut log tails UTF-8 safe and reads at least one byte", () => {
|
||||
const logPath = path.join(makeTempDir(tempDirs, "openclaw-telegram-proof-"), "gateway.log");
|
||||
fs.writeFileSync(
|
||||
logPath,
|
||||
Buffer.concat([Buffer.from("x".repeat(100)), Buffer.from("😀"), Buffer.from("y".repeat(20))]),
|
||||
);
|
||||
|
||||
expect(readLogTail(logPath, 23)).toBe("y".repeat(20));
|
||||
expect(readLogTail(logPath, 24)).toBe(`😀${"y".repeat(20)}`);
|
||||
expect(readLogTail(logPath, 0)).toBe("y");
|
||||
});
|
||||
|
||||
it("keeps readiness timeout tails free of split surrogate pairs", async () => {
|
||||
const logPath = path.join(makeTempDir(tempDirs, "openclaw-telegram-proof-"), "gateway.log");
|
||||
fs.writeFileSync(logPath, `${"a".repeat(9)}😀${"b".repeat(3999)}`, "utf8");
|
||||
|
||||
let message = "";
|
||||
try {
|
||||
await waitForLog(logPath, /\[gateway\] ready/u, "gateway", 0);
|
||||
} catch (error) {
|
||||
message = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
const tail = message.split("\n").at(-1) ?? "";
|
||||
expect(tail).toBe("b".repeat(3999));
|
||||
expect(hasLoneSurrogate(tail)).toBe(false);
|
||||
});
|
||||
|
||||
it("honors short reads when a log shrinks during tailing", () => {
|
||||
vi.spyOn(fs, "statSync").mockReturnValue({
|
||||
isFile: () => true,
|
||||
@@ -804,6 +850,68 @@ fs.writeFileSync(process.env.OPENCLAW_TEST_ARGV_PATH, JSON.stringify(process.arg
|
||||
setTimeoutSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("keeps command failure tails free of split surrogate pairs", async () => {
|
||||
const root = makeTempDir(tempDirs, "openclaw-telegram-proof-");
|
||||
const scriptPath = path.join(root, "unicode-failure.mjs");
|
||||
fs.writeFileSync(
|
||||
scriptPath,
|
||||
`
|
||||
await new Promise((resolve) => {
|
||||
process.stdout.write("a".repeat(3) + "😀" + "b".repeat(65_535), resolve);
|
||||
});
|
||||
await new Promise((resolve) => {
|
||||
process.stderr.write("😀" + "c".repeat(262_143), resolve);
|
||||
});
|
||||
process.exitCode = 2;
|
||||
`,
|
||||
);
|
||||
let message = "";
|
||||
try {
|
||||
await runCommand({
|
||||
args: [scriptPath],
|
||||
command: process.execPath,
|
||||
cwd: root,
|
||||
});
|
||||
} catch (error) {
|
||||
message = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
const marker = "[stdout truncated to last 65536 characters]\n";
|
||||
const tail = message.split(marker).at(-1) ?? "";
|
||||
expect(message).toContain(marker);
|
||||
expect(tail.startsWith("b".repeat(100))).toBe(true);
|
||||
expect(tail.endsWith("c".repeat(100))).toBe(true);
|
||||
expect(tail).not.toContain("😀");
|
||||
expect(hasLoneSurrogate(tail)).toBe(false);
|
||||
});
|
||||
|
||||
it("decodes command output statefully across split stream chunks", async () => {
|
||||
const script = [
|
||||
'const emoji = Buffer.from("😀", "utf8");',
|
||||
"process.stdout.write(emoji.subarray(0, 2));",
|
||||
"process.stderr.write(emoji.subarray(0, 2));",
|
||||
"setTimeout(() => {",
|
||||
" process.stdout.write(emoji.subarray(2));",
|
||||
" process.stderr.write(emoji.subarray(2));",
|
||||
" process.exit(2);",
|
||||
"}, 100);",
|
||||
].join("\n");
|
||||
let message = "";
|
||||
try {
|
||||
await runCommand({
|
||||
args: ["-e", script],
|
||||
command: process.execPath,
|
||||
cwd: makeTempDir(tempDirs, "openclaw-telegram-proof-"),
|
||||
});
|
||||
} catch (error) {
|
||||
message = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
const output = message.split("failed with exit code 2\n").at(-1) ?? "";
|
||||
expect(output.match(/😀/gu)).toHaveLength(2);
|
||||
expect(output).not.toContain("�");
|
||||
});
|
||||
|
||||
posixIt("kills timed-out command process groups when the leader exits first", async () => {
|
||||
const root = makeTempDir(tempDirs, "openclaw-telegram-proof-");
|
||||
const scriptPath = path.join(root, "trap-term.mjs");
|
||||
@@ -1069,7 +1177,7 @@ setInterval(() => {}, 1000);
|
||||
}
|
||||
});
|
||||
|
||||
posixIt("cleans local SUT children when gateway startup fails", async () => {
|
||||
posixIt("keeps local SUT startup tails Unicode-safe and cleans child processes", async () => {
|
||||
const root = makeTempDir(tempDirs, "openclaw-telegram-proof-");
|
||||
const outputDir = makeTempDir(tempDirs, "openclaw-telegram-proof-");
|
||||
const mockScript = path.join(root, "scripts/e2e/mock-openai-server.mjs");
|
||||
@@ -1096,13 +1204,15 @@ setInterval(() => {}, 1000);
|
||||
writeExecutable(
|
||||
gatewayScript,
|
||||
`
|
||||
process.stderr.write("gateway startup failed\\n");
|
||||
const output = "😀" + "x".repeat(7998) + "😀" + "y".repeat(3999);
|
||||
process.stderr.write(output);
|
||||
process.exit(2);
|
||||
`,
|
||||
);
|
||||
|
||||
await expect(
|
||||
startLocalSut(
|
||||
let message = "";
|
||||
try {
|
||||
await startLocalSut(
|
||||
{
|
||||
gatewayPort: 19042,
|
||||
groupId: "group",
|
||||
@@ -1126,9 +1236,14 @@ process.exit(2);
|
||||
webhookUrlSet: false,
|
||||
}),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("gateway exited before ready");
|
||||
);
|
||||
} catch (error) {
|
||||
message = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
expect(message).toContain("gateway exited before ready");
|
||||
expect(message.endsWith("y".repeat(3999))).toBe(true);
|
||||
expect(hasLoneSurrogate(message)).toBe(false);
|
||||
await waitFor(() => fs.existsSync(mockTermPath));
|
||||
const mockPid = Number.parseInt(fs.readFileSync(mockPidPath, "utf8"), 10);
|
||||
await waitFor(() => !isProcessAlive(mockPid));
|
||||
|
||||
Reference in New Issue
Block a user