diff --git a/CHANGELOG.md b/CHANGELOG.md index f7ff8491f870..4d154c11e9b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,8 @@ Docs: https://docs.openclaw.ai - Release/CI/E2E: bound mock OpenAI readiness probes in web-search and Telegram RTT Docker smokes so stalled HTTP accepts cannot hang cleanup or fall through. - Tooling: cancel oversized pnpm audit advisory responses before failing so registry error paths do not leave response bodies open. - Release/CI/E2E: stop tracked gateway and mock service process groups so descendant helpers do not survive E2E cleanup. +- Release/CI/E2E: exit Telegram credential proof wrappers promptly after forwarded shutdown signals while keeping the descendant force-kill guard armed. +- Release/CI/E2E: reject oversized ClickClack fixture request bodies before release journey smokes can accumulate unbounded payloads. - Release/CI/E2E: fail secret-provider proof runs when temporary state cleanup still fails after retries instead of hiding the cleanup error. - Release/CI/E2E: fail package-candidate ref proofs when temporary source worktree cleanup fails instead of leaving stale worktrees behind. - Release/CI/E2E: remove package tarball extract directories when tar extraction fails before validation can continue. diff --git a/extensions/codex/src/app-server/attempt-startup.test.ts b/extensions/codex/src/app-server/attempt-startup.test.ts index 6f9967b66fa3..8245f74a58b0 100644 --- a/extensions/codex/src/app-server/attempt-startup.test.ts +++ b/extensions/codex/src/app-server/attempt-startup.test.ts @@ -48,6 +48,8 @@ const bundleMcpThreadConfig = { fingerprint: undefined, } satisfies CodexBundleMcpThreadConfig; +const HARNESS_REQUEST_TIMEOUT_MS = 15_000; + function readHarnessMessages(writes: string[]): Array<{ id?: number; method?: string }> { return writes.map((write) => JSON.parse(write) as { id?: number; method?: string }); } @@ -105,7 +107,7 @@ function startThreadWithHarness( async function answerInitialize(harness: ClientHarness): Promise { await vi.waitFor(() => expect(harness.writes.length).toBeGreaterThanOrEqual(1), { interval: 1, - timeout: 5_000, + timeout: HARNESS_REQUEST_TIMEOUT_MS, }); const initialize = JSON.parse(harness.writes[0] ?? "{}") as { id?: number }; harness.send({ id: initialize.id, result: { userAgent: "openclaw/0.125.0 (macOS; test)" } }); @@ -120,7 +122,7 @@ async function waitForRequest( expect(readHarnessMessages(harness.writes).some((write) => write.method === method)).toBe( true, ), - { interval: 1, timeout: 5_000 }, + { interval: 1, timeout: HARNESS_REQUEST_TIMEOUT_MS }, ); const request = readHarnessMessages(harness.writes).find((write) => write.method === method); if (!request) { @@ -238,9 +240,10 @@ describe("startCodexAttemptThread", () => { harness: retained, skipStartSpy: true, }); + const rejected = expect(run).rejects.toThrow("codex app-server startup timed out"); const threadStart = await waitForThreadStart(retained); - await expect(run).rejects.toThrow("codex app-server startup timed out"); + await rejected; expect(retained.process.stdin.destroyed).toBe(false); retained.send({ id: threadStart.id, result: { threadId: "late-thread" } }); @@ -249,11 +252,18 @@ describe("startCodexAttemptThread", () => { }); it("closes the shared app-server when startup times out during initialize", async () => { - const { harness, run } = startThreadWithHarness(100); + const { harness, run } = startThreadWithHarness(2_000); + const runError = run.then( + () => undefined, + (error: unknown) => error, + ); - await vi.waitFor(() => expect(harness.writes.length).toBeGreaterThanOrEqual(1)); + const initialize = await waitForRequest(harness, "initialize"); + expect(initialize.id).toBeDefined(); - await expect(run).rejects.toThrow("codex app-server startup timed out"); + const error = await runError; + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toBe("codex app-server startup timed out"); await vi.waitFor(() => expect(harness.stdinDestroyed).toBe(true), { interval: 1, timeout: 2_000, @@ -270,19 +280,29 @@ describe("startCodexAttemptThread", () => { abandonSignal?: AbortSignal; } | undefined; + let resolveFactoryDone: () => void = () => undefined; + const factoryDone = new Promise((resolve) => { + resolveFactoryDone = resolve; + }); const { harness, run } = startThreadWithHarness(100, new AbortController().signal, { attemptClientFactory: (factoryHarness) => async (_startOptions, _authProfileId, _agentDir, _config, options) => { - observedFactoryOptions = options; - await new Promise((resolve) => { - setTimeout(resolve, 250); - }); - options?.onStartedClient?.(factoryHarness.client); - return factoryHarness.client; + try { + observedFactoryOptions = options; + await new Promise((resolve) => { + setTimeout(resolve, 250); + }); + options?.onStartedClient?.(factoryHarness.client); + return factoryHarness.client; + } finally { + resolveFactoryDone(); + } }, }); + const rejected = expect(run).rejects.toThrow("codex app-server startup timed out"); - await expect(run).rejects.toThrow("codex app-server startup timed out"); + await rejected; + await factoryDone; await vi.waitFor(() => expect(harness.stdinDestroyed).toBe(true), { interval: 1, timeout: 2_000, @@ -296,7 +316,7 @@ describe("startCodexAttemptThread", () => { it("clears the shared app-server when cancellation abandons an in-flight thread request", async () => { const abortController = new AbortController(); - const { harness, run } = startThreadWithHarness(5_000, abortController.signal); + const { harness, run } = startThreadWithHarness(30_000, abortController.signal); const runError = run.then( () => undefined, (error: unknown) => error, diff --git a/extensions/codex/src/app-server/client.test.ts b/extensions/codex/src/app-server/client.test.ts index bcbaf521ea09..0c3f55c6716f 100644 --- a/extensions/codex/src/app-server/client.test.ts +++ b/extensions/codex/src/app-server/client.test.ts @@ -408,9 +408,10 @@ describe("CodexAppServerClient", () => { // Start a pending request so we can verify it gets properly rejected. const pending = harness.client.request("test/method"); - // Simulate the child process closing its pipe — a write to the now-dead - // stdin emits an asynchronous EPIPE error on the stream. - harness.process.stdin.destroy(Object.assign(new Error("write EPIPE"), { code: "EPIPE" })); + // Simulate the child process closing its pipe: stdin emits an asynchronous + // EPIPE error before the transport observes a process exit. + const pipeError = Object.assign(new Error("write EPIPE"), { code: "EPIPE" }); + harness.process.stdin.emit("error", pipeError); // The pending request must be rejected with the pipe error rather than // an unhandled exception tearing down the gateway. diff --git a/extensions/codex/src/app-server/test-support.ts b/extensions/codex/src/app-server/test-support.ts index b57e95f416b9..3dbd00703590 100644 --- a/extensions/codex/src/app-server/test-support.ts +++ b/extensions/codex/src/app-server/test-support.ts @@ -43,7 +43,9 @@ export function createClientHarness() { const result = destroyStdin(error); if (!exitEmitted) { exitEmitted = true; - queueMicrotask(emitProcessExit); + // Let stdin surface pipe errors before the harness emits the fake child exit. + // Otherwise close-reason tests can race EPIPE against a synthetic clean exit. + setImmediate(emitProcessExit); } return result; }) as typeof stdin.destroy; diff --git a/scripts/e2e/lib/release-user-journey/clickclack-fixture.mjs b/scripts/e2e/lib/release-user-journey/clickclack-fixture.mjs index 7e4e1441cc87..d3996554a884 100644 --- a/scripts/e2e/lib/release-user-journey/clickclack-fixture.mjs +++ b/scripts/e2e/lib/release-user-journey/clickclack-fixture.mjs @@ -4,6 +4,7 @@ import http from "node:http"; import { readPositiveIntEnv } from "../env-limits.mjs"; const port = readPositiveIntEnv("CLICKCLACK_FIXTURE_PORT", 44181); +const requestMaxBytes = readPositiveIntEnv("CLICKCLACK_FIXTURE_REQUEST_MAX_BYTES", 4 * 1024 * 1024); const token = process.env.CLICKCLACK_FIXTURE_TOKEN ?? "clickclack-release-token"; const statePath = process.env.CLICKCLACK_FIXTURE_STATE ?? "/tmp/openclaw-clickclack-fixture.json"; const workspace = { @@ -86,21 +87,65 @@ function checkAuth(req, res) { function readBody(req) { return new Promise((resolve, reject) => { let body = ""; + let bytes = 0; + let settled = false; req.setEncoding("utf8"); req.on("data", (chunk) => { + if (settled) { + return; + } + bytes += Buffer.byteLength(chunk, "utf8"); + if (bytes > requestMaxBytes) { + settled = true; + body = ""; + req.resume(); + reject(requestBodyTooLargeError()); + return; + } body += chunk; }); req.on("end", () => { + if (settled) { + return; + } + settled = true; try { resolve(body ? JSON.parse(body) : {}); } catch { resolve({}); } }); - req.on("error", reject); + req.on("error", (error) => { + if (!settled) { + settled = true; + reject(error instanceof Error ? error : new Error(String(error))); + } + }); }); } +function requestBodyTooLargeError() { + return Object.assign(new Error(`ClickClack fixture request body exceeded ${requestMaxBytes} bytes`), { + code: "ETOOBIG", + }); +} + +function isRequestBodyTooLargeError(error) { + return error instanceof Error && error.code === "ETOOBIG"; +} + +function handleRequestError(res, error) { + if (res.headersSent) { + res.destroy(); + return; + } + if (isRequestBodyTooLargeError(error)) { + json(res, 413, { error: error.message }); + return; + } + json(res, 500, { error: String(error instanceof Error ? error.message : error) }); +} + function createMessage({ body, author = humanUser, parentMessageId }) { messageSeq += 1; const id = `msg_${messageSeq}`; @@ -171,8 +216,8 @@ function broadcast(event) { } } -const server = http.createServer((req, res) => { - void (async () => { +async function handleRequest(req, res) { + try { const url = new URL(req.url ?? "/", "http://127.0.0.1"); if (!checkAuth(req, res)) { return; @@ -244,7 +289,13 @@ const server = http.createServer((req, res) => { return; } json(res, 404, { error: `unhandled ${req.method} ${url.pathname}` }); - })(); + } catch (error) { + handleRequestError(res, error); + } +} + +const server = http.createServer((req, res) => { + void handleRequest(req, res); }); server.on("upgrade", (req, socket) => { diff --git a/scripts/e2e/telegram-user-credential-io.ts b/scripts/e2e/telegram-user-credential-io.ts index 8b1e13ec02a2..e54f8e38d22a 100644 --- a/scripts/e2e/telegram-user-credential-io.ts +++ b/scripts/e2e/telegram-user-credential-io.ts @@ -20,7 +20,7 @@ type RunCommandOptions = { const DEFAULT_OUTPUT_LIMIT = 128 * 1024; const DEFAULT_FETCH_BODY_LIMIT = 1024 * 1024; -const KILL_GRACE_MS = 5_000; +const KILL_GRACE_MS = readKillGraceMs(); const SIGNAL_EXIT_CODES = { SIGHUP: 129, SIGINT: 130, @@ -30,11 +30,38 @@ const ACTIVE_CHILD_TREE_KILLERS = new Set<(signal: NodeJS.Signals) => void>(); let forwardedSignalExitCode: number | undefined; let forwardedSignalForceKillTimer: NodeJS.Timeout | undefined; +function readKillGraceMs() { + const raw = process.env.OPENCLAW_QA_CREDENTIAL_KILL_GRACE_MS?.trim(); + if (!raw) { + return 5_000; + } + if (!/^\d+$/u.test(raw)) { + throw new Error(`OPENCLAW_QA_CREDENTIAL_KILL_GRACE_MS must be a non-negative integer; got: ${raw}`); + } + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`OPENCLAW_QA_CREDENTIAL_KILL_GRACE_MS must be a non-negative integer; got: ${raw}`); + } + return parsed; +} + +function finishForwardedSignalIfIdle() { + if (forwardedSignalExitCode === undefined || ACTIVE_CHILD_TREE_KILLERS.size > 0) { + return; + } + if (forwardedSignalForceKillTimer) { + clearTimeout(forwardedSignalForceKillTimer); + forwardedSignalForceKillTimer = undefined; + } + process.exit(forwardedSignalExitCode); +} + for (const signal of Object.keys(SIGNAL_EXIT_CODES) as Array) { process.on(signal, () => { forwardedSignalExitCode ??= SIGNAL_EXIT_CODES[signal]; if (ACTIVE_CHILD_TREE_KILLERS.size === 0) { - process.exit(forwardedSignalExitCode); + finishForwardedSignalIfIdle(); + return; } const activeKillers = Array.from(ACTIVE_CHILD_TREE_KILLERS); for (const killChildTree of activeKillers) { @@ -156,7 +183,7 @@ export function runCommand( return; } if (forwardedSignalExitCode !== undefined) { - activeChildTree.unregister(); + activeChildTree.unregister({ finishForwardedSignal: !childProcessTreeMayStillExist(child) }); return; } if (timedOutError && killTimer && childProcessTreeMayStillExist(child)) { @@ -218,8 +245,11 @@ function registerActiveChildProcessTree(child: ReturnType) { ACTIVE_CHILD_TREE_KILLERS.add(killChildTree); return { killChildTree, - unregister: () => { + unregister: (options: { finishForwardedSignal?: boolean } = {}) => { ACTIVE_CHILD_TREE_KILLERS.delete(killChildTree); + if (options.finishForwardedSignal ?? true) { + finishForwardedSignalIfIdle(); + } }, }; } diff --git a/src/agents/code-mode.test.ts b/src/agents/code-mode.test.ts index a88dffc6014e..1416ee41c968 100644 --- a/src/agents/code-mode.test.ts +++ b/src/agents/code-mode.test.ts @@ -1384,7 +1384,7 @@ describe("Code Mode", () => { tools: { codeMode: { enabled: true, - timeoutMs: 100, + timeoutMs: 500, }, }, } as never; diff --git a/src/tui/tui-pty-harness.e2e.test.ts b/src/tui/tui-pty-harness.e2e.test.ts index 27a5f39c8ae0..2e70130c9d1b 100644 --- a/src/tui/tui-pty-harness.e2e.test.ts +++ b/src/tui/tui-pty-harness.e2e.test.ts @@ -11,11 +11,11 @@ type FixtureLogEntry = { }; const activeRuns: PtyRun[] = []; -const STARTUP_TIMEOUT_MS = 10_000; +const STARTUP_TIMEOUT_MS = 20_000; const OUTPUT_TIMEOUT_MS = 2_000; const EXIT_TIMEOUT_MS = 4_000; const TEST_TIMEOUT_MS = 5_000; -const STARTUP_TEST_TIMEOUT_MS = 10_000; +const STARTUP_TEST_TIMEOUT_MS = 25_000; async function readFixtureLog(logPath: string): Promise { try { diff --git a/test/scripts/e2e-helper-env-limits.test.ts b/test/scripts/e2e-helper-env-limits.test.ts index dcbd79c976fb..ed750130545a 100644 --- a/test/scripts/e2e-helper-env-limits.test.ts +++ b/test/scripts/e2e-helper-env-limits.test.ts @@ -1,5 +1,8 @@ import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; import { createServer, type Server } from "node:http"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; const browserFixturePath = "scripts/e2e/lib/browser-cdp-snapshot/fixture-server.mjs"; @@ -57,6 +60,45 @@ async function listen(server: Server): Promise { return `http://127.0.0.1:${address.port}`; } +async function allocatePort(): Promise { + const server = createServer(); + const url = await listen(server); + await new Promise((resolve) => server.close(() => resolve())); + return Number(new URL(url).port); +} + +async function waitForOutput( + child: ReturnType, + matches: (text: string) => boolean, + getOutput: () => string, +): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < 3_000) { + if (matches(getOutput())) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error(`timed out waiting for fixture output. Output: ${getOutput()}`); +} + +async function stopChild(child: ReturnType): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + child.kill("SIGTERM"); + await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill("SIGKILL"); + resolve(); + }, 1_000); + child.once("exit", () => { + clearTimeout(timer); + resolve(); + }); + }); +} + describe("e2e helper numeric env limits", () => { it("rejects loose Browser CDP fixture ports", async () => { const result = await runScriptAsync(browserFixturePath, [], { FIXTURE_PORT: "18080http" }); @@ -74,6 +116,49 @@ describe("e2e helper numeric env limits", () => { expect(result.stderr).toContain("invalid CLICKCLACK_FIXTURE_PORT: 44181tcp"); }); + it("rejects oversized ClickClack fixture request bodies", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-clickclack-fixture-")); + const port = await allocatePort(); + const child = spawn(process.execPath, [clickclackFixturePath], { + env: { + ...process.env, + CLICKCLACK_FIXTURE_PORT: String(port), + CLICKCLACK_FIXTURE_REQUEST_MAX_BYTES: "16", + CLICKCLACK_FIXTURE_STATE: path.join(tempDir, "state.json"), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + output += chunk; + }); + child.stderr.on("data", (chunk) => { + output += chunk; + }); + try { + await waitForOutput( + child, + (text) => text.includes(`clickclack fixture listening on ${port}`), + () => output, + ); + + const response = await fetch(`http://127.0.0.1:${port}/fixture/inbound`, { + body: JSON.stringify({ body: "x".repeat(64) }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + const body = await response.json(); + + expect(response.status).toBe(413); + expect(body).toEqual({ error: "ClickClack fixture request body exceeded 16 bytes" }); + } finally { + await stopChild(child); + fs.rmSync(tempDir, { force: true, recursive: true }); + } + }); + it("rejects loose Open WebUI HTTP probe timeouts", () => { const result = runScript(httpProbePath, ["http://127.0.0.1:9"], { OPENCLAW_HTTP_PROBE_TIMEOUT_MS: "8000ms", diff --git a/test/scripts/telegram-user-credential.test.ts b/test/scripts/telegram-user-credential.test.ts index 804a0dae8c1f..f3e4c9037eec 100644 --- a/test/scripts/telegram-user-credential.test.ts +++ b/test/scripts/telegram-user-credential.test.ts @@ -1,3 +1,4 @@ +import { spawn } from "node:child_process"; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -53,6 +54,24 @@ async function waitForDead(pid: number, timeoutMs: number): Promise { throw new Error(`process still alive: ${pid}`); } +async function waitForExit( + child: ReturnType, + timeoutMs: number, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + if (child.exitCode !== null || child.signalCode !== null) { + return { code: child.exitCode, signal: child.signalCode as NodeJS.Signals | null }; + } + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`process did not exit within ${timeoutMs}ms`)); + }, timeoutMs); + child.once("exit", (code, signal) => { + clearTimeout(timer); + resolve({ code, signal }); + }); + }); +} + afterEach(() => { for (const dir of tempDirs.splice(0)) { rmSync(dir, { force: true, recursive: true }); @@ -147,36 +166,86 @@ setInterval(() => {}, 1000); }, ); + it.runIf(process.platform !== "win32")("kills timed-out child process groups", async () => { + const dir = makeTempDir("openclaw-telegram-credential-tree-timeout-"); + const childPidPath = path.join(dir, "child.pid"); + let childPid: number | undefined; + + try { + const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"; + const parentScript = [ + "const { spawn } = require('node:child_process');", + "const fs = require('node:fs');", + `const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`, + `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid));`, + "setInterval(() => {}, 1000);", + ].join(""); + + const runPromise = runCommand(process.execPath, ["-e", parentScript], dir, { + timeoutKillGraceMs: 25, + timeoutMs: 100, + }); + await waitForFile(childPidPath, 2_000); + childPid = Number.parseInt(readFileSync(childPidPath, "utf8"), 10); + + await expect(runPromise).rejects.toMatchObject({ + code: "ETIMEDOUT", + message: expect.stringContaining("timed out after 100ms"), + }); + await waitForDead(childPid, 2_000); + } finally { + if (childPid !== undefined && isProcessAlive(childPid)) { + process.kill(childPid, "SIGKILL"); + } + } + }); + it.runIf(process.platform !== "win32")( - "kills timed-out child process groups", + "exits promptly after forwarded SIGTERM children exit cleanly", async () => { - const dir = makeTempDir("openclaw-telegram-credential-tree-timeout-"); + const dir = makeTempDir("openclaw-telegram-credential-signal-"); + const runnerPath = path.join(dir, "runner.mjs"); + const readyPath = path.join(dir, "ready.txt"); const childPidPath = path.join(dir, "child.pid"); + const ioModuleUrl = new URL( + "../../scripts/e2e/telegram-user-credential-io.ts", + import.meta.url, + ).href; + const childScript = [ + "const fs = require('node:fs');", + `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid));`, + `fs.writeFileSync(${JSON.stringify(readyPath)}, 'ready');`, + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => {}, 1000);", + ].join(""); + writeFileSync( + runnerPath, + [ + `import { runCommand } from ${JSON.stringify(ioModuleUrl)};`, + `await runCommand(process.execPath, ['-e', ${JSON.stringify(childScript)}], undefined, { timeoutMs: 30_000 });`, + "", + ].join("\n"), + "utf8", + ); + const runner = spawn(process.execPath, ["--import", "tsx", runnerPath], { + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); let childPid: number | undefined; - try { - const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);"; - const parentScript = [ - "const { spawn } = require('node:child_process');", - "const fs = require('node:fs');", - `const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`, - `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid));`, - "setInterval(() => {}, 1000);", - ].join(""); - - const runPromise = runCommand(process.execPath, ["-e", parentScript], dir, { - timeoutKillGraceMs: 25, - timeoutMs: 100, - }); - await waitForFile(childPidPath, 2_000); + await waitForFile(readyPath, 2_000); childPid = Number.parseInt(readFileSync(childPidPath, "utf8"), 10); + const startedAt = Date.now(); + runner.kill("SIGTERM"); + const exit = await waitForExit(runner, 2_000); - await expect(runPromise).rejects.toMatchObject({ - code: "ETIMEDOUT", - message: expect.stringContaining("timed out after 100ms"), - }); + expect(exit).toEqual({ code: 143, signal: null }); + expect(Date.now() - startedAt).toBeLessThan(1_500); await waitForDead(childPid, 2_000); } finally { + if (runner.exitCode === null && runner.signalCode === null) { + runner.kill("SIGKILL"); + } if (childPid !== undefined && isProcessAlive(childPid)) { process.kill(childPid, "SIGKILL"); } @@ -184,6 +253,68 @@ setInterval(() => {}, 1000); }, ); + it.runIf(process.platform !== "win32")( + "keeps the forwarded signal force-kill armed while grandchildren survive", + async () => { + const dir = makeTempDir("openclaw-telegram-credential-grandchild-signal-"); + const runnerPath = path.join(dir, "runner.mjs"); + const readyPath = path.join(dir, "ready.txt"); + const grandchildPidPath = path.join(dir, "grandchild.pid"); + const ioModuleUrl = new URL( + "../../scripts/e2e/telegram-user-credential-io.ts", + import.meta.url, + ).href; + const grandchildScript = [ + "const fs = require('node:fs');", + `fs.writeFileSync(${JSON.stringify(grandchildPidPath)}, String(process.pid));`, + "process.on('SIGTERM', () => {});", + "setInterval(() => {}, 1000);", + ].join(""); + const parentScript = [ + "const { spawn } = require('node:child_process');", + "const fs = require('node:fs');", + `const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' });`, + `fs.writeFileSync(${JSON.stringify(readyPath)}, String(grandchild.pid));`, + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => {}, 1000);", + ].join(""); + writeFileSync( + runnerPath, + [ + `import { runCommand } from ${JSON.stringify(ioModuleUrl)};`, + `await runCommand(process.execPath, ['-e', ${JSON.stringify(parentScript)}], undefined, { timeoutMs: 30_000 });`, + "", + ].join("\n"), + "utf8", + ); + const runner = spawn(process.execPath, ["--import", "tsx", runnerPath], { + env: { + ...process.env, + OPENCLAW_QA_CREDENTIAL_KILL_GRACE_MS: "100", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let grandchildPid: number | undefined; + try { + await waitForFile(readyPath, 2_000); + await waitForFile(grandchildPidPath, 2_000); + grandchildPid = Number.parseInt(readFileSync(grandchildPidPath, "utf8"), 10); + runner.kill("SIGTERM"); + const exit = await waitForExit(runner, 2_000); + + expect(exit).toEqual({ code: 143, signal: null }); + await waitForDead(grandchildPid, 2_000); + } finally { + if (runner.exitCode === null && runner.signalCode === null) { + runner.kill("SIGKILL"); + } + if (grandchildPid !== undefined && isProcessAlive(grandchildPid)) { + process.kill(grandchildPid, "SIGKILL"); + } + } + }, + ); + it("aborts broker fetches that never return", async () => { let signal: AbortSignal | undefined; await expect( diff --git a/ui/src/ui/e2e/chat-flow.e2e.test.ts b/ui/src/ui/e2e/chat-flow.e2e.test.ts index a74bcf4c5a3e..85cf3eb9f659 100644 --- a/ui/src/ui/e2e/chat-flow.e2e.test.ts +++ b/ui/src/ui/e2e/chat-flow.e2e.test.ts @@ -108,6 +108,35 @@ async function controlUiEventPayloads( }, event); } +async function waitForControlUiChatSendPhases( + page: Page, + runId: string, + phases: string[], +): Promise { + await page.waitForFunction( + ({ expectedPhases, expectedRunId }) => { + const app = document.querySelector("openclaw-app") as + | (Element & { eventLogBuffer?: unknown[] }) + | null; + const observedPhases = new Set( + (app?.eventLogBuffer ?? []).flatMap((entry) => { + const candidate = entry as { + event?: unknown; + payload?: { phase?: unknown; runId?: unknown }; + }; + return candidate.event === "control-ui.chat.send" && + candidate.payload?.runId === expectedRunId && + typeof candidate.payload.phase === "string" + ? [candidate.payload.phase] + : []; + }), + ); + return expectedPhases.every((phase) => observedPhases.has(phase)); + }, + { expectedPhases: phases, expectedRunId: runId }, + ); +} + function chatSessionListResponse() { return { count: 2, @@ -344,6 +373,7 @@ describeControlUiE2e("Control UI mocked Gateway E2E", () => { const runId = requireString(params.idempotencyKey, "chat send idempotency key"); await page.locator(".chat-thread").getByText(prompt).waitFor({ timeout: 10_000 }); + await waitForControlUiChatSendPhases(page, runId, ["ack"]); await gateway.emitGatewayEvent("chat", { deltaText: "First token visible.", message: { @@ -357,28 +387,45 @@ describeControlUiE2e("Control UI mocked Gateway E2E", () => { state: "delta", }); await page.getByText("First token visible.").waitFor({ timeout: 10_000 }); - await page.waitForFunction((expectedRunId) => { - const app = document.querySelector("openclaw-app") as - | (Element & { eventLogBuffer?: unknown[] }) - | null; - return (app?.eventLogBuffer ?? []).some((entry) => { - const candidate = entry as { - event?: unknown; - payload?: { phase?: unknown; runId?: unknown }; - }; - return ( - candidate.event === "control-ui.chat.send" && - candidate.payload?.phase === "first-assistant-visible" && - candidate.payload.runId === expectedRunId - ); - }); - }, runId); - const firstOutputEvents = await controlUiEventPayloads(page, "control-ui.chat.send"); - expect( - firstOutputEvents.some( - (payload) => payload.phase === "first-assistant-visible" && payload.runId === runId, - ), - ).toBe(true); + await waitForControlUiChatSendPhases(page, runId, [ + "pending-visible", + "request-start", + "ack", + "first-assistant-visible", + ]); + const sendTimingEvents = (await controlUiEventPayloads(page, "control-ui.chat.send")).filter( + (payload) => payload.runId === runId, + ); + const sendTimingByPhase = new Map( + sendTimingEvents.map((payload) => [payload.phase, payload]), + ); + expect(sendTimingEvents.map((payload) => payload.phase)).toEqual( + expect.arrayContaining([ + "pending-visible", + "request-start", + "ack", + "first-assistant-visible", + ]), + ); + const ackTiming = sendTimingByPhase.get("ack"); + expect(ackTiming).toMatchObject({ + ackStatus: "started", + runId, + sendState: "sending", + sessionKey: "global", + }); + expect(ackTiming?.requestDurationMs).toEqual(expect.any(Number)); + const firstVisibleTiming = sendTimingByPhase.get("first-assistant-visible"); + expect(firstVisibleTiming).toMatchObject({ + ackStatus: "started", + eventState: "delta", + runId, + sendState: "sending", + sessionKey: "global", + }); + expect(firstVisibleTiming?.ackToFirstAssistantEventMs).toEqual(expect.any(Number)); + expect(firstVisibleTiming?.firstAssistantPaintMs).toEqual(expect.any(Number)); + expect(firstVisibleTiming?.requestToFirstAssistantEventMs).toEqual(expect.any(Number)); await gateway.resolveDeferred("chat.startup", { agentsList: { agents: [{ id: "ops", name: "OpenClaw" }],