diff --git a/extensions/qa-lab/api.ts b/extensions/qa-lab/api.ts index eed290c13f9b..2b852044bea4 100644 --- a/extensions/qa-lab/api.ts +++ b/extensions/qa-lab/api.ts @@ -99,14 +99,10 @@ export { } from "./src/self-check.js"; export { runQaE2eSelfCheck, runQaLabSelfCheck } from "./src/self-check-runner.js"; export { - testing, - buildQaRuntimeEnv, type QaCliBackendAuthMode, type QaGatewayChildListeningContext, type QaGatewayChildCommand, type QaGatewayChildStateMutationContext, - resolveQaControlUiRoot, - resolveQaGatewayChildProviderMode, startQaGatewayChild, } from "./src/gateway-child.js"; export { diff --git a/extensions/qa-lab/src/bundled-plugin-staging.ts b/extensions/qa-lab/src/bundled-plugin-staging.ts index b53cbd40398b..f66a4e46de00 100644 --- a/extensions/qa-lab/src/bundled-plugin-staging.ts +++ b/extensions/qa-lab/src/bundled-plugin-staging.ts @@ -36,7 +36,7 @@ function isQaOpenAiResponsesProviderConfig(config: ModelProviderConfig) { ); } -export function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) { +function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) { assertSafeQaBundledPluginId(params.pluginId); const candidates = [ path.join(params.repoRoot, "dist", "extensions", params.pluginId), diff --git a/extensions/qa-lab/src/cli-paths.ts b/extensions/qa-lab/src/cli-paths.ts index b934f36a64d7..e943dd647780 100644 --- a/extensions/qa-lab/src/cli-paths.ts +++ b/extensions/qa-lab/src/cli-paths.ts @@ -1,5 +1,4 @@ // Qa Lab plugin module implements cli paths behavior. -import fs from "node:fs/promises"; import path from "node:path"; import { assertNoSymlinkParents, pathScope } from "openclaw/plugin-sdk/security-runtime"; @@ -29,25 +28,6 @@ export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: strin return resolved.path; } -async function resolveNearestExistingPath(targetPath: string) { - let current = path.resolve(targetPath); - while (true) { - try { - await fs.lstat(current); - return current; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - } - const parent = path.dirname(current); - if (parent === current) { - throw new Error(`failed to resolve existing path for ${targetPath}`); - } - current = parent; - } -} - function assertRepoRelativePath(repoRoot: string, targetPath: string, label: string) { const relative = path.relative(repoRoot, targetPath); if (relative.startsWith("..") || path.isAbsolute(relative)) { @@ -72,18 +52,6 @@ async function assertNoSymlinkSegments(repoRoot: string, targetPath: string, lab } } -export async function assertRepoBoundPath(repoRoot: string, targetPath: string, label: string) { - const repoRootResolved = path.resolve(repoRoot); - const targetResolved = path.resolve(targetPath); - assertRepoRelativePath(repoRootResolved, targetResolved, label); - await assertNoSymlinkSegments(repoRootResolved, targetResolved, label); - const repoRootReal = await fs.realpath(repoRootResolved); - const nearestExistingPath = await resolveNearestExistingPath(targetResolved); - const nearestExistingReal = await fs.realpath(nearestExistingPath); - assertRepoRelativePath(repoRootReal, nearestExistingReal, label); - return targetResolved; -} - export async function ensureRepoBoundDirectory( repoRoot: string, targetDir: string, diff --git a/extensions/qa-lab/src/gateway-child-artifacts.ts b/extensions/qa-lab/src/gateway-child-artifacts.ts new file mode 100644 index 000000000000..2d6d0df8e0c0 --- /dev/null +++ b/extensions/qa-lab/src/gateway-child-artifacts.ts @@ -0,0 +1,72 @@ +// Qa Lab plugin module owns sanitized gateway debug artifacts and temp cleanup. +import fs from "node:fs/promises"; +import path from "node:path"; +import { ensureRepoBoundDirectory } from "./cli-paths.js"; +import { redactQaGatewayDebugText } from "./gateway-log-redaction.js"; + +async function writeSanitizedQaGatewayDebugLog(params: { sourcePath: string; targetPath: string }) { + const contents = await fs.readFile(params.sourcePath, "utf8").catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return ""; + } + throw error; + }); + await fs.writeFile(params.targetPath, redactQaGatewayDebugText(contents), "utf8"); +} + +async function clearQaGatewayArtifactDir(dir: string) { + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + await fs.rm(path.join(dir, entry.name), { recursive: true, force: true }); + } +} + +export async function cleanupQaGatewayTempRoots(params: { + tempRoot: string; + stagedBundledPluginsRoot?: string | null; +}) { + await fs.rm(params.tempRoot, { recursive: true, force: true }).catch(() => {}); + if (params.stagedBundledPluginsRoot) { + await fs.rm(params.stagedBundledPluginsRoot, { recursive: true, force: true }).catch(() => {}); + } +} + +export async function preserveQaGatewayDebugArtifacts(params: { + preserveToDir: string; + stdoutLogPath: string; + stderrLogPath: string; + tempRoot: string; + repoRoot?: string; +}) { + const preserveToDir = params.repoRoot + ? await ensureRepoBoundDirectory( + params.repoRoot, + params.preserveToDir, + "QA gateway artifact directory", + { + mode: 0o700, + }, + ) + : params.preserveToDir; + await fs.mkdir(preserveToDir, { recursive: true, mode: 0o700 }); + await clearQaGatewayArtifactDir(preserveToDir); + await Promise.all([ + writeSanitizedQaGatewayDebugLog({ + sourcePath: params.stdoutLogPath, + targetPath: path.join(preserveToDir, "gateway.stdout.log"), + }), + writeSanitizedQaGatewayDebugLog({ + sourcePath: params.stderrLogPath, + targetPath: path.join(preserveToDir, "gateway.stderr.log"), + }), + ]); + await fs.writeFile( + path.join(preserveToDir, "README.txt"), + [ + "Only sanitized gateway debug artifacts are preserved here.", + "The full QA gateway runtime was not copied because it may contain credentials or auth tokens.", + "Original runtime temp root omitted because local temp paths can identify the runner.", + "", + ].join("\n"), + "utf8", + ); +} diff --git a/extensions/qa-lab/src/gateway-child-command.ts b/extensions/qa-lab/src/gateway-child-command.ts new file mode 100644 index 000000000000..fa1ef27aa837 --- /dev/null +++ b/extensions/qa-lab/src/gateway-child-command.ts @@ -0,0 +1,106 @@ +// Qa Lab plugin module owns gateway child command bootstrap behavior. +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { formatErrorMessage, toErrorObject } from "openclaw/plugin-sdk/error-runtime"; +import { + appendQaChildOutput, + appendQaChildOutputTail, + createQaChildOutputCapture, + createQaChildOutputTail, + formatQaChildOutputTail, + readQaChildOutput, +} from "./child-output.js"; +import { hasQaGatewayChildExited, monitorQaChildFailure } from "./gateway-child-process.js"; +import type { QaGatewayProcessBoundaryConfig } from "./gateway-process-boundary.js"; + +type QaGatewayChildDirectCommand = { + executablePath: string; + argsPrefix?: string[]; + argsSuffix?: string[]; + cwd?: string; + tempParentDir?: string; + usePackagedPlugins?: boolean; + processBoundary?: undefined; +}; + +type QaGatewayChildVerifiedCommand = Omit & { + processBoundary: QaGatewayProcessBoundaryConfig; +}; + +export type QaGatewayChildCommand = QaGatewayChildDirectCommand | QaGatewayChildVerifiedCommand; + +export function resolveQaGatewayChildCommand(repoRoot: string): QaGatewayChildCommand { + for (const relativePath of ["scripts/run-node.mjs", "dist/index.mjs", "dist/index.js"]) { + const entryPath = path.join(repoRoot, relativePath); + if (existsSync(entryPath)) { + return { + executablePath: process.execPath, + argsPrefix: [entryPath], + cwd: repoRoot, + usePackagedPlugins: true, + }; + } + } + + throw new Error( + "OpenClaw CLI entry not found: expected scripts/run-node.mjs or dist/index.(m)js", + ); +} + +export async function runQaGatewayCliCommand(params: { + executablePath: string; + argsPrefix: readonly string[]; + args: readonly string[]; + cwd: string; + env: NodeJS.ProcessEnv; + stdin?: string; +}): Promise { + const hasStdin = params.stdin !== undefined; + const child = spawn(params.executablePath, [...params.argsPrefix, ...params.args], { + cwd: params.cwd, + env: { ...params.env, OPENCLAW_CLI: "1" }, + stdio: [hasStdin ? "pipe" : "ignore", "pipe", "pipe"], + }); + const result = readQaGatewayCliCommand(child); + if (hasStdin) { + child.stdin?.once("error", () => {}); + child.stdin?.end(params.stdin); + } + return await result; +} + +async function readQaGatewayCliCommand(child: ChildProcess): Promise { + const stdout = createQaChildOutputCapture(); + const stderr = createQaChildOutputTail(); + child.stdout?.on("data", (chunk) => appendQaChildOutput(stdout, chunk)); + child.stderr?.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk)); + const exitCode = await new Promise((resolve, reject) => { + monitorQaChildFailure(child, (failure) => { + if (failure.source === "process") { + reject(toErrorObject(failure.error, "OpenClaw CLI process failed")); + return; + } + if (!hasQaGatewayChildExited(child) && !child.killed) { + try { + child.kill("SIGKILL"); + } catch { + // The child exited between the state check and signal. + } + } + reject( + new Error( + `qa gateway cli ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`, + { cause: failure.error }, + ), + ); + }); + child.once("close", (code) => resolve(code ?? 1)); + }); + const stdoutText = readQaChildOutput(stdout); + if (exitCode !== 0) { + const stderrText = formatQaChildOutputTail(stderr, "stderr"); + throw new Error(`OpenClaw CLI exited ${exitCode}: ${stderrText || stdoutText}`); + } + return stdoutText; +} diff --git a/extensions/qa-lab/src/gateway-child-env.ts b/extensions/qa-lab/src/gateway-child-env.ts new file mode 100644 index 000000000000..2d6e998fa07a --- /dev/null +++ b/extensions/qa-lab/src/gateway-child-env.ts @@ -0,0 +1,181 @@ +// Qa Lab plugin module owns gateway child runtime environment behavior. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { buildQaCodexAppServerArgs } from "./codex-app-server-args.js"; +import type { QaProviderMode } from "./model-selection.js"; +import { + normalizeQaProviderModeEnv, + resolveQaLiveCliAuthEnv, + type QaCliBackendAuthMode, +} from "./providers/env.js"; +import { getQaProvider } from "./providers/index.js"; +import { + QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV, + QA_LIVE_SETUP_TOKEN_VALUE_ENV, +} from "./providers/live-frontier/auth.js"; +import { listMockCodexModelInfos } from "./providers/shared/mock-model-config.js"; +import type { RuntimeId } from "./runtime-parity.js"; + +const QA_MOCK_OPENAI_API_KEY = ["qa", "mock", "openai", "key"].join("-"); +const QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS = Object.freeze([ + "OPENCLAW_QA_CONVEX_SECRET_CI", + "OPENCLAW_QA_CONVEX_SECRET_MAINTAINER", + "OPENCLAW_QA_SUT_FORBIDDEN_SENTINEL", + "OPENCLAW_QA_TELEGRAM_GROUP_ID", + "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN", + "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", +]); + +function scrubQaGatewayChildSecretEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + for (const envKey of QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS) { + delete env[envKey]; + } + return env; +} + +function scrubQaGatewayChildTestRunnerEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + // The Gateway is a product child, not a nested Vitest worker. Leaking runner + // markers makes the dist launcher select test-only startup behavior. + delete env.VITEST; + delete env.VITEST_POOL_ID; + delete env.VITEST_WORKER_ID; + if (env.NODE_ENV === "test") { + delete env.NODE_ENV; + } + return env; +} + +export function buildQaRuntimeEnv(params: { + configPath: string; + gatewayToken: string; + homeDir: string; + forwardHostHome?: boolean; + stateDir: string; + tempRoot: string; + xdgConfigHome: string; + xdgDataHome: string; + xdgCacheHome: string; + bundledPluginsDir?: string; + stagedBundledPluginsRoot?: string | null; + compatibilityHostVersion?: string; + providerMode?: QaProviderMode; + baseEnv?: NodeJS.ProcessEnv; + runtimeEnvPatch?: NodeJS.ProcessEnv; + forwardHostHomeForClaudeCli?: boolean; + claudeCliAuthMode?: QaCliBackendAuthMode; +}) { + const baseEnv = params.baseEnv ?? process.env; + const provider = params.providerMode ? getQaProvider(params.providerMode) : null; + const forwardedHostHome = params.forwardHostHome + ? baseEnv.HOME?.trim() || os.homedir() + : undefined; + const env: NodeJS.ProcessEnv = { + ...baseEnv, + HOME: forwardedHostHome ?? params.homeDir, + ...(provider?.appliesLiveEnvAliases + ? resolveQaLiveCliAuthEnv(baseEnv, { + forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli, + claudeCliAuthMode: params.claudeCliAuthMode, + }) + : {}), + OPENCLAW_HOME: params.homeDir, + OPENCLAW_CONFIG_PATH: params.configPath, + OPENCLAW_STATE_DIR: params.stateDir, + OPENCLAW_OAUTH_DIR: path.join(params.stateDir, "credentials"), + OPENCLAW_GATEWAY_TOKEN: params.gatewayToken, + OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1", + OPENCLAW_SKIP_GMAIL_WATCHER: "1", + OPENCLAW_SKIP_CANVAS_HOST: "1", + OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1", + OPENCLAW_NO_RESPAWN: "1", + OPENCLAW_TEST_FAST: "1", + OPENCLAW_EMBEDDED_ABORT_SETTLE_TIMEOUT_MS: "2000", + OPENCLAW_QA_PARENT_PID: String(process.pid), + OPENCLAW_QA_TEMP_ROOT: params.tempRoot, + ...(params.stagedBundledPluginsRoot + ? { OPENCLAW_QA_STAGED_RUNTIME_ROOT: params.stagedBundledPluginsRoot } + : {}), + OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER: "1", + // QA uses the fast runtime envelope for speed, but it still exercises + // normal config-driven heartbeats and runtime config writes. + OPENCLAW_ALLOW_SLOW_REPLY_TESTS: "1", + XDG_CONFIG_HOME: params.xdgConfigHome, + XDG_DATA_HOME: params.xdgDataHome, + XDG_CACHE_HOME: params.xdgCacheHome, + ...(params.bundledPluginsDir ? { OPENCLAW_BUNDLED_PLUGINS_DIR: params.bundledPluginsDir } : {}), + ...(params.compatibilityHostVersion + ? { OPENCLAW_COMPATIBILITY_HOST_VERSION: params.compatibilityHostVersion } + : {}), + }; + const normalizedEnv = normalizeQaProviderModeEnv(env, params.providerMode); + // Test-runner skip flags are parent controls; each QA child declares its own runtime needs. + delete normalizedEnv.OPENCLAW_SKIP_CHANNELS; + delete normalizedEnv.OPENCLAW_SKIP_PROVIDERS; + Object.assign(normalizedEnv, params.runtimeEnvPatch); + normalizedEnv.OPENCLAW_BUILD_PRIVATE_QA = "1"; + delete normalizedEnv[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]; + delete normalizedEnv[QA_LIVE_SETUP_TOKEN_VALUE_ENV]; + return scrubQaGatewayChildSecretEnv(scrubQaGatewayChildTestRunnerEnv(normalizedEnv)); +} + +export async function stageQaCodexMockModelCatalog(params: { + tempRoot: string; + forcedRuntime?: RuntimeId; + providerMode: QaProviderMode; + primaryModel?: string; + alternateModel?: string; +}): Promise { + if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") { + return undefined; + } + const modelCatalogPath = path.join(params.tempRoot, "codex-model-catalog.json"); + const selectedModelRefs = [params.primaryModel, params.alternateModel].filter( + (model): model is string => typeof model === "string" && model.length > 0, + ); + await fs.writeFile( + modelCatalogPath, + `${JSON.stringify({ models: listMockCodexModelInfos(selectedModelRefs) }, null, 2)}\n`, + { encoding: "utf8", mode: 0o600 }, + ); + return modelCatalogPath; +} + +export function buildQaForcedRuntimeEnvPatch(params: { + forcedRuntime?: RuntimeId; + providerMode: QaProviderMode; + providerBaseUrl?: string; + codexModelCatalogPath?: string; + nativeAppServerArgs?: string; +}): NodeJS.ProcessEnv | undefined { + if (!params.forcedRuntime) { + return undefined; + } + const patch: NodeJS.ProcessEnv = { + OPENCLAW_BUILD_PRIVATE_QA: "1", + OPENCLAW_QA_FORCE_RUNTIME: params.forcedRuntime, + }; + if (params.forcedRuntime !== "codex") { + return patch; + } + if (params.providerMode !== "mock-openai") { + patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({ + existingArgs: params.nativeAppServerArgs, + }); + return patch; + } + const providerBaseUrl = params.providerBaseUrl?.trim().replace(/\/+$/u, ""); + if (!providerBaseUrl) { + throw new Error("forced Codex mock QA requires the managed mock provider URL"); + } + if (!params.codexModelCatalogPath) { + throw new Error("forced Codex mock QA requires the staged native model catalog"); + } + patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({ + providerBaseUrl, + modelCatalogPath: params.codexModelCatalogPath, + }); + patch.OPENAI_API_KEY = QA_MOCK_OPENAI_API_KEY; + patch.CODEX_API_KEY = QA_MOCK_OPENAI_API_KEY; + return patch; +} diff --git a/extensions/qa-lab/src/gateway-child-process.ts b/extensions/qa-lab/src/gateway-child-process.ts new file mode 100644 index 000000000000..c603d54fab20 --- /dev/null +++ b/extensions/qa-lab/src/gateway-child-process.ts @@ -0,0 +1,295 @@ +// Qa Lab plugin module owns gateway child process lifecycle behavior. +import type { ChildProcess } from "node:child_process"; +import type { WriteStream } from "node:fs"; +import { finished } from "node:stream/promises"; +import { StringDecoder } from "node:string_decoder"; +import { setTimeout as sleep } from "node:timers/promises"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { QaSuiteInfraError } from "./errors.js"; +import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js"; +import { + inspectLinuxProcessGroup, + type QaLinuxProcessGroupInspector, +} from "./posix-process-group.js"; +import { runQaWindowsTaskkill } from "./windows-system-tools.js"; + +const QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 30_000; +const QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS = 10_000; +const QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS = 5_000; +const QA_GATEWAY_CHILD_RECENT_LOG_CHARS = 64 * 1_024; +const QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER = "[qa-lab] older gateway logs truncated\n"; +const QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS = 8_192; + +export type QaChildFailure = { + source: "process" | "stdout" | "stderr"; + error: unknown; +}; + +type QaGatewayChildLogSource = "internal" | "stderr" | "stdout"; + +export function hasQaGatewayChildExited(child: Pick) { + return child.exitCode !== null || child.signalCode !== null; +} + +export function monitorQaChildFailure( + child: ChildProcess, + onFailure: (failure: QaChildFailure) => void, +) { + let reported = false; + const report = (source: QaChildFailure["source"]) => (error: unknown) => { + if (reported) { + return; + } + reported = true; + onFailure({ source, error }); + }; + child.once("error", report("process")); + child.stdout?.once("error", report("stdout")); + child.stderr?.once("error", report("stderr")); +} + +export async function closeQaGatewayLogStream( + stream: WriteStream, + label: "stderr" | "stdout", + timeoutMs = QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS, +) { + if (stream.destroyed) { + return; + } + stream.end(); + const signal = AbortSignal.timeout(timeoutMs); + try { + await finished(stream, { cleanup: true, signal }); + } catch (error) { + if (!signal.aborted) { + throw error; + } + // Gateway logs are diagnostic only. Never let a stuck filesystem flush + // retain the stopped child runtime and its live transport credentials. + process.stderr.write( + `[qa-suite] ${label} gateway log flush exceeded ${timeoutMs}ms; forcing close\n`, + ); + stream.destroy(); + } +} + +export function createQaGatewayChildLogCollector() { + const decoders: Record = { + internal: new StringDecoder("utf8"), + stderr: new StringDecoder("utf8"), + stdout: new StringDecoder("utf8"), + }; + let recent = ""; + let end = 0; + + const readFrom = (mark: number) => { + const start = end - recent.length; + const wasTruncated = mark < start; + const text = recent.slice(Math.max(0, mark - start)); + return `${wasTruncated ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${text}`; + }; + return { + push(source: QaGatewayChildLogSource, chunk: Buffer) { + const text = decoders[source].write(chunk); + end += text.length; + recent += text; + if (recent.length > QA_GATEWAY_CHILD_RECENT_LOG_CHARS) { + recent = sliceUtf16Safe(recent, -QA_GATEWAY_CHILD_RECENT_LOG_CHARS); + } + }, + mark() { + return end; + }, + readSince(mark: number) { + return readFrom(mark); + }, + text() { + return `${end > recent.length ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${recent}`.trim(); + }, + }; +} + +function formatQaGatewayChildFailure(failure: QaChildFailure) { + return failure.source === "process" + ? `gateway failed to spawn: ${formatErrorMessage(failure.error)}` + : `gateway child ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`; +} + +export function throwQaGatewayChildFailure( + getChildFailure: (() => QaChildFailure | null) | undefined, + logs: () => string, +) { + const failure = getChildFailure?.(); + if (!failure) { + return; + } + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `${formatQaGatewayChildFailure(failure)}\n${logs()}`, + { cause: failure.error }, + ); +} + +export function monitorQaGatewayChildFailure( + child: ChildProcess, + output: { push(source: QaGatewayChildLogSource, chunk: Buffer): void }, +) { + let childFailure: QaChildFailure | null = null; + monitorQaChildFailure(child, (failure) => { + childFailure = failure; + const description = + failure.source === "process" + ? `gateway child process error: ${formatErrorMessage(failure.error)}` + : formatQaGatewayChildFailure(failure); + output.push("internal", Buffer.from(`[qa-lab] ${description}\n`)); + if (failure.source !== "process" && !hasQaGatewayChildExited(child)) { + // A broken parent-side pipe means QA can no longer observe the Gateway. + // Stop the detached process tree so the existing lifecycle reports the failure. + signalQaGatewayChildProcessTree(child, "SIGTERM"); + } + }); + return () => childFailure; +} + +export function formatQaGatewayProcessBoundaryStartupFailure(error: unknown, logs: string) { + const logTail = sliceUtf16Safe( + redactQaGatewayDebugText(logs), + -QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS, + ); + return `${formatErrorMessage(error)}${formatQaGatewayLogsForError(logTail)}`; +} + +function isProcessAlreadyExitedError(error: unknown): boolean { + return (error as NodeJS.ErrnoException | undefined)?.code === "ESRCH"; +} + +function boundQaGatewayProcessTreeDiagnostics(details: string) { + if (details.length <= 2_048) { + return details; + } + return `${sliceUtf16Safe(details, 0, 2_045)}...`; +} + +function isQaGatewayChildProcessTreeAlive( + child: ChildProcess, + inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector = inspectLinuxProcessGroup, +) { + if (!child.pid) { + return false; + } + if (process.platform === "win32") { + return !hasQaGatewayChildExited(child); + } + try { + process.kill(-child.pid, 0); + if (process.platform === "linux") { + // Linux can retain zombie-only process groups after SIGKILL while Node's + // child metadata is still unsettled. Runnable /proc members are the owner. + return inspectLinuxProcessGroupFn(child.pid)?.alive ?? true; + } + return true; + } catch (error) { + if (!isProcessAlreadyExitedError(error) && !hasQaGatewayChildExited(child)) { + return true; + } + } + return false; +} + +function signalQaGatewayChildProcessTree(child: ChildProcess, signal: NodeJS.Signals) { + if (!child.pid) { + return; + } + try { + if (process.platform === "win32") { + if (runQaWindowsTaskkill({ pid: child.pid, signal })) { + return; + } + child.kill(signal); + return; + } + process.kill(-child.pid, signal); + } catch { + try { + child.kill(signal); + } catch { + // The child already exited. + } + } +} + +async function waitForQaGatewayChildExit( + child: ChildProcess, + timeoutMs: number, + inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector, +) { + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) { + return true; + } + await sleep(Math.min(25, Math.max(0, deadline - Date.now()))); + } + return !isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn); +} + +type QaGatewayChildStopOptions = { + gracefulTimeoutMs?: number; + forceTimeoutMs?: number; + inspectLinuxProcessGroup?: QaLinuxProcessGroupInspector; +}; + +function resolveQaGatewayChildStopTimeouts(opts?: QaGatewayChildStopOptions) { + return { + gracefulTimeoutMs: opts?.gracefulTimeoutMs ?? QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS, + forceTimeoutMs: opts?.forceTimeoutMs ?? QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS, + }; +} + +function formatQaGatewayProcessTreeDiagnostics( + child: ChildProcess, + inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector, +) { + const childExitRecorded = hasQaGatewayChildExited(child); + if (process.platform !== "linux" || !child.pid) { + return `pid=${child.pid ?? "unknown"} childExitRecorded=${childExitRecorded}`; + } + const inspection = inspectLinuxProcessGroupFn(child.pid); + const processGroupDetails = + inspection?.diagnostics ?? `pgid=${child.pid} members=unknown (/proc unavailable)`; + return boundQaGatewayProcessTreeDiagnostics( + `${processGroupDetails} childExitRecorded=${childExitRecorded}`, + ); +} + +export async function stopQaGatewayChildProcessTree( + child: ChildProcess, + opts?: QaGatewayChildStopOptions, +) { + const inspectLinuxProcessGroupFn = opts?.inspectLinuxProcessGroup ?? inspectLinuxProcessGroup; + if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) { + return; + } + const timeouts = resolveQaGatewayChildStopTimeouts(opts); + signalQaGatewayChildProcessTree(child, "SIGTERM"); + if ( + await waitForQaGatewayChildExit(child, timeouts.gracefulTimeoutMs, inspectLinuxProcessGroupFn) + ) { + return; + } + signalQaGatewayChildProcessTree(child, "SIGKILL"); + const stopped = await waitForQaGatewayChildExit( + child, + timeouts.forceTimeoutMs, + inspectLinuxProcessGroupFn, + ); + if (!stopped) { + throw new Error( + `qa gateway process tree remained alive after forced shutdown: ${formatQaGatewayProcessTreeDiagnostics( + child, + inspectLinuxProcessGroupFn, + )}`, + ); + } +} diff --git a/extensions/qa-lab/src/gateway-child-readiness.ts b/extensions/qa-lab/src/gateway-child-readiness.ts new file mode 100644 index 000000000000..054b98935275 --- /dev/null +++ b/extensions/qa-lab/src/gateway-child-readiness.ts @@ -0,0 +1,269 @@ +// Qa Lab plugin module owns gateway readiness and retry behavior. +import { setTimeout as sleep } from "node:timers/promises"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; +import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; +import { QaSuiteInfraError } from "./errors.js"; +import { + hasQaGatewayChildExited, + type QaChildFailure, + throwQaGatewayChildFailure, +} from "./gateway-child-process.js"; +import { formatQaGatewayLogsForError } from "./gateway-log-redaction.js"; + +export const QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS = 5; +const QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS = 90_000; +const QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX = + "OpenClaw plugin migration inputs changed during startup convergence;"; + +type QaGatewayStartupRetryKind = "bind-collision" | "migration-convergence-restart"; + +type QaGatewayHealthChild = { + exitCode: number | null; + signalCode: NodeJS.Signals | null; +}; + +function classifyQaGatewayStartupRetry(details: string): QaGatewayStartupRetryKind | null { + if (details.includes(QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX)) { + return "migration-convergence-restart"; + } + if ( + details.includes("another gateway instance is already listening on ws://") || + details.includes("failed to bind gateway socket on ws://") || + details.includes("EADDRINUSE") || + details.includes("address already in use") + ) { + return "bind-collision"; + } + return null; +} + +export function resolveQaGatewayStartupRetry(params: { + attempt: number; + details: string; + migrationConvergenceRestartUsed: boolean; +}) { + if (params.attempt >= QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS) { + return null; + } + const kind = classifyQaGatewayStartupRetry(params.details); + if ( + !kind || + (kind === "migration-convergence-restart" && params.migrationConvergenceRestartUsed) + ) { + return null; + } + return { + kind, + reuseLaunchState: kind === "migration-convergence-restart", + migrationConvergenceRestartUsed: + params.migrationConvergenceRestartUsed || kind === "migration-convergence-restart", + }; +} + +function isRetryableGatewayCallError(details: string): boolean { + return ( + details.includes("handshake timeout") || + details.includes("gateway closed (1000") || + details.includes("gateway closed (1012)") || + details.includes("gateway closed (1006") || + details.includes("abnormal closure") || + details.includes("service restart") + ); +} + +export async function callQaGatewayWithRetry(params: { + deadlineMs?: number; + logs: () => string; + request: (options: { deadlineMs?: number; timeoutMs: number }) => Promise; + throwChildFailure: () => void; + timeoutMs: number; + waitForReady: (timeoutMs: number) => Promise; +}) { + const remainingMs = () => + params.deadlineMs === undefined ? undefined : params.deadlineMs - Date.now(); + const deadlineError = () => + new Error(`gateway call deadline exceeded${formatQaGatewayLogsForError(params.logs())}`); + let lastDetails = ""; + for (let attempt = 1; attempt <= 3; attempt += 1) { + params.throwChildFailure(); + const requestRemainingMs = remainingMs(); + if (requestRemainingMs !== undefined && requestRemainingMs <= 0) { + throw deadlineError(); + } + try { + return await params.request({ + ...(params.deadlineMs === undefined ? {} : { deadlineMs: params.deadlineMs }), + timeoutMs: + requestRemainingMs === undefined + ? params.timeoutMs + : Math.min(params.timeoutMs, requestRemainingMs), + }); + } catch (error) { + params.throwChildFailure(); + const details = formatErrorMessage(error); + lastDetails = details; + if (attempt >= 3 || !isRetryableGatewayCallError(details)) { + throw new Error(`${details}${formatQaGatewayLogsForError(params.logs())}`, { + cause: error, + }); + } + const readinessRemainingMs = remainingMs(); + if (readinessRemainingMs !== undefined && readinessRemainingMs <= 0) { + throw deadlineError(); + } + await params.waitForReady( + readinessRemainingMs === undefined + ? Math.max(10_000, params.timeoutMs) + : Math.min(Math.max(10_000, params.timeoutMs), readinessRemainingMs), + ); + } + } + throw new Error(`${lastDetails}${formatQaGatewayLogsForError(params.logs())}`); +} + +async function fetchLocalGatewayHealth(params: { + baseUrl: string; + healthPath: "/readyz" | "/healthz"; + timeoutMs?: number; +}): Promise { + const { response, release } = await fetchWithSsrFGuard({ + url: `${params.baseUrl}${params.healthPath}`, + init: { + method: "HEAD", + headers: { + connection: "close", + }, + signal: AbortSignal.timeout(params.timeoutMs ?? 2_000), + }, + policy: { allowPrivateNetwork: true }, + auditContext: "qa-lab-gateway-child-health", + }); + try { + return response.ok; + } finally { + await release(); + } +} + +async function fetchLocalGatewayListening(baseUrl: string): Promise { + const { release } = await fetchWithSsrFGuard({ + url: `${baseUrl}/healthz`, + init: { + method: "HEAD", + headers: { + connection: "close", + }, + signal: AbortSignal.timeout(2_000), + }, + policy: { allowPrivateNetwork: true }, + auditContext: "qa-lab-gateway-child-listening", + }); + await release(); + return true; +} + +export async function waitForQaGatewayRestartBoundary(params: { + readLogsSince: (mark: number) => string; + mark: number; + pollMs?: number; + timeoutMs?: number; +}) { + const timeoutMs = params.timeoutMs ?? QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS; + const pollMs = resolveTimerTimeoutMs(params.pollMs ?? 100, 100, 0); + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (params.readLogsSince(params.mark).includes("restart mode:")) { + return; + } + const remainingMs = timeoutMs - (Date.now() - startedAt); + if (remainingMs <= 0) { + break; + } + await sleep(Math.min(pollMs, remainingMs)); + } + throw new Error(`qa gateway child did not reach restart boundary within ${timeoutMs}ms`); +} + +export async function waitForGatewayReady(params: { + baseUrl: string; + logs: () => string; + child: QaGatewayHealthChild; + getChildFailure?: () => QaChildFailure | null; + timeoutMs?: number; +}) { + const deadline = Date.now() + (params.timeoutMs ?? 60_000); + let remainingMs: number; + while ((remainingMs = deadline - Date.now()) > 0) { + throwQaGatewayChildFailure(params.getChildFailure, params.logs); + if (hasQaGatewayChildExited(params.child)) { + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `gateway exited before becoming healthy (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`, + ); + } + // Listener liveness can turn green before the Gateway can admit startup or restart work. + try { + if ( + await fetchLocalGatewayHealth({ + baseUrl: params.baseUrl, + healthPath: "/readyz", + timeoutMs: Math.min(2_000, remainingMs), + }) + ) { + return; + } + } catch { + // retry until timeout + } + await sleep(Math.min(250, Math.max(0, deadline - Date.now()))); + } + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `gateway failed to become healthy:\n${params.logs()}`, + ); +} + +export async function waitForGatewayListening(params: { + baseUrl: string; + logs: () => string; + child: QaGatewayHealthChild; + getChildFailure?: () => QaChildFailure | null; + timeoutMs?: number; +}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) { + throwQaGatewayChildFailure(params.getChildFailure, params.logs); + if (params.child.exitCode !== null || params.child.signalCode !== null) { + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `gateway exited before listening (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`, + ); + } + try { + if (await fetchLocalGatewayListening(params.baseUrl)) { + return; + } + } catch { + // retry until the HTTP listener accepts requests + } + await sleep(100); + } + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `gateway failed to listen before timeout:\n${params.logs()}`, + ); +} + +export function isRetryableRpcStartupError(error: unknown) { + const details = formatErrorMessage(error); + return ( + details.includes("gateway timeout after") || + details.includes("handshake timeout") || + details.includes("gateway token mismatch") || + details.includes("token mismatch") || + details.includes("gateway closed (1000") || + details.includes("gateway closed (1006") || + details.includes("gateway closed (1012)") + ); +} diff --git a/extensions/qa-lab/src/gateway-child.test.ts b/extensions/qa-lab/src/gateway-child.test.ts index cae7bbcdba8e..ca4a273f9190 100644 --- a/extensions/qa-lab/src/gateway-child.test.ts +++ b/extensions/qa-lab/src/gateway-child.test.ts @@ -9,12 +9,41 @@ import { pathToFileURL } from "node:url"; import { toErrorObject } from "openclaw/plugin-sdk/error-runtime"; import { afterEach, describe, expect, it, vi } from "vitest"; import { - testing, + createQaBundledPluginsDir, + resolveQaOwnerPluginIdsForProviderIds, + resolveQaRuntimeHostVersion, +} from "./bundled-plugin-staging.js"; +import { preserveQaGatewayDebugArtifacts } from "./gateway-child-artifacts.js"; +import { resolveQaGatewayChildCommand, runQaGatewayCliCommand } from "./gateway-child-command.js"; +import { + buildQaForcedRuntimeEnvPatch, buildQaRuntimeEnv, - resolveQaControlUiRoot, - startQaGatewayChild, -} from "./gateway-child.js"; + stageQaCodexMockModelCatalog, +} from "./gateway-child-env.js"; +import { + closeQaGatewayLogStream, + createQaGatewayChildLogCollector, + formatQaGatewayProcessBoundaryStartupFailure, + monitorQaGatewayChildFailure, + stopQaGatewayChildProcessTree, + throwQaGatewayChildFailure, +} from "./gateway-child-process.js"; +import { + callQaGatewayWithRetry, + isRetryableRpcStartupError, + resolveQaGatewayStartupRetry, + waitForGatewayReady, + waitForQaGatewayRestartBoundary, +} from "./gateway-child-readiness.js"; +import { startQaGatewayChild } from "./gateway-child.js"; +import { readQaLiveProviderConfigOverrides } from "./providers/live-config.js"; +import { + assertQaLiveCodexAuthAvailable, + stageQaLiveAnthropicSetupToken, + stageQaLiveApiKeyProfiles, +} from "./providers/live-frontier/auth.js"; import { readQaAuthProfiles } from "./providers/shared/auth-store.js"; +import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js"; import { createTempDirHarness } from "./temp-dir.test-helper.js"; const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn()); @@ -105,16 +134,6 @@ function requireSsrFetchCall(index = 0): SsrFetchCall { return call[0] as SsrFetchCall; } -async function expectPathMissing(filePath: string): Promise { - try { - await lstat(filePath); - } catch (error) { - expect((error as NodeJS.ErrnoException).code).toBe("ENOENT"); - return; - } - throw new Error(`expected ${filePath} to be missing`); -} - async function writeJsonFixture(filePath: string, value: unknown, space?: number) { await mkdir(path.dirname(filePath), { recursive: true }); await writeFile(filePath, JSON.stringify(value, null, space), "utf8"); @@ -187,7 +206,7 @@ async function readJsonLines(filePath: string): Promise { it("runs CLI commands with the Gateway fixture environment", async () => { - const output = await testing.runQaGatewayCliCommand({ + const output = await runQaGatewayCliCommand({ executablePath: process.execPath, argsPrefix: [ "--eval", @@ -203,7 +222,7 @@ describe("runQaGatewayCliCommand", () => { it("reports CLI stderr when a fixture command fails", async () => { await expect( - testing.runQaGatewayCliCommand({ + runQaGatewayCliCommand({ executablePath: process.execPath, argsPrefix: ["--eval", 'process.stderr.write("fixture failure"); process.exit(7)'], args: [], @@ -212,25 +231,6 @@ describe("runQaGatewayCliCommand", () => { }), ).rejects.toThrow("OpenClaw CLI exited 7: fixture failure"); }); - - it.each(["stdout", "stderr"] as const)( - "rejects and stops the CLI child when its %s pipe fails", - async (streamName) => { - const child = spawn(process.execPath, ["--eval", "setInterval(() => {}, 1000)"], { - stdio: ["ignore", "pipe", "pipe"], - }); - const close = once(child, "close"); - const result = testing.readQaGatewayCliCommand(child); - const message = `synthetic ${streamName} read failure`; - - child[streamName]?.destroy(new Error(message)); - - await expect(result).rejects.toThrow( - `qa gateway cli ${streamName} stream failed: ${message}`, - ); - await close; - }, - ); }); describe("monitorQaGatewayChildFailure", () => { @@ -240,8 +240,8 @@ describe("monitorQaGatewayChildFailure", () => { stdio: ["ignore", "pipe", "pipe"], }); const close = once(child, "close"); - const output = testing.createQaGatewayChildLogCollector(); - const getFailure = testing.monitorQaGatewayChildFailure(child, output); + const output = createQaGatewayChildLogCollector(); + const getFailure = monitorQaGatewayChildFailure(child, output); const error = new Error("synthetic gateway stdout read failure"); child.stdout?.destroy(error); @@ -253,7 +253,7 @@ describe("monitorQaGatewayChildFailure", () => { "gateway child stdout stream failed: synthetic gateway stdout read failure", ); expect(output.text()).not.toContain("later stderr read failure"); - expect(() => testing.throwQaGatewayChildFailure(getFailure, () => output.text())).toThrow( + expect(() => throwQaGatewayChildFailure(getFailure, () => output.text())).toThrow( "gateway child stdout stream failed: synthetic gateway stdout read failure", ); }); @@ -263,7 +263,7 @@ describe("formatQaGatewayProcessBoundaryStartupFailure", () => { it("includes only a bounded, redacted launcher log tail", () => { const prefix = "x".repeat(9_000); const longSecret = "s".repeat(9_000); - const message = testing.formatQaGatewayProcessBoundaryStartupFailure( + const message = formatQaGatewayProcessBoundaryStartupFailure( new Error("launcher exited before identity"), `${prefix}\nAuthorization: Bearer ${longSecret}\nlauncher stage=mount-proc`, ); @@ -277,7 +277,7 @@ describe("formatQaGatewayProcessBoundaryStartupFailure", () => { }); it("preserves complete Unicode code points at the retained log-tail boundary", () => { - const message = testing.formatQaGatewayProcessBoundaryStartupFailure( + const message = formatQaGatewayProcessBoundaryStartupFailure( new Error("launcher exited before identity"), `P😀${"z".repeat(8_191)}`, ); @@ -304,7 +304,7 @@ describe("waitForGatewayReady", () => { }); try { - const readiness = testing.waitForGatewayReady({ + const readiness = waitForGatewayReady({ baseUrl, logs: () => `${phase} logs`, child: { exitCode: null, signalCode: null }, @@ -316,6 +316,11 @@ describe("waitForGatewayReady", () => { expect(fetchWithSsrFGuardMock.mock.calls.map(([request]) => request.url)).toEqual([ `${baseUrl}/readyz`, ]); + const healthRequest = requireSsrFetchCall(); + expect(healthRequest.init?.method).toBe("HEAD"); + expect(healthRequest.init?.headers).toEqual({ connection: "close" }); + expect(healthRequest.policy).toEqual({ allowPrivateNetwork: true }); + expect(healthRequest.auditContext).toBe("qa-lab-gateway-child-health"); expect(release).toHaveBeenCalledTimes(1); ready = true; @@ -349,7 +354,7 @@ describe("waitForGatewayReady", () => { const startedAt = Date.now(); await expect( - testing.waitForGatewayReady({ + waitForGatewayReady({ baseUrl: "http://127.0.0.1:43124", logs: () => "near-expiry logs", child: { exitCode: null, signalCode: null }, @@ -363,16 +368,9 @@ describe("waitForGatewayReady", () => { }); describe("Gateway child fixture helpers", () => { - it("creates an empty transport config seam", () => { - expect(testing.createQaGatewayEmptyTransport()).toEqual({ - requiredPluginIds: [], - createGatewayConfig: expect.any(Function), - }); - }); - it("stages native Codex model metadata before starting the private mock runtime", async () => { const tempRoot = await tempDirs.makeTempDir("qa-codex-model-catalog-"); - const modelCatalogPath = await testing.stageQaCodexMockModelCatalog({ + const modelCatalogPath = await stageQaCodexMockModelCatalog({ tempRoot, forcedRuntime: "codex", providerMode: "mock-openai", @@ -388,16 +386,19 @@ describe("Gateway child fixture helpers", () => { expect.objectContaining({ slug: "gpt-5.6-luna", apply_patch_tool_type: "freeform", + supports_reasoning_summary_parameter: true, tool_mode: "direct", }), expect.objectContaining({ slug: "gpt-5.6-luna-alt", apply_patch_tool_type: "freeform", + supports_reasoning_summary_parameter: true, tool_mode: "direct", }), ]); + expect(catalog.models[0]).not.toHaveProperty("supports_reasoning_summaries"); expect( - testing.buildQaForcedRuntimeEnvPatch({ + buildQaForcedRuntimeEnvPatch({ forcedRuntime: "codex", providerMode: "mock-openai", providerBaseUrl: "http://127.0.0.1:44080/v1", @@ -413,14 +414,14 @@ describe("Gateway child fixture helpers", () => { it("does not stage a Codex catalog for other runtimes or live providers", async () => { const tempRoot = await tempDirs.makeTempDir("qa-codex-model-catalog-unused-"); await expect( - testing.stageQaCodexMockModelCatalog({ + stageQaCodexMockModelCatalog({ tempRoot, forcedRuntime: "openclaw", providerMode: "mock-openai", }), ).resolves.toBeUndefined(); await expect( - testing.stageQaCodexMockModelCatalog({ + stageQaCodexMockModelCatalog({ tempRoot, forcedRuntime: "codex", providerMode: "live-frontier", @@ -431,46 +432,13 @@ describe("Gateway child fixture helpers", () => { ).rejects.toThrow(); }); - it("confines live Codex QA without replacing its native provider configuration", () => { - expect( - testing.buildQaForcedRuntimeEnvPatch({ - forcedRuntime: "codex", - providerMode: "live-frontier", - }), - ).toEqual({ - OPENCLAW_BUILD_PRIVATE_QA: "1", - OPENCLAW_QA_FORCE_RUNTIME: "codex", - OPENCLAW_CODEX_APP_SERVER_ARGS: - "app-server -c sandbox_workspace_write.exclude_tmpdir_env_var=true " + - "-c sandbox_workspace_write.exclude_slash_tmp=true --listen stdio://", - }); - }); - - it("preserves preconfigured live Codex arguments while enforcing QA containment", () => { - expect( - testing.buildQaForcedRuntimeEnvPatch({ - forcedRuntime: "codex", - providerMode: "live-frontier", - nativeAppServerArgs: - 'app-server -c openai_base_url="https://live.example/v1" --listen stdio://', - }), - ).toEqual({ - OPENCLAW_BUILD_PRIVATE_QA: "1", - OPENCLAW_QA_FORCE_RUNTIME: "codex", - OPENCLAW_CODEX_APP_SERVER_ARGS: - 'app-server -c openai_base_url="https://live.example/v1" --listen stdio:// ' + - "-c sandbox_workspace_write.exclude_tmpdir_env_var=true " + - "-c sandbox_workspace_write.exclude_slash_tmp=true", - }); - }); - it("resolves the repo runner before a built Gateway CLI fallback", async () => { const repoRoot = await tempDirs.makeTempDir("qa-gateway-command-"); await mkdir(path.join(repoRoot, "scripts"), { recursive: true }); const runnerPath = path.join(repoRoot, "scripts", "run-node.mjs"); await writeFile(runnerPath, "export {};\n", "utf8"); - expect(testing.resolveQaGatewayChildCommand(repoRoot)).toEqual({ + expect(resolveQaGatewayChildCommand(repoRoot)).toEqual({ executablePath: process.execPath, argsPrefix: [runnerPath], cwd: repoRoot, @@ -480,7 +448,7 @@ describe("Gateway child fixture helpers", () => { await mkdir(path.join(repoRoot, "dist"), { recursive: true }); await writeFile(path.join(repoRoot, "dist", "index.js"), "export {};\n", "utf8"); await rm(path.join(repoRoot, "scripts"), { recursive: true }); - expect(testing.resolveQaGatewayChildCommand(repoRoot)).toEqual({ + expect(resolveQaGatewayChildCommand(repoRoot)).toEqual({ executablePath: process.execPath, argsPrefix: [path.join(repoRoot, "dist", "index.js")], cwd: repoRoot, @@ -679,11 +647,6 @@ describe("buildQaRuntimeEnv", () => { expect(env.GEMINI_API_KEY).toBe("gemini-live"); }); - it("defaults gateway-child provider mode to mock-openai when omitted", () => { - expect(testing.resolveQaGatewayChildProviderMode(undefined)).toBe("mock-openai"); - expect(testing.resolveQaGatewayChildProviderMode("live-frontier")).toBe("live-frontier"); - }); - it("keeps explicit provider env vars over live aliases", () => { const env = buildQaRuntimeEnv({ ...createParams({ @@ -959,15 +922,6 @@ describe("buildQaRuntimeEnv", () => { }, ); - it("treats restart socket closures as retryable gateway call errors", () => { - expect(testing.isRetryableGatewayCallError("gateway closed (1006 abnormal closure)")).toBe( - true, - ); - expect(testing.isRetryableGatewayCallError("gateway closed (1012 service restart)")).toBe(true); - expect(testing.isRetryableGatewayCallError("service restart in progress")).toBe(true); - expect(testing.isRetryableGatewayCallError("permission denied")).toBe(false); - }); - it("preserves relative gateway retry timeouts without an absolute deadline", async () => { const request = vi .fn() @@ -976,7 +930,7 @@ describe("buildQaRuntimeEnv", () => { const waitForReady = vi.fn(async () => {}); await expect( - testing.callQaGatewayWithRetry({ + callQaGatewayWithRetry({ logs: () => "qa logs", request, throwChildFailure: vi.fn(), @@ -1002,7 +956,7 @@ describe("buildQaRuntimeEnv", () => { }); await expect( - testing.callQaGatewayWithRetry({ + callQaGatewayWithRetry({ deadlineMs: 10_000, logs: () => "qa logs", request, @@ -1021,7 +975,7 @@ describe("buildQaRuntimeEnv", () => { it("waits for a fresh in-process restart boundary after the current log offset", async () => { let logs = "old restart mode: in-process restart\n"; const mark = logs.length; - const wait = testing.waitForQaGatewayRestartBoundary({ + const wait = waitForQaGatewayRestartBoundary({ readLogsSince: (since) => logs.slice(since), mark, pollMs: 1, @@ -1034,11 +988,11 @@ describe("buildQaRuntimeEnv", () => { }); it("keeps restart offsets stable after stderr output", async () => { - const output = testing.createQaGatewayChildLogCollector(); + const output = createQaGatewayChildLogCollector(); output.push("stdout", Buffer.from("gateway ready\n")); output.push("stderr", Buffer.from("stderr warning\n")); const mark = output.mark(); - const wait = testing.waitForQaGatewayRestartBoundary({ + const wait = waitForQaGatewayRestartBoundary({ readLogsSince: (since) => output.readSince(since), mark, pollMs: 1, @@ -1054,7 +1008,7 @@ describe("buildQaRuntimeEnv", () => { }); it("bounds diagnostics while monotonic marks retain fresh output semantics", () => { - const output = testing.createQaGatewayChildLogCollector(); + const output = createQaGatewayChildLogCollector(); output.push("stdout", Buffer.from(`old😀${"x".repeat(70_000)}`)); const mark = output.mark(); output.push("stdout", Buffer.from("fresh restart mode: in-process restart\n")); @@ -1068,7 +1022,7 @@ describe("buildQaRuntimeEnv", () => { }); it("decodes interleaved stdout and stderr independently", () => { - const output = testing.createQaGatewayChildLogCollector(); + const output = createQaGatewayChildLogCollector(); const stdout = Buffer.from("before 😀 after\n"); output.push("stdout", stdout.subarray(0, 9)); @@ -1081,7 +1035,7 @@ describe("buildQaRuntimeEnv", () => { it("times out when a SIGUSR1 restart never reaches the boundary", async () => { await expect( - testing.waitForQaGatewayRestartBoundary({ + waitForQaGatewayRestartBoundary({ readLogsSince: () => "signal SIGUSR1 received\n", mark: 0, pollMs: 1, @@ -1092,7 +1046,7 @@ describe("buildQaRuntimeEnv", () => { it("keeps oversized restart-boundary poll intervals within the timeout", async () => { await expect( - testing.waitForQaGatewayRestartBoundary({ + waitForQaGatewayRestartBoundary({ readLogsSince: () => "signal SIGUSR1 received\n", mark: 0, pollMs: Number.MAX_SAFE_INTEGER, @@ -1105,7 +1059,7 @@ describe("buildQaRuntimeEnv", () => { const stateDir = await tempDirs.makeTempDir("qa-setup-token-state-"); const token = `sk-ant-oat01-${"c".repeat(80)}`; - const cfg = await testing.stageQaLiveAnthropicSetupToken({ + const cfg = await stageQaLiveAnthropicSetupToken({ cfg: {}, stateDir, env: { @@ -1128,7 +1082,7 @@ describe("buildQaRuntimeEnv", () => { it("stages live env API-key profiles for isolated QA workers", async () => { const stateDir = await tempDirs.makeTempDir("qa-live-api-key-state-"); - const cfg = await testing.stageQaLiveApiKeyProfiles({ + const cfg = await stageQaLiveApiKeyProfiles({ cfg: {}, stateDir, providerIds: ["openai"], @@ -1157,45 +1111,10 @@ describe("buildQaRuntimeEnv", () => { } }); - it("stages the OpenAI API-key fallback for live OpenAI QA workers", async () => { - const stateDir = await tempDirs.makeTempDir("qa-live-codex-api-key-state-"); - - const cfg = await testing.stageQaLiveApiKeyProfiles({ - cfg: {}, - stateDir, - providerIds: ["openai"], - env: { - OPENCLAW_LIVE_OPENAI_KEY: "qa-live-codex-fallback-key", - }, - }); - - for (const [profileId, provider] of [ - ["qa-live-openai-env", "openai"], - ["qa-live-openai-env", "openai"], - ] as const) { - const configProfile = requireAuthProfile(cfg.auth?.profiles, profileId); - expect(configProfile.provider).toBe(provider); - expect(configProfile.mode).toBe("api_key"); - } - - for (const agentId of ["main", "qa"]) { - const storeProfiles = readAuthProfileStore(stateDir, agentId).profiles; - for (const [profileId, provider] of [ - ["qa-live-openai-env", "openai"], - ["qa-live-openai-env", "openai"], - ] as const) { - const storeProfile = requireAuthProfile(storeProfiles, profileId); - expect(storeProfile.type).toBe("api_key"); - expect(storeProfile.provider).toBe(provider); - expect(storeProfile.key).toBe("qa-live-codex-fallback-key"); - } - } - }); - it("stages direct live OpenAI API-key aliases for isolated QA workers", async () => { const stateDir = await tempDirs.makeTempDir("qa-live-codex-direct-key-state-"); - const cfg = await testing.stageQaLiveApiKeyProfiles({ + const cfg = await stageQaLiveApiKeyProfiles({ cfg: {}, stateDir, providerIds: ["openai"], @@ -1213,7 +1132,7 @@ describe("buildQaRuntimeEnv", () => { expect(storeProfile.key).toBe("qa-live-direct-codex-key"); expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg, providerIds: ["openai"], env: { @@ -1226,20 +1145,7 @@ describe("buildQaRuntimeEnv", () => { it("fails fast when live OpenAI runs have no portable QA auth", () => { expect(() => - testing.assertQaLiveCodexAuthAvailable({ - cfg: {}, - providerIds: ["openai"], - env: { - CODEX_HOME: path.join(os.tmpdir(), "missing-openclaw-codex-home"), - }, - readCodexCredentials: () => null, - }), - ).toThrow("QA live-frontier cannot run Codex-backed OpenAI models"); - }); - - it("fails fast when default OpenAI model refs route through Codex without portable QA auth", () => { - expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg: {}, providerIds: ["openai"], env: { @@ -1252,7 +1158,7 @@ describe("buildQaRuntimeEnv", () => { it("does not require Codex auth for custom OpenAI-compatible provider configs", () => { expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg: { models: { providers: { @@ -1274,7 +1180,7 @@ describe("buildQaRuntimeEnv", () => { it("fails fast when forced Codex runtime uses OpenAI model refs without portable QA auth", () => { expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg: {}, providerIds: ["openai"], env: { @@ -1288,7 +1194,7 @@ describe("buildQaRuntimeEnv", () => { it("accepts OpenAI API-key fallback auth for forced Codex runtime QA runs", () => { expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg: {}, providerIds: ["openai"], env: { @@ -1302,7 +1208,7 @@ describe("buildQaRuntimeEnv", () => { it("stages configured OpenAI API keys for live QA runs", async () => { const stateDir = await tempDirs.makeTempDir("qa-live-codex-config-key-state-"); - const cfg = await testing.stageQaLiveApiKeyProfiles({ + const cfg = await stageQaLiveApiKeyProfiles({ cfg: { models: { providers: { @@ -1333,7 +1239,7 @@ describe("buildQaRuntimeEnv", () => { } expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg, providerIds: ["openai"], env: {}, @@ -1347,7 +1253,7 @@ describe("buildQaRuntimeEnv", () => { const env = { OPENCLAW_LIVE_CODEX_API_KEY: "qa-configured-env-ref-not-a-real-key", }; - const cfg = await testing.stageQaLiveApiKeyProfiles({ + const cfg = await stageQaLiveApiKeyProfiles({ cfg: { models: { providers: { @@ -1377,7 +1283,7 @@ describe("buildQaRuntimeEnv", () => { expect(storeProfile.key).toBe("qa-configured-env-ref-not-a-real-key"); expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg, providerIds: ["openai"], env, @@ -1388,7 +1294,7 @@ describe("buildQaRuntimeEnv", () => { it("stages configured OpenAI env markers for live QA runs", async () => { const stateDir = await tempDirs.makeTempDir("qa-live-codex-config-marker-state-"); - const cfg = await testing.stageQaLiveApiKeyProfiles({ + const cfg = await stageQaLiveApiKeyProfiles({ cfg: { models: { providers: { @@ -1416,7 +1322,7 @@ describe("buildQaRuntimeEnv", () => { expect(storeProfile.key).toBe("qa-configured-marker-not-a-real-key"); expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg, providerIds: ["openai"], env: {}, @@ -1435,7 +1341,7 @@ describe("buildQaRuntimeEnv", () => { })); expect(() => - testing.assertQaLiveCodexAuthAvailable({ + assertQaLiveCodexAuthAvailable({ cfg: {}, providerIds: ["openai"], env: { @@ -1454,7 +1360,7 @@ describe("buildQaRuntimeEnv", () => { it("stages placeholder mock auth profiles per agent dir so mock-openai runs can resolve credentials", async () => { const stateDir = await tempDirs.makeTempDir("qa-mock-auth-"); - const cfg = await testing.stageQaMockAuthProfiles({ + const cfg = await stageQaMockAuthProfiles({ cfg: {}, stateDir, }); @@ -1591,7 +1497,7 @@ describe("buildQaRuntimeEnv", () => { it("stages mock profiles only for the requested agents and providers when callers override the defaults", async () => { const stateDir = await tempDirs.makeTempDir("qa-mock-auth-override-"); - const cfg = await testing.stageQaMockAuthProfiles({ + const cfg = await stageQaMockAuthProfiles({ cfg: {}, stateDir, agentIds: ["qa"], @@ -1616,27 +1522,6 @@ describe("buildQaRuntimeEnv", () => { ).rejects.toThrow(/ENOENT/); }); - it("allows loopback gateway health probes through the SSRF guard", async () => { - const release = vi.fn(async () => {}); - fetchWithSsrFGuardMock.mockResolvedValue({ - response: { ok: true }, - release, - }); - - await expect( - testing.fetchLocalGatewayHealth({ - baseUrl: "http://127.0.0.1:18789", - healthPath: "/readyz", - }), - ).resolves.toBe(true); - - const request = requireSsrFetchCall(); - expect(request.url).toBe("http://127.0.0.1:18789/readyz"); - expect(request.policy).toEqual({ allowPrivateNetwork: true }); - expect(request.auditContext).toBe("qa-lab-gateway-child-health"); - expect(release).toHaveBeenCalledTimes(1); - }); - it("force-stops gateway children that ignore the graceful signal", async () => { const child = Object.assign(new EventEmitter(), { pid: 12345, @@ -1661,8 +1546,8 @@ describe("buildQaRuntimeEnv", () => { return true; }); - await testing.stopQaGatewayChildProcessTree( - child as unknown as Parameters[0], + await stopQaGatewayChildProcessTree( + child as unknown as Parameters[0], { gracefulTimeoutMs: 1, forceTimeoutMs: 10, @@ -1679,22 +1564,6 @@ describe("buildQaRuntimeEnv", () => { expect([child.exitCode, child.signalCode]).not.toEqual([null, null]); }); - it("lets the gateway finish its bounded shutdown before process-tree escalation", () => { - expect(testing.resolveQaGatewayChildStopTimeouts()).toEqual({ - gracefulTimeoutMs: 30_000, - forceTimeoutMs: 10_000, - }); - expect( - testing.resolveQaGatewayChildStopTimeouts({ - gracefulTimeoutMs: 1, - forceTimeoutMs: 2, - }), - ).toEqual({ - gracefulTimeoutMs: 1, - forceTimeoutMs: 2, - }); - }); - it("force-closes a gateway log stream whose final flush never settles", async () => { const stream = new Writable({ write(_chunk, _encoding, callback) { @@ -1706,7 +1575,7 @@ describe("buildQaRuntimeEnv", () => { }); const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - await testing.closeWriteStream(stream as never, "stdout", 1); + await closeQaGatewayLogStream(stream as never, "stdout", 1); expect(stream.destroyed).toBe(true); expect(stderr).toHaveBeenCalledWith( @@ -1726,7 +1595,7 @@ describe("buildQaRuntimeEnv", () => { vi.spyOn(process, "kill").mockImplementation(() => true); await expect( - testing.stopQaGatewayChildProcessTree(child as never, { + stopQaGatewayChildProcessTree(child as never, { gracefulTimeoutMs: 1, forceTimeoutMs: 1, inspectLinuxProcessGroup: () => null, @@ -1751,7 +1620,7 @@ describe("buildQaRuntimeEnv", () => { vi.spyOn(process, "kill").mockImplementation(() => true); try { await expect( - testing.stopQaGatewayChildProcessTree(child as never, { + stopQaGatewayChildProcessTree(child as never, { gracefulTimeoutMs: 1, forceTimeoutMs: 1, inspectLinuxProcessGroup: () => ({ @@ -1770,146 +1639,6 @@ describe("buildQaRuntimeEnv", () => { } }); - it("classifies Linux zombie-only process groups as stopped", () => { - const inspection = testing.inspectLinuxProcessGroupStats(123, [ - "123 (gateway child) Z 1 123 123 0 -1 0", - "124 (helper (worker)) X 1 123 123 0 -1 0", - "125 (unrelated) S 1 999 999 0 -1 0", - ]); - - expect(inspection).toEqual({ - alive: false, - diagnostics: - 'pgid=123 members=[pid=123 state=Z command="gateway child", pid=124 state=X command="helper (worker)"]', - }); - }); - - it("classifies Linux process groups with a runnable descendant as alive", () => { - const inspection = testing.inspectLinuxProcessGroupStats(123, [ - "123 (gateway child) Z 1 123 123 0 -1 0", - "126 (live helper) D 1 123 123 0 -1 0", - ]); - - expect(inspection).toEqual({ - alive: true, - diagnostics: - 'pgid=123 members=[pid=123 state=Z command="gateway child", pid=126 state=D command="live helper"]', - }); - }); - - it("classifies an empty Linux process-group snapshot as unknown", () => { - expect( - testing.inspectLinuxProcessGroupStats(123, ["125 (unrelated) S 1 999 999 0 -1 0"]), - ).toEqual({ - alive: null, - diagnostics: "pgid=123 members=[]", - }); - }); - - it("bounds Linux process-group diagnostics", () => { - const stats = Array.from( - { length: 300 }, - (_, index) => `${index + 1} (${`worker-${index}`.padEnd(32, "x")}) S 1 123 123 0 -1 0`, - ); - - const inspection = testing.inspectLinuxProcessGroupStats(123, stats); - - expect(inspection.alive).toBe(true); - expect(inspection.diagnostics.length).toBeLessThanOrEqual(2_048); - expect(inspection.diagnostics).toMatch(/\.\.\.$/u); - }); - - it("trusts Linux runnable-member inspection before child exit metadata settles", () => { - const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); - Object.defineProperty(process, "platform", { value: "linux", configurable: true }); - const processKill = vi.spyOn(process, "kill").mockImplementation(() => true); - const child = { - pid: 12345, - exitCode: null, - signalCode: null, - }; - try { - expect( - testing.isQaGatewayChildProcessTreeAlive(child as never, () => ({ - alive: false, - diagnostics: 'pgid=12345 members=[pid=12345 state=Z command="gateway"]', - })), - ).toBe(false); - expect( - testing.isQaGatewayChildProcessTreeAlive(child as never, () => ({ - alive: true, - diagnostics: 'pgid=12345 members=[pid=12346 state=S command="worker"]', - })), - ).toBe(true); - expect( - testing.isQaGatewayChildProcessTreeAlive(child as never, () => ({ - alive: null, - diagnostics: "pgid=12345 members=[]", - })), - ).toBe(true); - expect(testing.isQaGatewayChildProcessTreeAlive(child as never, () => null)).toBe(true); - expect(processKill).toHaveBeenCalledWith(-12345, 0); - } finally { - if (platformDescriptor) { - Object.defineProperty(process, "platform", platformDescriptor); - } - } - }); - - it("force-kills Windows gateway process trees when graceful taskkill fails", () => { - const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); - const originalSystemRoot = process.env.SystemRoot; - const originalWindir = process.env.WINDIR; - Object.defineProperty(process, "platform", { value: "win32", configurable: true }); - process.env.SystemRoot = "C:\\Windows"; - delete process.env.WINDIR; - try { - const child = Object.assign(new EventEmitter(), { - pid: 12345, - exitCode: null as number | null, - signalCode: null as string | null, - kill: vi.fn(), - }); - const runTaskkill = vi - .fn() - .mockReturnValueOnce({ status: 1 }) - .mockReturnValueOnce({ status: 0 }); - - testing.signalQaGatewayChildProcessTree( - child as unknown as Parameters[0], - "SIGTERM", - runTaskkill, - ); - - const taskkillPath = path.win32.join("C:\\Windows", "System32", "taskkill.exe"); - expect(runTaskkill).toHaveBeenNthCalledWith(1, taskkillPath, ["/PID", "12345", "/T"], { - stdio: "ignore", - windowsHide: true, - timeout: 5_000, - }); - expect(runTaskkill).toHaveBeenNthCalledWith(2, taskkillPath, ["/PID", "12345", "/T", "/F"], { - stdio: "ignore", - windowsHide: true, - timeout: 5_000, - }); - expect(child.kill).not.toHaveBeenCalled(); - } finally { - if (platformDescriptor) { - Object.defineProperty(process, "platform", platformDescriptor); - } - if (originalSystemRoot === undefined) { - delete process.env.SystemRoot; - } else { - process.env.SystemRoot = originalSystemRoot; - } - if (originalWindir === undefined) { - delete process.env.WINDIR; - } else { - process.env.WINDIR = originalWindir; - } - } - }); - it("does not trust an exited gateway wrapper while its process group is alive", async () => { const child = Object.assign(new EventEmitter(), { pid: 12346, @@ -1933,8 +1662,8 @@ describe("buildQaRuntimeEnv", () => { return true; }); - await testing.stopQaGatewayChildProcessTree( - child as unknown as Parameters[0], + await stopQaGatewayChildProcessTree( + child as unknown as Parameters[0], { gracefulTimeoutMs: 1, forceTimeoutMs: 50, @@ -1955,22 +1684,24 @@ describe("buildQaRuntimeEnv", () => { } }); - it("classifies bind collisions separately from migration convergence restarts", () => { + it.each([ + ["another gateway instance is already listening on ws://127.0.0.1:43124", "bind-collision"], + [ + "failed to bind gateway socket on ws://127.0.0.1:43124: Error: listen EADDRINUSE", + "bind-collision", + ], + [ + "OpenClaw plugin migration inputs changed during startup convergence; refusing to report the gateway ready. Restart OpenClaw so state migrations run against the final config and plugin inventory.", + "migration-convergence-restart", + ], + ] as const)("classifies %s", (details, expectedKind) => { expect( - testing.classifyQaGatewayStartupRetry( - "another gateway instance is already listening on ws://127.0.0.1:43124", - ), - ).toBe("bind-collision"); - expect( - testing.classifyQaGatewayStartupRetry( - "failed to bind gateway socket on ws://127.0.0.1:43124: Error: listen EADDRINUSE", - ), - ).toBe("bind-collision"); - expect( - testing.classifyQaGatewayStartupRetry( - "OpenClaw plugin migration inputs changed during startup convergence; refusing to report the gateway ready. Restart OpenClaw so state migrations run against the final config and plugin inventory.", - ), - ).toBe("migration-convergence-restart"); + resolveQaGatewayStartupRetry({ + attempt: 1, + details, + migrationConvergenceRestartUsed: false, + })?.kind, + ).toBe(expectedKind); }); it.each([ @@ -1979,11 +1710,17 @@ describe("buildQaRuntimeEnv", () => { "Restart OpenClaw so state migrations can continue.", "gateway failed to become healthy", ])("does not retry unrelated startup failure: %s", (details) => { - expect(testing.classifyQaGatewayStartupRetry(details)).toBeNull(); + expect( + resolveQaGatewayStartupRetry({ + attempt: 1, + details, + migrationConvergenceRestartUsed: false, + }), + ).toBeNull(); }); it("restarts migration convergence once with the same launch state", () => { - const first = testing.resolveQaGatewayStartupRetry({ + const first = resolveQaGatewayStartupRetry({ attempt: 1, details: "OpenClaw plugin migration inputs changed during startup convergence; refusing readiness.", @@ -1996,7 +1733,7 @@ describe("buildQaRuntimeEnv", () => { migrationConvergenceRestartUsed: true, }); expect( - testing.resolveQaGatewayStartupRetry({ + resolveQaGatewayStartupRetry({ attempt: 2, details: "OpenClaw plugin migration inputs changed during startup convergence; refusing readiness.", @@ -2007,7 +1744,7 @@ describe("buildQaRuntimeEnv", () => { it("rotates launch state only for a bind collision", () => { expect( - testing.resolveQaGatewayStartupRetry({ + resolveQaGatewayStartupRetry({ attempt: 1, details: "listen EADDRINUSE: address already in use", migrationConvergenceRestartUsed: false, @@ -2021,14 +1758,14 @@ describe("buildQaRuntimeEnv", () => { it("fails immediately for generic exits and after the startup attempt budget", () => { expect( - testing.resolveQaGatewayStartupRetry({ + resolveQaGatewayStartupRetry({ attempt: 1, details: "gateway exited with code 1", migrationConvergenceRestartUsed: false, }), ).toBeNull(); expect( - testing.resolveQaGatewayStartupRetry({ + resolveQaGatewayStartupRetry({ attempt: 5, details: "listen EADDRINUSE", migrationConvergenceRestartUsed: false, @@ -2038,35 +1775,11 @@ describe("buildQaRuntimeEnv", () => { it("treats startup token mismatches as retryable rpc startup errors", () => { expect( - testing.isRetryableRpcStartupError( + isRetryableRpcStartupError( "unauthorized: gateway token mismatch (set gateway.remote.token to match gateway.auth.token)", ), ).toBe(true); - expect(testing.isRetryableRpcStartupError("permission denied")).toBe(false); - }); - - it("probes gateway health with a one-shot HEAD request through the SSRF guard", async () => { - const release = vi.fn(async () => {}); - fetchWithSsrFGuardMock.mockResolvedValue({ - response: { ok: true }, - release, - }); - - await expect( - testing.fetchLocalGatewayHealth({ - baseUrl: "http://127.0.0.1:43124", - healthPath: "/readyz", - }), - ).resolves.toBe(true); - - const request = requireSsrFetchCall(); - expect(request.url).toBe("http://127.0.0.1:43124/readyz"); - expect(request.init?.method).toBe("HEAD"); - expect(request.init?.headers).toEqual({ connection: "close" }); - expect(request.init?.signal).toBeInstanceOf(AbortSignal); - expect(request.policy).toEqual({ allowPrivateNetwork: true }); - expect(request.auditContext).toBe("qa-lab-gateway-child-health"); - expect(release).toHaveBeenCalledTimes(1); + expect(isRetryableRpcStartupError("permission denied")).toBe(false); }); it("preserves only sanitized gateway debug artifacts", async () => { @@ -2110,7 +1823,7 @@ describe("buildQaRuntimeEnv", () => { await mkdir(path.join(tempRoot, "state"), { recursive: true }); await writeFile(path.join(tempRoot, "state", "secret.txt"), "do-not-copy", "utf8"); - await testing.preserveQaGatewayDebugArtifacts({ + await preserveQaGatewayDebugArtifacts({ preserveToDir: artifactDir, stdoutLogPath, stderrLogPath, @@ -2156,143 +1869,9 @@ describe("buildQaRuntimeEnv", () => { tempRoot, ); }); - - it("rejects preserved gateway artifacts outside the repo root", async () => { - await expect( - testing.assertQaArtifactDirWithinRepo("/tmp/openclaw-repo", "/tmp/outside"), - ).rejects.toThrow("QA gateway artifact directory must stay within the repo root."); - }); - - it("rejects preserved gateway artifacts that traverse symlinks", async () => { - const repoRoot = await tempDirs.makeTempDir("qa-gateway-guard-repo-"); - const outsideRoot = await tempDirs.makeTempDir("qa-gateway-guard-outside-"); - await mkdir(path.join(repoRoot, ".artifacts"), { recursive: true }); - await symlink(outsideRoot, path.join(repoRoot, ".artifacts", "qa-e2e"), "dir"); - - await expect( - testing.assertQaArtifactDirWithinRepo( - repoRoot, - path.join(repoRoot, ".artifacts", "qa-e2e", "gateway-runtime"), - ), - ).rejects.toThrow("QA gateway artifact directory must not traverse symlinks."); - }); - - it("cleans startup temp roots when they are not preserved", async () => { - const tempRoot = await tempDirs.makeTempDir("qa-gateway-cleanup-src-"); - const stagedRoot = await tempDirs.makeTempDir("qa-gateway-cleanup-stage-"); - - await writeFile(path.join(tempRoot, "openclaw.json"), "{}", "utf8"); - await writeFile(path.join(stagedRoot, "marker.txt"), "x", "utf8"); - - await testing.cleanupQaGatewayTempRoots({ - tempRoot, - stagedBundledPluginsRoot: stagedRoot, - }); - - await expectPathMissing(tempRoot); - await expectPathMissing(stagedRoot); - }); -}); - -describe("resolveQaControlUiRoot", () => { - it("returns the built control ui root when repo assets exist", async () => { - const repoRoot = await tempDirs.makeTempDir("qa-control-ui-root-"); - const controlUiRoot = path.join(repoRoot, "dist", "control-ui"); - await mkdir(controlUiRoot, { recursive: true }); - await writeFile(path.join(controlUiRoot, "index.html"), "", "utf8"); - - expect(resolveQaControlUiRoot({ repoRoot })).toBe(controlUiRoot); - }); - - it("returns undefined when control ui is disabled or not built", async () => { - const repoRoot = await tempDirs.makeTempDir("qa-control-ui-root-missing-"); - - expect(resolveQaControlUiRoot({ repoRoot })).toBeUndefined(); - expect(resolveQaControlUiRoot({ repoRoot, controlUiEnabled: false })).toBeUndefined(); - }); }); describe("qa bundled plugin dir", () => { - it("prefers a built bundled plugin when present", async () => { - const repoRoot = await tempDirs.makeTempDir("qa-bundled-root-"); - await writeJsonFixture( - path.join(repoRoot, "dist", "extensions", "qa-channel", "package.json"), - {}, - ); - await writeJsonFixture( - path.join(repoRoot, "dist-runtime", "extensions", "qa-channel", "package.json"), - {}, - ); - await writeJsonFixture(path.join(repoRoot, "extensions", "qa-channel", "package.json"), {}); - - expect( - testing.resolveQaBundledPluginSourceDir({ - repoRoot, - pluginId: "qa-channel", - }), - ).toBe(path.join(repoRoot, "dist", "extensions", "qa-channel")); - }); - - it("falls back to the source bundled plugin when no built copy exists", async () => { - const repoRoot = await tempDirs.makeTempDir("qa-bundled-source-root-"); - await writeJsonFixture(path.join(repoRoot, "extensions", "qa-channel", "package.json"), {}); - - expect( - testing.resolveQaBundledPluginSourceDir({ - repoRoot, - pluginId: "qa-channel", - }), - ).toBe(path.join(repoRoot, "extensions", "qa-channel")); - }); - - it("resolves bundled plugins by manifest id when the directory name differs", async () => { - const repoRoot = await tempDirs.makeTempDir("qa-bundled-manifest-id-root-"); - await writeJsonFixture( - path.join(repoRoot, "dist", "extensions", "kimi-coding", "openclaw.plugin.json"), - { id: "kimi", providers: ["kimi"] }, - ); - await writeJsonFixture( - path.join(repoRoot, "dist", "extensions", "kimi-coding", "package.json"), - {}, - ); - - expect( - testing.resolveQaBundledPluginSourceDir({ - repoRoot, - pluginId: "kimi", - }), - ).toBe(path.join(repoRoot, "dist", "extensions", "kimi-coding")); - }); - - it("uses a source bundled plugin when the built copy is missing CLI metadata", async () => { - const repoRoot = await tempDirs.makeTempDir("qa-bundled-cli-metadata-root-"); - await writeJsonFixture( - path.join(repoRoot, "dist", "extensions", "memory-core", "package.json"), - {}, - ); - await writeJsonFixture( - path.join(repoRoot, "dist", "extensions", "memory-core", "openclaw.plugin.json"), - { id: "memory-core", kind: "memory" }, - ); - await writeJsonFixture(path.join(repoRoot, "extensions", "memory-core", "package.json"), {}); - await writeJsonFixture( - path.join(repoRoot, "extensions", "memory-core", "openclaw.plugin.json"), - { id: "memory-core", kind: "memory" }, - ); - await writeFile( - path.join(repoRoot, "extensions", "memory-core", "cli-metadata.ts"), - "export default { id: 'memory-core' };\n", - "utf8", - ); - - expect( - testing.resolveQaBundledPluginSourceDir({ - repoRoot, - pluginId: "memory-core", - }), - ).toBe(path.join(repoRoot, "extensions", "memory-core")); - }); - it("creates a scoped bundled plugin tree for allowed plugins plus always-allowed runtime facades", async () => { const repoRoot = await tempDirs.makeTempDir("qa-bundled-scope-"); await writeFile( @@ -2350,7 +1929,7 @@ describe("qa bundled plugin dir", () => { await writeFile(path.join(repoRoot, "dist", "shared-chunk-abc123.js"), "export {};\n", "utf8"); const tempRoot = await tempDirs.makeTempDir("qa-bundled-target-"); - const { bundledPluginsDir, stagedRoot } = await testing.createQaBundledPluginsDir({ + const { bundledPluginsDir, stagedRoot } = await createQaBundledPluginsDir({ repoRoot, tempRoot, allowedPluginIds: ["qa-channel", "memory-core"], @@ -2439,7 +2018,7 @@ describe("qa bundled plugin dir", () => { ); const tempRoot = await tempDirs.makeTempDir("qa-bundled-mixed-target-"); - const { bundledPluginsDir } = await testing.createQaBundledPluginsDir({ + const { bundledPluginsDir } = await createQaBundledPluginsDir({ repoRoot, tempRoot, allowedPluginIds: ["runtime-only"], @@ -2486,7 +2065,7 @@ describe("qa bundled plugin dir", () => { const tempRoot = await tempDirs.makeTempDir("qa-bundled-invalid-target-"); await expect( - testing.createQaBundledPluginsDir({ + createQaBundledPluginsDir({ repoRoot, tempRoot, allowedPluginIds: ["../escape"], @@ -2503,7 +2082,7 @@ describe("qa bundled plugin dir", () => { ); const tempRoot = await tempDirs.makeTempDir("qa-bundled-external-target-"); - const { bundledPluginsDir } = await testing.createQaBundledPluginsDir({ + const { bundledPluginsDir } = await createQaBundledPluginsDir({ repoRoot, tempRoot, allowedPluginIds: ["external-fixture"], @@ -2570,7 +2149,7 @@ describe("qa bundled plugin dir", () => { await symlink(fakeDepPackageDir, path.join(repoRoot, "node_modules", "fake-dep"), "dir"); const tempRoot = await tempDirs.makeTempDir("qa-bundled-source-target-"); - const { bundledPluginsDir, stagedRoot } = await testing.createQaBundledPluginsDir({ + const { bundledPluginsDir, stagedRoot } = await createQaBundledPluginsDir({ repoRoot, tempRoot, allowedPluginIds: ["qa-channel"], @@ -2615,7 +2194,7 @@ describe("qa bundled plugin dir", () => { ); await expect( - testing.resolveQaOwnerPluginIdsForProviderIds({ + resolveQaOwnerPluginIdsForProviderIds({ repoRoot, providerIds: ["codex-cli"], }), @@ -2634,7 +2213,7 @@ describe("qa bundled plugin dir", () => { ); await expect( - testing.resolveQaOwnerPluginIdsForProviderIds({ + resolveQaOwnerPluginIdsForProviderIds({ repoRoot, providerIds: ["custom-openai"], providerConfigs: { @@ -2688,7 +2267,7 @@ describe("qa bundled plugin dir", () => { }, }); - const overrides = await testing.readQaLiveProviderConfigOverrides({ + const overrides = await readQaLiveProviderConfigOverrides({ providerIds: ["custom-openai"], env: { OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH: configPath }, }); @@ -2711,7 +2290,7 @@ describe("qa bundled plugin dir", () => { }, }); - const overrides = await testing.readQaLiveProviderConfigOverrides({ + const overrides = await readQaLiveProviderConfigOverrides({ providerIds: ["openai"], env: { OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH: configPath }, }); @@ -2737,7 +2316,7 @@ describe("qa bundled plugin dir", () => { }, }); - const overrides = await testing.readQaLiveProviderConfigOverrides({ + const overrides = await readQaLiveProviderConfigOverrides({ providerIds: ["openai"], env: { OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH: configPath }, }); @@ -2746,30 +2325,6 @@ describe("qa bundled plugin dir", () => { expect(overrides["openai"]?.api).toBe("openai-responses"); }); - it("does not copy OpenAI provider configs for custom OpenAI-compatible runs", async () => { - const configPath = await writeTempProviderConfig({ - models: { - providers: { - openai: { - baseUrl: "https://proxy.example.test/v1", - models: [], - apiKey: { - source: "env", - id: "OPENCLAW_LIVE_CODEX_API_KEY", - }, - }, - }, - }, - }); - - const overrides = await testing.readQaLiveProviderConfigOverrides({ - providerIds: ["openai"], - env: { OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH: configPath }, - }); - expect(Object.keys(overrides)).toEqual(["openai"]); - expect(overrides.openai?.baseUrl).toBe("https://proxy.example.test/v1"); - }); - it("raises the QA runtime host version to the highest allowed plugin floor", async () => { const repoRoot = await tempDirs.makeTempDir("qa-runtime-version-"); await writeJsonFixture(path.join(repoRoot, "package.json"), { version: "2026.4.7-1" }); @@ -2783,7 +2338,7 @@ describe("qa bundled plugin dir", () => { }); await expect( - testing.resolveQaRuntimeHostVersion({ + resolveQaRuntimeHostVersion({ repoRoot, allowedPluginIds: ["memory-core", "qa-channel"], }), @@ -2802,7 +2357,7 @@ describe("qa bundled plugin dir", () => { }); await expect( - testing.resolveQaRuntimeHostVersion({ + resolveQaRuntimeHostVersion({ repoRoot, allowedPluginIds: ["qa-channel"], }), diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index 4b9f2fe6f166..172976a0d149 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -1,106 +1,83 @@ // Qa Lab plugin module implements gateway child behavior. -import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { spawn, type ChildProcess } from "node:child_process"; import { randomUUID } from "node:crypto"; import { createWriteStream, existsSync, type WriteStream } from "node:fs"; import fs from "node:fs/promises"; import net from "node:net"; -import os from "node:os"; import path from "node:path"; -import { finished } from "node:stream/promises"; -import { StringDecoder } from "node:string_decoder"; import { setTimeout as sleep } from "node:timers/promises"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage, toErrorObject } from "openclaw/plugin-sdk/error-runtime"; -import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; -import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; -import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; -import { - isRecord, - normalizeOptionalString, - normalizeStringEntries, - uniqueStrings, -} from "openclaw/plugin-sdk/string-coerce-runtime"; +import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { createQaBundledPluginsDir, - resolveQaBundledPluginSourceDir, resolveQaOwnerPluginIdsForProviderIds, resolveQaRuntimeHostVersion, resolveQaStagedBundledPluginsRoot, } from "./bundled-plugin-staging.js"; -import { - appendQaChildOutput, - appendQaChildOutputTail, - createQaChildOutputCapture, - createQaChildOutputTail, - formatQaChildOutputTail, - readQaChildOutput, -} from "./child-output.js"; -import { assertRepoBoundPath, ensureRepoBoundDirectory } from "./cli-paths.js"; -import { buildQaCodexAppServerArgs } from "./codex-app-server-args.js"; import { QaSuiteInfraError } from "./errors.js"; -import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js"; +import { + cleanupQaGatewayTempRoots, + preserveQaGatewayDebugArtifacts, +} from "./gateway-child-artifacts.js"; +import { + resolveQaGatewayChildCommand, + runQaGatewayCliCommand, + type QaGatewayChildCommand, +} from "./gateway-child-command.js"; +import { + buildQaForcedRuntimeEnvPatch, + buildQaRuntimeEnv, + stageQaCodexMockModelCatalog, +} from "./gateway-child-env.js"; +import { + closeQaGatewayLogStream, + createQaGatewayChildLogCollector, + formatQaGatewayProcessBoundaryStartupFailure, + monitorQaGatewayChildFailure, + stopQaGatewayChildProcessTree, + throwQaGatewayChildFailure, + type QaChildFailure, +} from "./gateway-child-process.js"; +import { + callQaGatewayWithRetry, + isRetryableRpcStartupError, + QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS, + resolveQaGatewayStartupRetry, + waitForGatewayListening, + waitForGatewayReady, + waitForQaGatewayRestartBoundary, +} from "./gateway-child-readiness.js"; +import { redactQaGatewayDebugText } from "./gateway-log-redaction.js"; import { createQaGatewayProcessBoundaryController, - type QaGatewayProcessBoundaryConfig, type QaGatewayVerifiedProcessIdentity, } from "./gateway-process-boundary.js"; import { startQaGatewayRpcClient } from "./gateway-rpc-client.js"; import { splitQaModelRef, type QaProviderMode } from "./model-selection.js"; import { resolveQaNodeExecPath } from "./node-exec.js"; -import { - inspectLinuxProcessGroup, - inspectLinuxProcessGroupStats, - type QaLinuxProcessGroupInspector, -} from "./posix-process-group.js"; import { readProcessTreeCpuMs, readProcessTreeRssBytes } from "./process-tree-cpu.js"; -import { - normalizeQaProviderModeEnv, - QA_LIVE_PROVIDER_CONFIG_PATH_ENV, - resolveQaLiveCliAuthEnv, - resolveQaLiveProviderConfigPath, - type QaCliBackendAuthMode, -} from "./providers/env.js"; +import type { QaCliBackendAuthMode } from "./providers/env.js"; import { DEFAULT_QA_PROVIDER_MODE, getQaProvider } from "./providers/index.js"; +import { readQaLiveProviderConfigOverrides } from "./providers/live-config.js"; import { assertQaLiveCodexAuthAvailable, - QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV, - QA_LIVE_SETUP_TOKEN_VALUE_ENV, stageQaLiveApiKeyProfiles, stageQaLiveAnthropicSetupToken, } from "./providers/live-frontier/auth.js"; import { buildQaMockProfileId, stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js"; -import { listMockCodexModelInfos } from "./providers/shared/mock-model-config.js"; import { seedQaAgentWorkspace } from "./qa-agent-workspace.js"; import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js"; import type { QaTransportAdapter } from "./qa-transport.js"; import type { RuntimeId } from "./runtime-parity.js"; -import { resolveQaWindowsSystem32ExePath } from "./windows-system-tools.js"; +export type { QaGatewayChildCommand } from "./gateway-child-command.js"; export type { QaCliBackendAuthMode } from "./providers/env.js"; -const QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS = 5; const QA_GATEWAY_CHILD_RPC_STARTUP_TIMEOUT_MS = 30_000; const QA_GATEWAY_CHILD_RPC_RETRY_HEALTH_TIMEOUT_MS = 60_000; -const QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS = 90_000; -// The Gateway owns a 25s shutdown watchdog. Let it flush provider state before -// the QA parent escalates to a process-tree kill. -const QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 30_000; -// Loaded Docker runners can take several seconds to reap a force-killed process group. -const QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS = 10_000; -const QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS = 5_000; -const QA_GATEWAY_CHILD_RECENT_LOG_CHARS = 64 * 1_024; -const QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER = "[qa-lab] older gateway logs truncated\n"; const QA_PACKAGE_AUTH_FAILURE_MAX_CHARS = 2_048; -const QA_MOCK_OPENAI_API_KEY = ["qa", "mock", "openai", "key"].join("-"); -const QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS = Object.freeze([ - "OPENCLAW_QA_CONVEX_SECRET_CI", - "OPENCLAW_QA_CONVEX_SECRET_MAINTAINER", - "OPENCLAW_QA_SUT_FORBIDDEN_SENTINEL", - "OPENCLAW_QA_TELEGRAM_GROUP_ID", - "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN", - "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", -]); export type QaGatewayChildStateMutationContext = { configPath: string; @@ -109,21 +86,6 @@ export type QaGatewayChildStateMutationContext = { tempRoot: string; }; -type QaGatewayChildDirectCommand = { - executablePath: string; - argsPrefix?: string[]; - argsSuffix?: string[]; - cwd?: string; - tempParentDir?: string; - usePackagedPlugins?: boolean; - processBoundary?: undefined; -}; - -type QaGatewayChildVerifiedCommand = Omit & { - processBoundary: QaGatewayProcessBoundaryConfig; -}; - -export type QaGatewayChildCommand = QaGatewayChildDirectCommand | QaGatewayChildVerifiedCommand; export type QaGatewayChildListeningContext = { attempt: number; baseUrl: string; @@ -133,25 +95,6 @@ export type QaGatewayChildListeningContext = { runtimeEnv: NodeJS.ProcessEnv; }; -function scrubQaGatewayChildSecretEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { - for (const envKey of QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS) { - delete env[envKey]; - } - return env; -} - -function scrubQaGatewayChildTestRunnerEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { - // The Gateway is a product child, not a nested Vitest worker. Leaking runner - // markers makes the dist launcher select test-only startup behavior. - delete env.VITEST; - delete env.VITEST_POOL_ID; - delete env.VITEST_WORKER_ID; - if (env.NODE_ENV === "test") { - delete env.NODE_ENV; - } - return env; -} - function createQaGatewayEmptyTransport() { return { requiredPluginIds: [] as const, @@ -159,44 +102,84 @@ function createQaGatewayEmptyTransport() { } satisfies Pick; } -function resolveQaGatewayChildCommand(repoRoot: string): QaGatewayChildCommand { - for (const relativePath of ["scripts/run-node.mjs", "dist/index.mjs", "dist/index.js"]) { - const entryPath = path.join(repoRoot, relativePath); - if (existsSync(entryPath)) { - return { - executablePath: process.execPath, - argsPrefix: [entryPath], - cwd: repoRoot, - usePackagedPlugins: true, - }; - } - } +async function getFreePort() { + return await new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", (error) => reject(error)); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("failed to allocate port")); + return; + } + server.close((error) => (error ? reject(error) : resolve(address.port))); + }); + }); +} - throw new Error( - "OpenClaw CLI entry not found: expected scripts/run-node.mjs or dist/index.(m)js", +function appendQaGatewayTempRoot(details: string, tempRoot: string) { + return details.includes(tempRoot) + ? details + : `${details}\nQA gateway temp root preserved at ${tempRoot}`; +} + +function throwQaGatewayStartupError(params: { + error: unknown; + message: string; + cleanupErrors: unknown[]; +}): never { + const primaryError = + params.error instanceof QaSuiteInfraError + ? new QaSuiteInfraError(params.error.code, params.message, { cause: params.error }) + : new Error(params.message, { cause: params.error }); + if (params.cleanupErrors.length === 0) { + throw primaryError; + } + throw new AggregateError( + [primaryError, ...params.cleanupErrors], + "qa gateway startup and cleanup failed", + { cause: primaryError }, ); } -async function runQaGatewayCliCommand(params: { - executablePath: string; - argsPrefix: readonly string[]; - args: readonly string[]; - cwd: string; - env: NodeJS.ProcessEnv; - stdin?: string; -}): Promise { - const hasStdin = params.stdin !== undefined; - const child = spawn(params.executablePath, [...params.argsPrefix, ...params.args], { - cwd: params.cwd, - env: { ...params.env, OPENCLAW_CLI: "1" }, - stdio: [hasStdin ? "pipe" : "ignore", "pipe", "pipe"], - }); - const result = readQaGatewayCliCommand(child); - if (hasStdin) { - child.stdin?.once("error", () => {}); - child.stdin?.end(params.stdin); +type QaGatewayProcessBoundaryController = Awaited< + ReturnType +>; + +async function stopQaGatewayChildWithBoundary(params: { + child: ChildProcess; + controller: QaGatewayProcessBoundaryController | null; + identity: QaGatewayVerifiedProcessIdentity | null; + opts?: { gracefulTimeoutMs?: number; forceTimeoutMs?: number }; +}) { + const errors: unknown[] = []; + if (params.controller && params.identity) { + try { + await params.controller.markExited(params.identity); + } catch (error) { + errors.push(error); + } } - return await result; + try { + await stopQaGatewayChildProcessTree(params.child, params.opts); + } catch (error) { + errors.push(error); + } + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "qa gateway process-boundary cleanup failed"); + } +} + +function resolveQaControlUiRoot(params: { repoRoot: string; controlUiEnabled?: boolean }) { + if (params.controlUiEnabled === false) { + return undefined; + } + const controlUiRoot = path.join(params.repoRoot, "dist", "control-ui"); + const indexPath = path.join(controlUiRoot, "index.html"); + return existsSync(indexPath) ? controlUiRoot : undefined; } function createQaPackagedMockApiKey(): string { @@ -243,994 +226,6 @@ async function stageQaPackagedMockAuthProfiles(params: { } } -type QaChildFailure = { - source: "process" | "stdout" | "stderr"; - error: unknown; -}; - -function monitorQaChildFailure(child: ChildProcess, onFailure: (failure: QaChildFailure) => void) { - let reported = false; - const report = (source: QaChildFailure["source"]) => (error: unknown) => { - if (reported) { - return; - } - reported = true; - onFailure({ source, error }); - }; - child.once("error", report("process")); - child.stdout?.once("error", report("stdout")); - child.stderr?.once("error", report("stderr")); -} - -async function readQaGatewayCliCommand(child: ChildProcess): Promise { - const stdout = createQaChildOutputCapture(); - const stderr = createQaChildOutputTail(); - child.stdout?.on("data", (chunk) => appendQaChildOutput(stdout, chunk)); - child.stderr?.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk)); - const exitCode = await new Promise((resolve, reject) => { - monitorQaChildFailure(child, (failure) => { - if (failure.source === "process") { - reject(toErrorObject(failure.error, "OpenClaw CLI process failed")); - return; - } - if (!hasChildExited(child) && !child.killed) { - try { - child.kill("SIGKILL"); - } catch { - // The child exited between the state check and signal. - } - } - reject( - new Error( - `qa gateway cli ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`, - { cause: failure.error }, - ), - ); - }); - child.once("close", (code) => resolve(code ?? 1)); - }); - const stdoutText = readQaChildOutput(stdout); - if (exitCode !== 0) { - const stderrText = formatQaChildOutputTail(stderr, "stderr"); - throw new Error(`OpenClaw CLI exited ${exitCode}: ${stderrText || stdoutText}`); - } - return stdoutText; -} - -async function getFreePort() { - return await new Promise((resolve, reject) => { - const server = net.createServer(); - server.once("error", (error) => reject(error)); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - if (!address || typeof address === "string") { - reject(new Error("failed to allocate port")); - return; - } - server.close((error) => (error ? reject(error) : resolve(address.port))); - }); - }); -} - -async function closeWriteStream( - stream: WriteStream, - label: "stderr" | "stdout", - timeoutMs = QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS, -) { - if (stream.destroyed) { - return; - } - stream.end(); - const signal = AbortSignal.timeout(timeoutMs); - try { - await finished(stream, { cleanup: true, signal }); - } catch (error) { - if (!signal.aborted) { - throw error; - } - // Gateway logs are diagnostic only. Never let a stuck filesystem flush - // retain the stopped child runtime and its live transport credentials. - process.stderr.write( - `[qa-suite] ${label} gateway log flush exceeded ${timeoutMs}ms; forcing close\n`, - ); - stream.destroy(); - } -} - -async function writeSanitizedQaGatewayDebugLog(params: { sourcePath: string; targetPath: string }) { - const contents = await fs.readFile(params.sourcePath, "utf8").catch((error: unknown) => { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return ""; - } - throw error; - }); - await fs.writeFile(params.targetPath, redactQaGatewayDebugText(contents), "utf8"); -} - -async function assertQaArtifactDirWithinRepo(repoRoot: string, artifactDir: string) { - return await assertRepoBoundPath(repoRoot, artifactDir, "QA gateway artifact directory"); -} - -async function clearQaGatewayArtifactDir(dir: string) { - for (const entry of await fs.readdir(dir, { withFileTypes: true })) { - await fs.rm(path.join(dir, entry.name), { recursive: true, force: true }); - } -} - -async function cleanupQaGatewayTempRoots(params: { - tempRoot: string; - stagedBundledPluginsRoot?: string | null; -}) { - await fs.rm(params.tempRoot, { recursive: true, force: true }).catch(() => {}); - if (params.stagedBundledPluginsRoot) { - await fs.rm(params.stagedBundledPluginsRoot, { recursive: true, force: true }).catch(() => {}); - } -} - -async function preserveQaGatewayDebugArtifacts(params: { - preserveToDir: string; - stdoutLogPath: string; - stderrLogPath: string; - tempRoot: string; - repoRoot?: string; -}) { - const preserveToDir = params.repoRoot - ? await ensureRepoBoundDirectory( - params.repoRoot, - params.preserveToDir, - "QA gateway artifact directory", - { - mode: 0o700, - }, - ) - : params.preserveToDir; - await fs.mkdir(preserveToDir, { recursive: true, mode: 0o700 }); - await clearQaGatewayArtifactDir(preserveToDir); - await Promise.all([ - writeSanitizedQaGatewayDebugLog({ - sourcePath: params.stdoutLogPath, - targetPath: path.join(preserveToDir, "gateway.stdout.log"), - }), - writeSanitizedQaGatewayDebugLog({ - sourcePath: params.stderrLogPath, - targetPath: path.join(preserveToDir, "gateway.stderr.log"), - }), - ]); - await fs.writeFile( - path.join(preserveToDir, "README.txt"), - [ - "Only sanitized gateway debug artifacts are preserved here.", - "The full QA gateway runtime was not copied because it may contain credentials or auth tokens.", - "Original runtime temp root omitted because local temp paths can identify the runner.", - "", - ].join("\n"), - "utf8", - ); -} - -type QaGatewayStartupRetryKind = "bind-collision" | "migration-convergence-restart"; - -const QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX = - "OpenClaw plugin migration inputs changed during startup convergence;"; - -function classifyQaGatewayStartupRetry(details: string): QaGatewayStartupRetryKind | null { - if (details.includes(QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX)) { - return "migration-convergence-restart"; - } - if ( - details.includes("another gateway instance is already listening on ws://") || - details.includes("failed to bind gateway socket on ws://") || - details.includes("EADDRINUSE") || - details.includes("address already in use") - ) { - return "bind-collision"; - } - return null; -} - -function resolveQaGatewayStartupRetry(params: { - attempt: number; - details: string; - migrationConvergenceRestartUsed: boolean; -}) { - if (params.attempt >= QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS) { - return null; - } - const kind = classifyQaGatewayStartupRetry(params.details); - if ( - !kind || - (kind === "migration-convergence-restart" && params.migrationConvergenceRestartUsed) - ) { - return null; - } - return { - kind, - reuseLaunchState: kind === "migration-convergence-restart", - migrationConvergenceRestartUsed: - params.migrationConvergenceRestartUsed || kind === "migration-convergence-restart", - }; -} - -function appendQaGatewayTempRoot(details: string, tempRoot: string) { - return details.includes(tempRoot) - ? details - : `${details}\nQA gateway temp root preserved at ${tempRoot}`; -} - -function throwQaGatewayStartupError(params: { - error: unknown; - message: string; - cleanupErrors: unknown[]; -}): never { - const primaryError = - params.error instanceof QaSuiteInfraError - ? new QaSuiteInfraError(params.error.code, params.message, { cause: params.error }) - : new Error(params.message, { cause: params.error }); - if (params.cleanupErrors.length === 0) { - throw primaryError; - } - throw new AggregateError( - [primaryError, ...params.cleanupErrors], - "qa gateway startup and cleanup failed", - { cause: primaryError }, - ); -} - -export function resolveQaGatewayChildProviderMode(providerMode?: QaProviderMode): QaProviderMode { - return providerMode ?? DEFAULT_QA_PROVIDER_MODE; -} - -export function buildQaRuntimeEnv(params: { - configPath: string; - gatewayToken: string; - homeDir: string; - forwardHostHome?: boolean; - stateDir: string; - tempRoot: string; - xdgConfigHome: string; - xdgDataHome: string; - xdgCacheHome: string; - bundledPluginsDir?: string; - stagedBundledPluginsRoot?: string | null; - compatibilityHostVersion?: string; - providerMode?: QaProviderMode; - baseEnv?: NodeJS.ProcessEnv; - runtimeEnvPatch?: NodeJS.ProcessEnv; - forwardHostHomeForClaudeCli?: boolean; - claudeCliAuthMode?: QaCliBackendAuthMode; -}) { - const baseEnv = params.baseEnv ?? process.env; - const provider = params.providerMode ? getQaProvider(params.providerMode) : null; - const forwardedHostHome = params.forwardHostHome - ? baseEnv.HOME?.trim() || os.homedir() - : undefined; - const env: NodeJS.ProcessEnv = { - ...baseEnv, - HOME: forwardedHostHome ?? params.homeDir, - ...(provider?.appliesLiveEnvAliases - ? resolveQaLiveCliAuthEnv(baseEnv, { - forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli, - claudeCliAuthMode: params.claudeCliAuthMode, - }) - : {}), - OPENCLAW_HOME: params.homeDir, - OPENCLAW_CONFIG_PATH: params.configPath, - OPENCLAW_STATE_DIR: params.stateDir, - OPENCLAW_OAUTH_DIR: path.join(params.stateDir, "credentials"), - OPENCLAW_GATEWAY_TOKEN: params.gatewayToken, - OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1", - OPENCLAW_SKIP_GMAIL_WATCHER: "1", - OPENCLAW_SKIP_CANVAS_HOST: "1", - OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1", - OPENCLAW_NO_RESPAWN: "1", - OPENCLAW_TEST_FAST: "1", - OPENCLAW_EMBEDDED_ABORT_SETTLE_TIMEOUT_MS: "2000", - OPENCLAW_QA_PARENT_PID: String(process.pid), - OPENCLAW_QA_TEMP_ROOT: params.tempRoot, - ...(params.stagedBundledPluginsRoot - ? { OPENCLAW_QA_STAGED_RUNTIME_ROOT: params.stagedBundledPluginsRoot } - : {}), - OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER: "1", - // QA uses the fast runtime envelope for speed, but it still exercises - // normal config-driven heartbeats and runtime config writes. - OPENCLAW_ALLOW_SLOW_REPLY_TESTS: "1", - XDG_CONFIG_HOME: params.xdgConfigHome, - XDG_DATA_HOME: params.xdgDataHome, - XDG_CACHE_HOME: params.xdgCacheHome, - ...(params.bundledPluginsDir ? { OPENCLAW_BUNDLED_PLUGINS_DIR: params.bundledPluginsDir } : {}), - ...(params.compatibilityHostVersion - ? { OPENCLAW_COMPATIBILITY_HOST_VERSION: params.compatibilityHostVersion } - : {}), - }; - const normalizedEnv = normalizeQaProviderModeEnv(env, params.providerMode); - // Test-runner skip flags are parent controls; each QA child declares its own runtime needs. - delete normalizedEnv.OPENCLAW_SKIP_CHANNELS; - delete normalizedEnv.OPENCLAW_SKIP_PROVIDERS; - Object.assign(normalizedEnv, params.runtimeEnvPatch); - normalizedEnv.OPENCLAW_BUILD_PRIVATE_QA = "1"; - delete normalizedEnv[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]; - delete normalizedEnv[QA_LIVE_SETUP_TOKEN_VALUE_ENV]; - return scrubQaGatewayChildSecretEnv(scrubQaGatewayChildTestRunnerEnv(normalizedEnv)); -} - -async function stageQaCodexMockModelCatalog(params: { - tempRoot: string; - forcedRuntime?: RuntimeId; - providerMode: QaProviderMode; - primaryModel?: string; - alternateModel?: string; -}): Promise { - if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") { - return undefined; - } - const modelCatalogPath = path.join(params.tempRoot, "codex-model-catalog.json"); - const selectedModelRefs = [params.primaryModel, params.alternateModel].filter( - (model): model is string => typeof model === "string" && model.length > 0, - ); - await fs.writeFile( - modelCatalogPath, - `${JSON.stringify({ models: listMockCodexModelInfos(selectedModelRefs) }, null, 2)}\n`, - { encoding: "utf8", mode: 0o600 }, - ); - return modelCatalogPath; -} - -function buildQaForcedRuntimeEnvPatch(params: { - forcedRuntime?: RuntimeId; - providerMode: QaProviderMode; - providerBaseUrl?: string; - codexModelCatalogPath?: string; - nativeAppServerArgs?: string; -}): NodeJS.ProcessEnv | undefined { - if (!params.forcedRuntime) { - return undefined; - } - const patch: NodeJS.ProcessEnv = { - OPENCLAW_BUILD_PRIVATE_QA: "1", - OPENCLAW_QA_FORCE_RUNTIME: params.forcedRuntime, - }; - if (params.forcedRuntime !== "codex") { - return patch; - } - if (params.providerMode !== "mock-openai") { - patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({ - existingArgs: params.nativeAppServerArgs, - }); - return patch; - } - const providerBaseUrl = params.providerBaseUrl?.trim().replace(/\/+$/u, ""); - if (!providerBaseUrl) { - throw new Error("forced Codex mock QA requires the managed mock provider URL"); - } - if (!params.codexModelCatalogPath) { - throw new Error("forced Codex mock QA requires the staged native model catalog"); - } - patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({ - providerBaseUrl, - modelCatalogPath: params.codexModelCatalogPath, - }); - patch.OPENAI_API_KEY = QA_MOCK_OPENAI_API_KEY; - patch.CODEX_API_KEY = QA_MOCK_OPENAI_API_KEY; - return patch; -} - -function isRetryableGatewayCallError(details: string): boolean { - return ( - details.includes("handshake timeout") || - details.includes("gateway closed (1000") || - details.includes("gateway closed (1012)") || - details.includes("gateway closed (1006") || - details.includes("abnormal closure") || - details.includes("service restart") - ); -} - -async function callQaGatewayWithRetry(params: { - deadlineMs?: number; - logs: () => string; - request: (options: { deadlineMs?: number; timeoutMs: number }) => Promise; - throwChildFailure: () => void; - timeoutMs: number; - waitForReady: (timeoutMs: number) => Promise; -}) { - const remainingMs = () => - params.deadlineMs === undefined ? undefined : params.deadlineMs - Date.now(); - const deadlineError = () => - new Error(`gateway call deadline exceeded${formatQaGatewayLogsForError(params.logs())}`); - let lastDetails = ""; - for (let attempt = 1; attempt <= 3; attempt += 1) { - params.throwChildFailure(); - const requestRemainingMs = remainingMs(); - if (requestRemainingMs !== undefined && requestRemainingMs <= 0) { - throw deadlineError(); - } - try { - return await params.request({ - ...(params.deadlineMs === undefined ? {} : { deadlineMs: params.deadlineMs }), - timeoutMs: - requestRemainingMs === undefined - ? params.timeoutMs - : Math.min(params.timeoutMs, requestRemainingMs), - }); - } catch (error) { - params.throwChildFailure(); - const details = formatErrorMessage(error); - lastDetails = details; - if (attempt >= 3 || !isRetryableGatewayCallError(details)) { - throw new Error(`${details}${formatQaGatewayLogsForError(params.logs())}`, { - cause: error, - }); - } - const readinessRemainingMs = remainingMs(); - if (readinessRemainingMs !== undefined && readinessRemainingMs <= 0) { - throw deadlineError(); - } - await params.waitForReady( - readinessRemainingMs === undefined - ? Math.max(10_000, params.timeoutMs) - : Math.min(Math.max(10_000, params.timeoutMs), readinessRemainingMs), - ); - } - } - throw new Error(`${lastDetails}${formatQaGatewayLogsForError(params.logs())}`); -} - -type QaGatewayChildLogSource = "internal" | "stderr" | "stdout"; - -function createQaGatewayChildLogCollector() { - const decoders: Record = { - internal: new StringDecoder("utf8"), - stderr: new StringDecoder("utf8"), - stdout: new StringDecoder("utf8"), - }; - let recent = ""; - let end = 0; - - const readFrom = (mark: number) => { - const start = end - recent.length; - const wasTruncated = mark < start; - const text = recent.slice(Math.max(0, mark - start)); - return `${wasTruncated ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${text}`; - }; - return { - push(source: QaGatewayChildLogSource, chunk: Buffer) { - const text = decoders[source].write(chunk); - end += text.length; - recent += text; - if (recent.length > QA_GATEWAY_CHILD_RECENT_LOG_CHARS) { - recent = sliceUtf16Safe(recent, -QA_GATEWAY_CHILD_RECENT_LOG_CHARS); - } - }, - mark() { - return end; - }, - readSince(mark: number) { - return readFrom(mark); - }, - text() { - return `${end > recent.length ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${recent}`.trim(); - }, - }; -} - -function formatQaGatewayChildFailure(failure: QaChildFailure) { - return failure.source === "process" - ? `gateway failed to spawn: ${formatErrorMessage(failure.error)}` - : `gateway child ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`; -} - -function throwQaGatewayChildFailure( - getChildFailure: (() => QaChildFailure | null) | undefined, - logs: () => string, -) { - const failure = getChildFailure?.(); - if (!failure) { - return; - } - throw new QaSuiteInfraError( - "gateway_startup_unhealthy", - `${formatQaGatewayChildFailure(failure)}\n${logs()}`, - { cause: failure.error }, - ); -} - -function monitorQaGatewayChildFailure( - child: ChildProcess, - output: { push(source: QaGatewayChildLogSource, chunk: Buffer): void }, -) { - let childFailure: QaChildFailure | null = null; - monitorQaChildFailure(child, (failure) => { - childFailure = failure; - const description = - failure.source === "process" - ? `gateway child process error: ${formatErrorMessage(failure.error)}` - : formatQaGatewayChildFailure(failure); - output.push("internal", Buffer.from(`[qa-lab] ${description}\n`)); - if (failure.source !== "process" && !hasChildExited(child)) { - // A broken parent-side pipe means QA can no longer observe the Gateway. - // Stop the detached process tree so the existing lifecycle reports the failure. - signalQaGatewayChildProcessTree(child, "SIGTERM"); - } - }); - return () => childFailure; -} - -const QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS = 8_192; - -function formatQaGatewayProcessBoundaryStartupFailure(error: unknown, logs: string) { - const logTail = sliceUtf16Safe( - redactQaGatewayDebugText(logs), - -QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS, - ); - return `${formatErrorMessage(error)}${formatQaGatewayLogsForError(logTail)}`; -} - -async function fetchLocalGatewayHealth(params: { - baseUrl: string; - healthPath: "/readyz" | "/healthz"; - timeoutMs?: number; -}): Promise { - const { response, release } = await fetchWithSsrFGuard({ - url: `${params.baseUrl}${params.healthPath}`, - init: { - method: "HEAD", - headers: { - connection: "close", - }, - signal: AbortSignal.timeout(params.timeoutMs ?? 2_000), - }, - policy: { allowPrivateNetwork: true }, - auditContext: "qa-lab-gateway-child-health", - }); - try { - return response.ok; - } finally { - await release(); - } -} - -async function fetchLocalGatewayListening(baseUrl: string): Promise { - const { release } = await fetchWithSsrFGuard({ - url: `${baseUrl}/healthz`, - init: { - method: "HEAD", - headers: { - connection: "close", - }, - signal: AbortSignal.timeout(2_000), - }, - policy: { allowPrivateNetwork: true }, - auditContext: "qa-lab-gateway-child-listening", - }); - await release(); - return true; -} - -async function waitForQaGatewayRestartBoundary(params: { - readLogsSince: (mark: number) => string; - mark: number; - pollMs?: number; - timeoutMs?: number; -}) { - const timeoutMs = params.timeoutMs ?? QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS; - const pollMs = resolveTimerTimeoutMs(params.pollMs ?? 100, 100, 0); - const startedAt = Date.now(); - while (Date.now() - startedAt < timeoutMs) { - if (params.readLogsSince(params.mark).includes("restart mode:")) { - return; - } - const remainingMs = timeoutMs - (Date.now() - startedAt); - if (remainingMs <= 0) { - break; - } - await sleep(Math.min(pollMs, remainingMs)); - } - throw new Error(`qa gateway child did not reach restart boundary within ${timeoutMs}ms`); -} - -export const testing = { - assertQaArtifactDirWithinRepo, - buildQaForcedRuntimeEnvPatch, - buildQaRuntimeEnv, - cleanupQaGatewayTempRoots, - fetchLocalGatewayHealth, - callQaGatewayWithRetry, - isRetryableGatewayCallError, - isRetryableRpcStartupError, - classifyQaGatewayStartupRetry, - resolveQaGatewayStartupRetry, - preserveQaGatewayDebugArtifacts, - redactQaGatewayDebugText, - readQaLiveProviderConfigOverrides, - resolveQaGatewayChildProviderMode, - resolveQaGatewayChildCommand, - createQaGatewayEmptyTransport, - waitForGatewayReady, - assertQaLiveCodexAuthAvailable, - stageQaLiveApiKeyProfiles, - stageQaLiveAnthropicSetupToken, - stageQaMockAuthProfiles, - stageQaCodexMockModelCatalog, - resolveQaLiveCliAuthEnv, - waitForQaGatewayRestartBoundary, - resolveQaOwnerPluginIdsForProviderIds, - resolveQaBundledPluginSourceDir, - resolveQaRuntimeHostVersion, - runQaGatewayCliCommand, - readQaGatewayCliCommand, - createQaGatewayChildLogCollector, - monitorQaGatewayChildFailure, - throwQaGatewayChildFailure, - formatQaGatewayProcessBoundaryStartupFailure, - createQaBundledPluginsDir, - signalQaGatewayChildProcessTree, - resolveQaGatewayChildStopTimeouts, - stopQaGatewayChildProcessTree, - inspectLinuxProcessGroupStats, - isQaGatewayChildProcessTreeAlive, - closeWriteStream, -}; - -function hasChildExited(child: ChildProcess) { - return child.exitCode !== null || child.signalCode !== null; -} - -function isProcessAlreadyExitedError(error: unknown): boolean { - return (error as NodeJS.ErrnoException | undefined)?.code === "ESRCH"; -} - -function boundQaGatewayProcessTreeDiagnostics(details: string) { - if (details.length <= 2_048) { - return details; - } - return `${sliceUtf16Safe(details, 0, 2_045)}...`; -} - -function isQaGatewayChildProcessTreeAlive( - child: ChildProcess, - inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector = inspectLinuxProcessGroup, -) { - if (!child.pid) { - return false; - } - if (process.platform === "win32") { - return !hasChildExited(child); - } - try { - process.kill(-child.pid, 0); - if (process.platform === "linux") { - // Linux can retain zombie-only process groups after SIGKILL while Node's - // child metadata is still unsettled. Runnable /proc members are the owner. - return inspectLinuxProcessGroupFn(child.pid)?.alive ?? true; - } - return true; - } catch (error) { - if (!isProcessAlreadyExitedError(error) && !hasChildExited(child)) { - return true; - } - } - return false; -} - -type QaGatewayTaskkillRunner = typeof spawnSync; - -function signalQaGatewayWindowsProcessTree( - pid: number, - signal: NodeJS.Signals, - runTaskkill: QaGatewayTaskkillRunner = spawnSync, -) { - const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe"); - const args = ["/PID", String(pid), "/T"]; - if (signal === "SIGKILL") { - args.push("/F"); - } - const result = runTaskkill(taskkillPath, args, { - stdio: "ignore", - windowsHide: true, - timeout: 5_000, - }); - if (!result.error && result.status === 0) { - return true; - } - if (signal !== "SIGKILL") { - const forceResult = runTaskkill(taskkillPath, [...args, "/F"], { - stdio: "ignore", - windowsHide: true, - timeout: 5_000, - }); - return !forceResult.error && forceResult.status === 0; - } - return false; -} - -function signalQaGatewayChildProcessTree( - child: ChildProcess, - signal: NodeJS.Signals, - runTaskkill: QaGatewayTaskkillRunner = spawnSync, -) { - if (!child.pid) { - return; - } - try { - if (process.platform === "win32") { - if (signalQaGatewayWindowsProcessTree(child.pid, signal, runTaskkill)) { - return; - } - child.kill(signal); - return; - } - process.kill(-child.pid, signal); - } catch { - try { - child.kill(signal); - } catch { - // The child already exited. - } - } -} - -async function waitForQaGatewayChildExit( - child: ChildProcess, - timeoutMs: number, - inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector, -) { - const deadline = Date.now() + timeoutMs; - while (Date.now() <= deadline) { - if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) { - return true; - } - await sleep(Math.min(25, Math.max(0, deadline - Date.now()))); - } - return !isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn); -} - -type QaGatewayChildStopOptions = { - gracefulTimeoutMs?: number; - forceTimeoutMs?: number; - inspectLinuxProcessGroup?: QaLinuxProcessGroupInspector; -}; - -function resolveQaGatewayChildStopTimeouts(opts?: QaGatewayChildStopOptions) { - return { - gracefulTimeoutMs: opts?.gracefulTimeoutMs ?? QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS, - forceTimeoutMs: opts?.forceTimeoutMs ?? QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS, - }; -} - -function formatQaGatewayProcessTreeDiagnostics( - child: ChildProcess, - inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector, -) { - const childExitRecorded = hasChildExited(child); - if (process.platform !== "linux" || !child.pid) { - return `pid=${child.pid ?? "unknown"} childExitRecorded=${childExitRecorded}`; - } - const inspection = inspectLinuxProcessGroupFn(child.pid); - const processGroupDetails = - inspection?.diagnostics ?? `pgid=${child.pid} members=unknown (/proc unavailable)`; - return boundQaGatewayProcessTreeDiagnostics( - `${processGroupDetails} childExitRecorded=${childExitRecorded}`, - ); -} - -async function stopQaGatewayChildProcessTree( - child: ChildProcess, - opts?: QaGatewayChildStopOptions, -) { - const inspectLinuxProcessGroupFn = opts?.inspectLinuxProcessGroup ?? inspectLinuxProcessGroup; - if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) { - return; - } - const timeouts = resolveQaGatewayChildStopTimeouts(opts); - signalQaGatewayChildProcessTree(child, "SIGTERM"); - if ( - await waitForQaGatewayChildExit(child, timeouts.gracefulTimeoutMs, inspectLinuxProcessGroupFn) - ) { - return; - } - signalQaGatewayChildProcessTree(child, "SIGKILL"); - const stopped = await waitForQaGatewayChildExit( - child, - timeouts.forceTimeoutMs, - inspectLinuxProcessGroupFn, - ); - if (!stopped) { - throw new Error( - `qa gateway process tree remained alive after forced shutdown: ${formatQaGatewayProcessTreeDiagnostics( - child, - inspectLinuxProcessGroupFn, - )}`, - ); - } -} - -type QaGatewayProcessBoundaryController = Awaited< - ReturnType ->; - -async function stopQaGatewayChildWithBoundary(params: { - child: ChildProcess; - controller: QaGatewayProcessBoundaryController | null; - identity: QaGatewayVerifiedProcessIdentity | null; - opts?: { gracefulTimeoutMs?: number; forceTimeoutMs?: number }; -}) { - const errors: unknown[] = []; - if (params.controller && params.identity) { - try { - await params.controller.markExited(params.identity); - } catch (error) { - errors.push(error); - } - } - try { - await stopQaGatewayChildProcessTree(params.child, params.opts); - } catch (error) { - errors.push(error); - } - if (errors.length === 1) { - throw errors[0]; - } - if (errors.length > 1) { - throw new AggregateError(errors, "qa gateway process-boundary cleanup failed"); - } -} - -function isQaModelProviderConfig(value: unknown): value is ModelProviderConfig { - return isRecord(value) && typeof value.baseUrl === "string" && Array.isArray(value.models); -} - -function normalizeQaLiveProviderConfig(value: unknown): ModelProviderConfig | null { - if (!isQaModelProviderConfig(value) && (!isRecord(value) || !Object.hasOwn(value, "apiKey"))) { - return null; - } - const { baseUrl: rawBaseUrl, ...providerConfig } = value; - const baseUrl = normalizeOptionalString(rawBaseUrl); - return { - ...providerConfig, - ...(baseUrl ? { baseUrl } : {}), - models: Array.isArray(value.models) ? value.models : [], - } as ModelProviderConfig; -} - -async function readQaLiveProviderConfigOverrides(params: { - providerIds: readonly string[]; - env?: NodeJS.ProcessEnv; -}) { - const providerIds = uniqueStrings(normalizeStringEntries(params.providerIds)); - if (providerIds.length === 0) { - return {}; - } - const configPath = resolveQaLiveProviderConfigPath(params.env); - if (!existsSync(configPath.path)) { - return {}; - } - try { - const raw = await fs.readFile(configPath.path, "utf8"); - const parsed = JSON.parse(raw) as unknown; - const providers = isRecord(parsed) - ? isRecord(parsed.models) - ? isRecord(parsed.models.providers) - ? parsed.models.providers - : {} - : {} - : {}; - const selected: Record = {}; - for (const providerId of providerIds) { - const providerConfig = normalizeQaLiveProviderConfig(providers[providerId]); - if (providerConfig) { - selected[providerId] = providerConfig; - } - } - return selected; - } catch (error) { - if (configPath.explicit) { - throw new Error( - `failed to read ${QA_LIVE_PROVIDER_CONFIG_PATH_ENV} provider config: ${formatErrorMessage(error)}`, - { cause: error }, - ); - } - return {}; - } -} - -async function waitForGatewayReady(params: { - baseUrl: string; - logs: () => string; - child: { - exitCode: number | null; - signalCode: NodeJS.Signals | null; - }; - getChildFailure?: () => QaChildFailure | null; - timeoutMs?: number; -}) { - const deadline = Date.now() + (params.timeoutMs ?? 60_000); - let remainingMs: number; - while ((remainingMs = deadline - Date.now()) > 0) { - throwQaGatewayChildFailure(params.getChildFailure, params.logs); - if (params.child.exitCode !== null || params.child.signalCode !== null) { - throw new QaSuiteInfraError( - "gateway_startup_unhealthy", - `gateway exited before becoming healthy (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`, - ); - } - // Listener liveness can turn green before the Gateway can admit startup or restart work. - try { - if ( - await fetchLocalGatewayHealth({ - baseUrl: params.baseUrl, - healthPath: "/readyz", - timeoutMs: Math.min(2_000, remainingMs), - }) - ) { - return; - } - } catch { - // retry until timeout - } - await sleep(Math.min(250, Math.max(0, deadline - Date.now()))); - } - throw new QaSuiteInfraError( - "gateway_startup_unhealthy", - `gateway failed to become healthy:\n${params.logs()}`, - ); -} - -async function waitForGatewayListening(params: { - baseUrl: string; - logs: () => string; - child: { - exitCode: number | null; - signalCode: NodeJS.Signals | null; - }; - getChildFailure?: () => QaChildFailure | null; - timeoutMs?: number; -}) { - const startedAt = Date.now(); - while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) { - throwQaGatewayChildFailure(params.getChildFailure, params.logs); - if (params.child.exitCode !== null || params.child.signalCode !== null) { - throw new QaSuiteInfraError( - "gateway_startup_unhealthy", - `gateway exited before listening (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`, - ); - } - try { - if (await fetchLocalGatewayListening(params.baseUrl)) { - return; - } - } catch { - // retry until the HTTP listener accepts requests - } - await sleep(100); - } - throw new QaSuiteInfraError( - "gateway_startup_unhealthy", - `gateway failed to listen before timeout:\n${params.logs()}`, - ); -} - -function isRetryableRpcStartupError(error: unknown) { - const details = formatErrorMessage(error); - return ( - details.includes("gateway timeout after") || - details.includes("handshake timeout") || - details.includes("gateway token mismatch") || - details.includes("token mismatch") || - details.includes("gateway closed (1000") || - details.includes("gateway closed (1006") || - details.includes("gateway closed (1012)") - ); -} - -export function resolveQaControlUiRoot(params: { repoRoot: string; controlUiEnabled?: boolean }) { - if (params.controlUiEnabled === false) { - return undefined; - } - const controlUiRoot = path.join(params.repoRoot, "dist", "control-ui"); - const indexPath = path.join(controlUiRoot, "index.html"); - return existsSync(indexPath) ? controlUiRoot : undefined; -} - export async function startQaGatewayChild(params: { repoRoot: string; command?: QaGatewayChildCommand; @@ -1301,7 +296,7 @@ export async function startQaGatewayChild(params: { fs.mkdir(xdgDataHome, { recursive: true }), fs.mkdir(xdgCacheHome, { recursive: true }), ]); - const providerMode = resolveQaGatewayChildProviderMode(params.providerMode); + const providerMode = params.providerMode ?? DEFAULT_QA_PROVIDER_MODE; const codexModelCatalogPath = await stageQaCodexMockModelCatalog({ tempRoot, forcedRuntime: params.forcedRuntime, @@ -1958,7 +953,7 @@ export async function startQaGatewayChild(params: { } for (const [label, stream] of gatewayLogStreams) { try { - await closeWriteStream(stream, label); + await closeQaGatewayLogStream(stream, label); } catch (error) { cleanupErrors.push(error); } @@ -2026,7 +1021,7 @@ export async function startQaGatewayChild(params: { } for (const [label, stream] of gatewayLogStreams) { try { - await closeWriteStream(stream, label); + await closeQaGatewayLogStream(stream, label); } catch (cleanupError) { cleanupErrors.push(cleanupError); } diff --git a/extensions/qa-lab/src/posix-process-group.test.ts b/extensions/qa-lab/src/posix-process-group.test.ts index 54e7acedc177..a6a9e48f352f 100644 --- a/extensions/qa-lab/src/posix-process-group.test.ts +++ b/extensions/qa-lab/src/posix-process-group.test.ts @@ -1,9 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { - inspectLinuxProcessGroupStats, - isQaPosixProcessGroupAlive, - signalQaPosixProcessGroup, -} from "./posix-process-group.js"; +import { isQaPosixProcessGroupAlive, signalQaPosixProcessGroup } from "./posix-process-group.js"; +import { inspectLinuxProcessGroupStats } from "./posix-process-stat.js"; afterEach(() => { vi.restoreAllMocks(); @@ -37,6 +34,19 @@ describe("POSIX process group inspection", () => { }); }); + it("bounds process group diagnostics", () => { + const stats = Array.from( + { length: 300 }, + (_, index) => `${index + 1} (${`worker-${index}`.padEnd(32, "x")}) S 1 123 123 0 -1 0`, + ); + + const inspection = inspectLinuxProcessGroupStats(123, stats); + + expect(inspection.alive).toBe(true); + expect(inspection.diagnostics.length).toBeLessThanOrEqual(2_048); + expect(inspection.diagnostics).toMatch(/\.\.\.$/u); + }); + it("fails closed when the Linux member snapshot is unavailable", () => { const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux"); const processKill = vi.spyOn(process, "kill").mockImplementation(() => true); diff --git a/extensions/qa-lab/src/posix-process-group.ts b/extensions/qa-lab/src/posix-process-group.ts index 7e2d328ab817..e8e21b8bc92d 100644 --- a/extensions/qa-lab/src/posix-process-group.ts +++ b/extensions/qa-lab/src/posix-process-group.ts @@ -1,66 +1,6 @@ import { readFileSync, readdirSync } from "node:fs"; import path from "node:path"; -import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; - -function parseLinuxProcessStat(raw: string) { - const commandStart = raw.indexOf("("); - const commandEnd = raw.lastIndexOf(")"); - if (commandStart <= 0 || commandEnd <= commandStart) { - return null; - } - const pid = Number.parseInt(raw.slice(0, commandStart).trim(), 10); - const fields = raw - .slice(commandEnd + 1) - .trim() - .split(/\s+/u); - const state = fields[0]; - const processGroupId = Number.parseInt(fields[2] ?? "", 10); - if ( - !Number.isSafeInteger(pid) || - pid <= 0 || - !state || - !Number.isSafeInteger(processGroupId) || - processGroupId <= 0 - ) { - return null; - } - return { - command: raw.slice(commandStart + 1, commandEnd), - pid, - processGroupId, - state, - }; -} - -function boundProcessGroupDiagnostics(details: string) { - if (details.length <= 2_048) { - return details; - } - return `${sliceUtf16Safe(details, 0, 2_045)}...`; -} - -export function inspectLinuxProcessGroupStats(processGroupId: number, stats: readonly string[]) { - const members = stats - .map((raw) => parseLinuxProcessStat(raw)) - .filter( - (entry): entry is NonNullable> => - entry?.processGroupId === processGroupId, - ) - .toSorted((left, right) => left.pid - right.pid); - const diagnostics = members - .map( - (member) => - `pid=${member.pid} state=${member.state} command=${JSON.stringify(member.command)}`, - ) - .join(", "); - return { - alive: - members.length === 0 - ? null - : members.some((entry) => entry.state !== "Z" && entry.state !== "X"), - diagnostics: boundProcessGroupDiagnostics(`pgid=${processGroupId} members=[${diagnostics}]`), - }; -} +import { inspectLinuxProcessGroupStats } from "./posix-process-stat.js"; type QaLinuxProcessGroupInspection = ReturnType; export type QaLinuxProcessGroupInspector = ( diff --git a/extensions/qa-lab/src/posix-process-stat.ts b/extensions/qa-lab/src/posix-process-stat.ts new file mode 100644 index 000000000000..283f05047f30 --- /dev/null +++ b/extensions/qa-lab/src/posix-process-stat.ts @@ -0,0 +1,62 @@ +// Qa Lab parses Linux process stat snapshots for process-group ownership. +import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; + +function parseLinuxProcessStat(raw: string) { + const commandStart = raw.indexOf("("); + const commandEnd = raw.lastIndexOf(")"); + if (commandStart <= 0 || commandEnd <= commandStart) { + return null; + } + const pid = Number.parseInt(raw.slice(0, commandStart).trim(), 10); + const fields = raw + .slice(commandEnd + 1) + .trim() + .split(/\s+/u); + const state = fields[0]; + const processGroupId = Number.parseInt(fields[2] ?? "", 10); + if ( + !Number.isSafeInteger(pid) || + pid <= 0 || + !state || + !Number.isSafeInteger(processGroupId) || + processGroupId <= 0 + ) { + return null; + } + return { + command: raw.slice(commandStart + 1, commandEnd), + pid, + processGroupId, + state, + }; +} + +function boundProcessGroupDiagnostics(details: string) { + if (details.length <= 2_048) { + return details; + } + return `${sliceUtf16Safe(details, 0, 2_045)}...`; +} + +export function inspectLinuxProcessGroupStats(processGroupId: number, stats: readonly string[]) { + const members = stats + .map((raw) => parseLinuxProcessStat(raw)) + .filter( + (entry): entry is NonNullable> => + entry?.processGroupId === processGroupId, + ) + .toSorted((left, right) => left.pid - right.pid); + const diagnostics = members + .map( + (member) => + `pid=${member.pid} state=${member.state} command=${JSON.stringify(member.command)}`, + ) + .join(", "); + return { + alive: + members.length === 0 + ? null + : members.some((entry) => entry.state !== "Z" && entry.state !== "X"), + diagnostics: boundProcessGroupDiagnostics(`pgid=${processGroupId} members=[${diagnostics}]`), + }; +} diff --git a/extensions/qa-lab/src/providers/live-config.ts b/extensions/qa-lab/src/providers/live-config.ts new file mode 100644 index 000000000000..19438c66458f --- /dev/null +++ b/extensions/qa-lab/src/providers/live-config.ts @@ -0,0 +1,70 @@ +// Qa Lab plugin module owns host live-provider config projection. +import { existsSync } from "node:fs"; +import fs from "node:fs/promises"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; +import { + isRecord, + normalizeOptionalString, + normalizeStringEntries, + uniqueStrings, +} from "openclaw/plugin-sdk/string-coerce-runtime"; +import { QA_LIVE_PROVIDER_CONFIG_PATH_ENV, resolveQaLiveProviderConfigPath } from "./env.js"; + +function isQaModelProviderConfig(value: unknown): value is ModelProviderConfig { + return isRecord(value) && typeof value.baseUrl === "string" && Array.isArray(value.models); +} + +function normalizeQaLiveProviderConfig(value: unknown): ModelProviderConfig | null { + if (!isQaModelProviderConfig(value) && (!isRecord(value) || !Object.hasOwn(value, "apiKey"))) { + return null; + } + const { baseUrl: rawBaseUrl, ...providerConfig } = value; + const baseUrl = normalizeOptionalString(rawBaseUrl); + return { + ...providerConfig, + ...(baseUrl ? { baseUrl } : {}), + models: Array.isArray(value.models) ? value.models : [], + } as ModelProviderConfig; +} + +export async function readQaLiveProviderConfigOverrides(params: { + providerIds: readonly string[]; + env?: NodeJS.ProcessEnv; +}) { + const providerIds = uniqueStrings(normalizeStringEntries(params.providerIds)); + if (providerIds.length === 0) { + return {}; + } + const configPath = resolveQaLiveProviderConfigPath(params.env); + if (!existsSync(configPath.path)) { + return {}; + } + try { + const raw = await fs.readFile(configPath.path, "utf8"); + const parsed = JSON.parse(raw) as unknown; + const providers = isRecord(parsed) + ? isRecord(parsed.models) + ? isRecord(parsed.models.providers) + ? parsed.models.providers + : {} + : {} + : {}; + const selected: Record = {}; + for (const providerId of providerIds) { + const providerConfig = normalizeQaLiveProviderConfig(providers[providerId]); + if (providerConfig) { + selected[providerId] = providerConfig; + } + } + return selected; + } catch (error) { + if (configPath.explicit) { + throw new Error( + `failed to read ${QA_LIVE_PROVIDER_CONFIG_PATH_ENV} provider config: ${formatErrorMessage(error)}`, + { cause: error }, + ); + } + return {}; + } +} diff --git a/extensions/qa-lab/src/providers/shared/mock-model-config.ts b/extensions/qa-lab/src/providers/shared/mock-model-config.ts index 215c167cb1ef..e385f13f6b38 100644 --- a/extensions/qa-lab/src/providers/shared/mock-model-config.ts +++ b/extensions/qa-lab/src/providers/shared/mock-model-config.ts @@ -159,7 +159,7 @@ export function listMockCodexModelInfos(selectedModelRefs: readonly string[] = [ upgrade: null, base_instructions: "You are Codex, a coding agent based on GPT-5.", include_skills_usage_instructions: false, - supports_reasoning_summaries: true, + supports_reasoning_summary_parameter: true, default_reasoning_summary: "none", support_verbosity: true, default_verbosity: "low", diff --git a/extensions/qa-lab/src/windows-system-tools.test.ts b/extensions/qa-lab/src/windows-system-tools.test.ts index 24b8a03cfa9c..88490a2b07d7 100644 --- a/extensions/qa-lab/src/windows-system-tools.test.ts +++ b/extensions/qa-lab/src/windows-system-tools.test.ts @@ -1,8 +1,9 @@ // Qa Lab tests cover Windows system tool path resolution. -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { resolveQaWindowsPowerShellExePath, resolveQaWindowsSystem32ExePath, + runQaWindowsTaskkill, } from "./windows-system-tools.js"; describe("qa-lab windows system tools", () => { @@ -15,6 +16,34 @@ describe("qa-lab windows system tools", () => { ); }); + it("force-kills a process tree when graceful taskkill fails", () => { + const runCommand = vi + .fn() + .mockReturnValueOnce({ status: 1 }) + .mockReturnValueOnce({ status: 0 }); + + expect( + runQaWindowsTaskkill({ + pid: 12345, + signal: "SIGTERM", + env: { SystemRoot: "D:\\Windows" }, + runCommand, + }), + ).toBe(true); + expect(runCommand).toHaveBeenNthCalledWith( + 1, + "D:\\Windows\\System32\\taskkill.exe", + ["/PID", "12345", "/T"], + { stdio: "ignore", windowsHide: true, timeout: 5_000 }, + ); + expect(runCommand).toHaveBeenNthCalledWith( + 2, + "D:\\Windows\\System32\\taskkill.exe", + ["/PID", "12345", "/T", "/F"], + { stdio: "ignore", windowsHide: true, timeout: 5_000 }, + ); + }); + it("falls back to the default Windows root when env roots are unsafe", () => { expect(resolveQaWindowsSystem32ExePath("taskkill.exe", { SystemRoot: "C:\\tmp;C:\\bad" })).toBe( "C:\\Windows\\System32\\taskkill.exe", diff --git a/extensions/qa-lab/src/windows-system-tools.ts b/extensions/qa-lab/src/windows-system-tools.ts index 3a19c094b6bb..5d97b7f98829 100644 --- a/extensions/qa-lab/src/windows-system-tools.ts +++ b/extensions/qa-lab/src/windows-system-tools.ts @@ -1,4 +1,5 @@ // Qa Lab resolves Windows system tools without trusting PATH. +import { spawnSync } from "node:child_process"; import path from "node:path"; const DEFAULT_WINDOWS_SYSTEM_ROOT = "C:\\Windows"; @@ -59,6 +60,37 @@ export function resolveQaWindowsSystem32ExePath( return path.win32.join(resolveQaWindowsSystemRoot(env), "System32", executableName); } +export function runQaWindowsTaskkill(params: { + pid: number; + signal: NodeJS.Signals; + env?: Record; + runCommand?: typeof spawnSync; +}) { + const runCommand = params.runCommand ?? spawnSync; + const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe", params.env); + const args = ["/PID", String(params.pid), "/T"]; + if (params.signal === "SIGKILL") { + args.push("/F"); + } + const result = runCommand(taskkillPath, args, { + stdio: "ignore", + windowsHide: true, + timeout: 5_000, + }); + if (!result.error && result.status === 0) { + return true; + } + if (params.signal !== "SIGKILL") { + const forceResult = runCommand(taskkillPath, [...args, "/F"], { + stdio: "ignore", + windowsHide: true, + timeout: 5_000, + }); + return !forceResult.error && forceResult.status === 0; + } + return false; +} + export function resolveQaWindowsPowerShellExePath( env: Record = process.env, ): string { diff --git a/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml b/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml index 1c1a02d33094..1402310242f5 100644 --- a/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml +++ b/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml @@ -20,7 +20,8 @@ scenario: - docs/channels/qa-channel.md codeRefs: - src/agents/system-prompt.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts - extensions/qa-lab/src/providers/mock-openai/server.ts execution: kind: flow diff --git a/qa/scenarios/channels/qa-channel-failed-tool-terminal-finalization.yaml b/qa/scenarios/channels/qa-channel-failed-tool-terminal-finalization.yaml index 46e6e7fc5a07..c2e039193e63 100644 --- a/qa/scenarios/channels/qa-channel-failed-tool-terminal-finalization.yaml +++ b/qa/scenarios/channels/qa-channel-failed-tool-terminal-finalization.yaml @@ -18,7 +18,8 @@ scenario: - The failed tool call is never replayed or described as successful. - The model receives one tools-disabled finalization and qa-channel delivers exactly one visible reply. codeRefs: - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts - src/agents/embedded-agent-runner/run/code-mode-repair.ts - extensions/qa-lab/src/providers/mock-openai/server.ts execution: diff --git a/qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml b/qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml index bed5f7b0f2bd..5ff9214d0c8b 100644 --- a/qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml +++ b/qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml @@ -15,7 +15,8 @@ scenario: - The runtime continues from settled tool results without replaying the write. - Telegram receives the exact recovery marker. codeRefs: - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts - src/agents/embedded-agent-runner/run/terminal-resolution.ts - extensions/qa-lab/src/providers/mock-openai/server.ts execution: diff --git a/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml index 5739143c438e..b144f23bbbc8 100644 --- a/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml @@ -27,7 +27,8 @@ scenario: - docs/help/testing.md codeRefs: - extensions/qa-lab/src/providers/mock-openai/server.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Verify Anthropic stream errors after signed thinking recover after a replay-safe read. diff --git a/qa/scenarios/runtime/approval-turn-tool-followthrough.yaml b/qa/scenarios/runtime/approval-turn-tool-followthrough.yaml index 6ecfbe07a281..b89a35fcffc4 100644 --- a/qa/scenarios/runtime/approval-turn-tool-followthrough.yaml +++ b/qa/scenarios/runtime/approval-turn-tool-followthrough.yaml @@ -18,7 +18,8 @@ scenario: codeRefs: - extensions/qa-lab/src/suite.ts - extensions/qa-lab/src/providers/mock-openai/server.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Verify a short approval like "ok do it" triggers immediate tool use instead of fake-progress narration. diff --git a/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml index 5955ffdbcf89..dbdaa04ed824 100644 --- a/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml @@ -21,7 +21,8 @@ scenario: - docs/help/testing.md codeRefs: - extensions/qa-lab/src/providers/mock-openai/server.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Verify empty OpenAI turns recover after a replay-safe read. diff --git a/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml b/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml index 43ed0cf53328..a981795358c8 100644 --- a/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml +++ b/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml @@ -19,7 +19,8 @@ scenario: - docs/help/testing.md codeRefs: - extensions/qa-lab/src/providers/mock-openai/server.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Verify empty-response retry exhaustion still surfaces a visible failure. diff --git a/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml b/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml index a1e4a00a3ec4..d67a42783f49 100644 --- a/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml +++ b/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml @@ -20,7 +20,8 @@ scenario: - docs/help/testing.md codeRefs: - extensions/qa-lab/src/providers/mock-openai/server.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Verify reasoning-only turns after a write do not auto-retry. diff --git a/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml index fbb8cc630a72..135b6652c378 100644 --- a/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml @@ -18,7 +18,8 @@ scenario: - docs/help/testing.md codeRefs: - extensions/qa-lab/src/providers/mock-openai/server.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Verify reasoning-only OpenAI turns recover after a replay-safe read. diff --git a/qa/scenarios/runtime/streaming-final-integrity.yaml b/qa/scenarios/runtime/streaming-final-integrity.yaml index ef40c9c61fdd..2a369ed2e1f7 100644 --- a/qa/scenarios/runtime/streaming-final-integrity.yaml +++ b/qa/scenarios/runtime/streaming-final-integrity.yaml @@ -19,7 +19,8 @@ scenario: - docs/concepts/streaming.md - docs/channels/qa-channel.md codeRefs: - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts - extensions/qa-lab/src/bus-state.ts - extensions/qa-lab/src/suite-runtime-transport.ts execution: diff --git a/qa/scenarios/runtime/telemetry-task-evidence-followthrough.yaml b/qa/scenarios/runtime/telemetry-task-evidence-followthrough.yaml index 0f0e62e50e2f..a3c3d07a28e5 100644 --- a/qa/scenarios/runtime/telemetry-task-evidence-followthrough.yaml +++ b/qa/scenarios/runtime/telemetry-task-evidence-followthrough.yaml @@ -20,7 +20,8 @@ scenario: codeRefs: - extensions/qa-channel/src/inbound.ts - extensions/qa-lab/src/providers/mock-openai/server.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Complete a proof-backed QA-channel task and verify artifact-before-reply ordering plus honest terminal status. diff --git a/qa/scenarios/scheduling/cron-failed-tool-terminal-finalization.yaml b/qa/scenarios/scheduling/cron-failed-tool-terminal-finalization.yaml index 980edac165c6..81cf64ade30d 100644 --- a/qa/scenarios/scheduling/cron-failed-tool-terminal-finalization.yaml +++ b/qa/scenarios/scheduling/cron-failed-tool-terminal-finalization.yaml @@ -19,7 +19,8 @@ scenario: - The failed tool never replays and qa-channel receives exactly one final reply. codeRefs: - src/cron/isolated-agent/run-executor.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts - extensions/qa-lab/src/providers/mock-openai/server.ts execution: kind: flow diff --git a/qa/scenarios/security/secret-redaction-tool-logs.yaml b/qa/scenarios/security/secret-redaction-tool-logs.yaml index 7f4d899589d6..74820c24891c 100644 --- a/qa/scenarios/security/secret-redaction-tool-logs.yaml +++ b/qa/scenarios/security/secret-redaction-tool-logs.yaml @@ -20,7 +20,8 @@ scenario: codeRefs: - extensions/qa-lab/src/suite-runtime-agent-process.ts - extensions/qa-lab/src/suite-runtime-transport.ts - - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts + - src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts execution: kind: flow summary: Verify fake secret fixtures are not echoed into channel-visible output. diff --git a/qa/scenarios/ui/control-ui-config-safe-write.yaml b/qa/scenarios/ui/control-ui-config-safe-write.yaml index a2563c20cee6..4cec548b0257 100644 --- a/qa/scenarios/ui/control-ui-config-safe-write.yaml +++ b/qa/scenarios/ui/control-ui-config-safe-write.yaml @@ -27,7 +27,7 @@ scenario: - docs/gateway/protocol.md codeRefs: - ui/src/e2e/config-safe-write.e2e.test.ts - - ui/src/lib/config/index.ts + - ui/src/lib/config/config-write-coordinator.ts - ui/src/components/settings-save-indicator.ts execution: kind: playwright