refactor(qa): remove gateway child test facade (#122693)

* refactor(qa): remove gateway child test facade

* test(qa): refresh scenario source references

* fix(qa): preserve packaged auth redaction boundary
This commit is contained in:
Peter Steinberger
2026-08-12 09:03:40 -07:00
committed by GitHub
parent 28a5889e9b
commit 71e8cc033b
31 changed files with 1409 additions and 1816 deletions
-4
View File
@@ -99,14 +99,10 @@ export {
} from "./src/self-check.js";
export { runQaE2eSelfCheck, runQaLabSelfCheck } from "./src/self-check-runner.js";
export {
testing,
buildQaRuntimeEnv,
type QaCliBackendAuthMode,
type QaGatewayChildListeningContext,
type QaGatewayChildCommand,
type QaGatewayChildStateMutationContext,
resolveQaControlUiRoot,
resolveQaGatewayChildProviderMode,
startQaGatewayChild,
} from "./src/gateway-child.js";
export {
@@ -36,7 +36,7 @@ function isQaOpenAiResponsesProviderConfig(config: ModelProviderConfig) {
);
}
export function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) {
function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) {
assertSafeQaBundledPluginId(params.pluginId);
const candidates = [
path.join(params.repoRoot, "dist", "extensions", params.pluginId),
-32
View File
@@ -1,5 +1,4 @@
// Qa Lab plugin module implements cli paths behavior.
import fs from "node:fs/promises";
import path from "node:path";
import { assertNoSymlinkParents, pathScope } from "openclaw/plugin-sdk/security-runtime";
@@ -29,25 +28,6 @@ export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: strin
return resolved.path;
}
async function resolveNearestExistingPath(targetPath: string) {
let current = path.resolve(targetPath);
while (true) {
try {
await fs.lstat(current);
return current;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
}
}
const parent = path.dirname(current);
if (parent === current) {
throw new Error(`failed to resolve existing path for ${targetPath}`);
}
current = parent;
}
}
function assertRepoRelativePath(repoRoot: string, targetPath: string, label: string) {
const relative = path.relative(repoRoot, targetPath);
if (relative.startsWith("..") || path.isAbsolute(relative)) {
@@ -72,18 +52,6 @@ async function assertNoSymlinkSegments(repoRoot: string, targetPath: string, lab
}
}
export async function assertRepoBoundPath(repoRoot: string, targetPath: string, label: string) {
const repoRootResolved = path.resolve(repoRoot);
const targetResolved = path.resolve(targetPath);
assertRepoRelativePath(repoRootResolved, targetResolved, label);
await assertNoSymlinkSegments(repoRootResolved, targetResolved, label);
const repoRootReal = await fs.realpath(repoRootResolved);
const nearestExistingPath = await resolveNearestExistingPath(targetResolved);
const nearestExistingReal = await fs.realpath(nearestExistingPath);
assertRepoRelativePath(repoRootReal, nearestExistingReal, label);
return targetResolved;
}
export async function ensureRepoBoundDirectory(
repoRoot: string,
targetDir: string,
@@ -0,0 +1,72 @@
// Qa Lab plugin module owns sanitized gateway debug artifacts and temp cleanup.
import fs from "node:fs/promises";
import path from "node:path";
import { ensureRepoBoundDirectory } from "./cli-paths.js";
import { redactQaGatewayDebugText } from "./gateway-log-redaction.js";
async function writeSanitizedQaGatewayDebugLog(params: { sourcePath: string; targetPath: string }) {
const contents = await fs.readFile(params.sourcePath, "utf8").catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return "";
}
throw error;
});
await fs.writeFile(params.targetPath, redactQaGatewayDebugText(contents), "utf8");
}
async function clearQaGatewayArtifactDir(dir: string) {
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
await fs.rm(path.join(dir, entry.name), { recursive: true, force: true });
}
}
export async function cleanupQaGatewayTempRoots(params: {
tempRoot: string;
stagedBundledPluginsRoot?: string | null;
}) {
await fs.rm(params.tempRoot, { recursive: true, force: true }).catch(() => {});
if (params.stagedBundledPluginsRoot) {
await fs.rm(params.stagedBundledPluginsRoot, { recursive: true, force: true }).catch(() => {});
}
}
export async function preserveQaGatewayDebugArtifacts(params: {
preserveToDir: string;
stdoutLogPath: string;
stderrLogPath: string;
tempRoot: string;
repoRoot?: string;
}) {
const preserveToDir = params.repoRoot
? await ensureRepoBoundDirectory(
params.repoRoot,
params.preserveToDir,
"QA gateway artifact directory",
{
mode: 0o700,
},
)
: params.preserveToDir;
await fs.mkdir(preserveToDir, { recursive: true, mode: 0o700 });
await clearQaGatewayArtifactDir(preserveToDir);
await Promise.all([
writeSanitizedQaGatewayDebugLog({
sourcePath: params.stdoutLogPath,
targetPath: path.join(preserveToDir, "gateway.stdout.log"),
}),
writeSanitizedQaGatewayDebugLog({
sourcePath: params.stderrLogPath,
targetPath: path.join(preserveToDir, "gateway.stderr.log"),
}),
]);
await fs.writeFile(
path.join(preserveToDir, "README.txt"),
[
"Only sanitized gateway debug artifacts are preserved here.",
"The full QA gateway runtime was not copied because it may contain credentials or auth tokens.",
"Original runtime temp root omitted because local temp paths can identify the runner.",
"",
].join("\n"),
"utf8",
);
}
@@ -0,0 +1,106 @@
// Qa Lab plugin module owns gateway child command bootstrap behavior.
import { spawn, type ChildProcess } from "node:child_process";
import { existsSync } from "node:fs";
import path from "node:path";
import { formatErrorMessage, toErrorObject } from "openclaw/plugin-sdk/error-runtime";
import {
appendQaChildOutput,
appendQaChildOutputTail,
createQaChildOutputCapture,
createQaChildOutputTail,
formatQaChildOutputTail,
readQaChildOutput,
} from "./child-output.js";
import { hasQaGatewayChildExited, monitorQaChildFailure } from "./gateway-child-process.js";
import type { QaGatewayProcessBoundaryConfig } from "./gateway-process-boundary.js";
type QaGatewayChildDirectCommand = {
executablePath: string;
argsPrefix?: string[];
argsSuffix?: string[];
cwd?: string;
tempParentDir?: string;
usePackagedPlugins?: boolean;
processBoundary?: undefined;
};
type QaGatewayChildVerifiedCommand = Omit<QaGatewayChildDirectCommand, "processBoundary"> & {
processBoundary: QaGatewayProcessBoundaryConfig;
};
export type QaGatewayChildCommand = QaGatewayChildDirectCommand | QaGatewayChildVerifiedCommand;
export function resolveQaGatewayChildCommand(repoRoot: string): QaGatewayChildCommand {
for (const relativePath of ["scripts/run-node.mjs", "dist/index.mjs", "dist/index.js"]) {
const entryPath = path.join(repoRoot, relativePath);
if (existsSync(entryPath)) {
return {
executablePath: process.execPath,
argsPrefix: [entryPath],
cwd: repoRoot,
usePackagedPlugins: true,
};
}
}
throw new Error(
"OpenClaw CLI entry not found: expected scripts/run-node.mjs or dist/index.(m)js",
);
}
export async function runQaGatewayCliCommand(params: {
executablePath: string;
argsPrefix: readonly string[];
args: readonly string[];
cwd: string;
env: NodeJS.ProcessEnv;
stdin?: string;
}): Promise<string> {
const hasStdin = params.stdin !== undefined;
const child = spawn(params.executablePath, [...params.argsPrefix, ...params.args], {
cwd: params.cwd,
env: { ...params.env, OPENCLAW_CLI: "1" },
stdio: [hasStdin ? "pipe" : "ignore", "pipe", "pipe"],
});
const result = readQaGatewayCliCommand(child);
if (hasStdin) {
child.stdin?.once("error", () => {});
child.stdin?.end(params.stdin);
}
return await result;
}
async function readQaGatewayCliCommand(child: ChildProcess): Promise<string> {
const stdout = createQaChildOutputCapture();
const stderr = createQaChildOutputTail();
child.stdout?.on("data", (chunk) => appendQaChildOutput(stdout, chunk));
child.stderr?.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk));
const exitCode = await new Promise<number>((resolve, reject) => {
monitorQaChildFailure(child, (failure) => {
if (failure.source === "process") {
reject(toErrorObject(failure.error, "OpenClaw CLI process failed"));
return;
}
if (!hasQaGatewayChildExited(child) && !child.killed) {
try {
child.kill("SIGKILL");
} catch {
// The child exited between the state check and signal.
}
}
reject(
new Error(
`qa gateway cli ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`,
{ cause: failure.error },
),
);
});
child.once("close", (code) => resolve(code ?? 1));
});
const stdoutText = readQaChildOutput(stdout);
if (exitCode !== 0) {
const stderrText = formatQaChildOutputTail(stderr, "stderr");
throw new Error(`OpenClaw CLI exited ${exitCode}: ${stderrText || stdoutText}`);
}
return stdoutText;
}
+181
View File
@@ -0,0 +1,181 @@
// Qa Lab plugin module owns gateway child runtime environment behavior.
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { buildQaCodexAppServerArgs } from "./codex-app-server-args.js";
import type { QaProviderMode } from "./model-selection.js";
import {
normalizeQaProviderModeEnv,
resolveQaLiveCliAuthEnv,
type QaCliBackendAuthMode,
} from "./providers/env.js";
import { getQaProvider } from "./providers/index.js";
import {
QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV,
QA_LIVE_SETUP_TOKEN_VALUE_ENV,
} from "./providers/live-frontier/auth.js";
import { listMockCodexModelInfos } from "./providers/shared/mock-model-config.js";
import type { RuntimeId } from "./runtime-parity.js";
const QA_MOCK_OPENAI_API_KEY = ["qa", "mock", "openai", "key"].join("-");
const QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS = Object.freeze([
"OPENCLAW_QA_CONVEX_SECRET_CI",
"OPENCLAW_QA_CONVEX_SECRET_MAINTAINER",
"OPENCLAW_QA_SUT_FORBIDDEN_SENTINEL",
"OPENCLAW_QA_TELEGRAM_GROUP_ID",
"OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN",
"OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN",
]);
function scrubQaGatewayChildSecretEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
for (const envKey of QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS) {
delete env[envKey];
}
return env;
}
function scrubQaGatewayChildTestRunnerEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
// The Gateway is a product child, not a nested Vitest worker. Leaking runner
// markers makes the dist launcher select test-only startup behavior.
delete env.VITEST;
delete env.VITEST_POOL_ID;
delete env.VITEST_WORKER_ID;
if (env.NODE_ENV === "test") {
delete env.NODE_ENV;
}
return env;
}
export function buildQaRuntimeEnv(params: {
configPath: string;
gatewayToken: string;
homeDir: string;
forwardHostHome?: boolean;
stateDir: string;
tempRoot: string;
xdgConfigHome: string;
xdgDataHome: string;
xdgCacheHome: string;
bundledPluginsDir?: string;
stagedBundledPluginsRoot?: string | null;
compatibilityHostVersion?: string;
providerMode?: QaProviderMode;
baseEnv?: NodeJS.ProcessEnv;
runtimeEnvPatch?: NodeJS.ProcessEnv;
forwardHostHomeForClaudeCli?: boolean;
claudeCliAuthMode?: QaCliBackendAuthMode;
}) {
const baseEnv = params.baseEnv ?? process.env;
const provider = params.providerMode ? getQaProvider(params.providerMode) : null;
const forwardedHostHome = params.forwardHostHome
? baseEnv.HOME?.trim() || os.homedir()
: undefined;
const env: NodeJS.ProcessEnv = {
...baseEnv,
HOME: forwardedHostHome ?? params.homeDir,
...(provider?.appliesLiveEnvAliases
? resolveQaLiveCliAuthEnv(baseEnv, {
forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli,
claudeCliAuthMode: params.claudeCliAuthMode,
})
: {}),
OPENCLAW_HOME: params.homeDir,
OPENCLAW_CONFIG_PATH: params.configPath,
OPENCLAW_STATE_DIR: params.stateDir,
OPENCLAW_OAUTH_DIR: path.join(params.stateDir, "credentials"),
OPENCLAW_GATEWAY_TOKEN: params.gatewayToken,
OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1",
OPENCLAW_SKIP_GMAIL_WATCHER: "1",
OPENCLAW_SKIP_CANVAS_HOST: "1",
OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1",
OPENCLAW_NO_RESPAWN: "1",
OPENCLAW_TEST_FAST: "1",
OPENCLAW_EMBEDDED_ABORT_SETTLE_TIMEOUT_MS: "2000",
OPENCLAW_QA_PARENT_PID: String(process.pid),
OPENCLAW_QA_TEMP_ROOT: params.tempRoot,
...(params.stagedBundledPluginsRoot
? { OPENCLAW_QA_STAGED_RUNTIME_ROOT: params.stagedBundledPluginsRoot }
: {}),
OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER: "1",
// QA uses the fast runtime envelope for speed, but it still exercises
// normal config-driven heartbeats and runtime config writes.
OPENCLAW_ALLOW_SLOW_REPLY_TESTS: "1",
XDG_CONFIG_HOME: params.xdgConfigHome,
XDG_DATA_HOME: params.xdgDataHome,
XDG_CACHE_HOME: params.xdgCacheHome,
...(params.bundledPluginsDir ? { OPENCLAW_BUNDLED_PLUGINS_DIR: params.bundledPluginsDir } : {}),
...(params.compatibilityHostVersion
? { OPENCLAW_COMPATIBILITY_HOST_VERSION: params.compatibilityHostVersion }
: {}),
};
const normalizedEnv = normalizeQaProviderModeEnv(env, params.providerMode);
// Test-runner skip flags are parent controls; each QA child declares its own runtime needs.
delete normalizedEnv.OPENCLAW_SKIP_CHANNELS;
delete normalizedEnv.OPENCLAW_SKIP_PROVIDERS;
Object.assign(normalizedEnv, params.runtimeEnvPatch);
normalizedEnv.OPENCLAW_BUILD_PRIVATE_QA = "1";
delete normalizedEnv[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV];
delete normalizedEnv[QA_LIVE_SETUP_TOKEN_VALUE_ENV];
return scrubQaGatewayChildSecretEnv(scrubQaGatewayChildTestRunnerEnv(normalizedEnv));
}
export async function stageQaCodexMockModelCatalog(params: {
tempRoot: string;
forcedRuntime?: RuntimeId;
providerMode: QaProviderMode;
primaryModel?: string;
alternateModel?: string;
}): Promise<string | undefined> {
if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") {
return undefined;
}
const modelCatalogPath = path.join(params.tempRoot, "codex-model-catalog.json");
const selectedModelRefs = [params.primaryModel, params.alternateModel].filter(
(model): model is string => typeof model === "string" && model.length > 0,
);
await fs.writeFile(
modelCatalogPath,
`${JSON.stringify({ models: listMockCodexModelInfos(selectedModelRefs) }, null, 2)}\n`,
{ encoding: "utf8", mode: 0o600 },
);
return modelCatalogPath;
}
export function buildQaForcedRuntimeEnvPatch(params: {
forcedRuntime?: RuntimeId;
providerMode: QaProviderMode;
providerBaseUrl?: string;
codexModelCatalogPath?: string;
nativeAppServerArgs?: string;
}): NodeJS.ProcessEnv | undefined {
if (!params.forcedRuntime) {
return undefined;
}
const patch: NodeJS.ProcessEnv = {
OPENCLAW_BUILD_PRIVATE_QA: "1",
OPENCLAW_QA_FORCE_RUNTIME: params.forcedRuntime,
};
if (params.forcedRuntime !== "codex") {
return patch;
}
if (params.providerMode !== "mock-openai") {
patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({
existingArgs: params.nativeAppServerArgs,
});
return patch;
}
const providerBaseUrl = params.providerBaseUrl?.trim().replace(/\/+$/u, "");
if (!providerBaseUrl) {
throw new Error("forced Codex mock QA requires the managed mock provider URL");
}
if (!params.codexModelCatalogPath) {
throw new Error("forced Codex mock QA requires the staged native model catalog");
}
patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({
providerBaseUrl,
modelCatalogPath: params.codexModelCatalogPath,
});
patch.OPENAI_API_KEY = QA_MOCK_OPENAI_API_KEY;
patch.CODEX_API_KEY = QA_MOCK_OPENAI_API_KEY;
return patch;
}
@@ -0,0 +1,295 @@
// Qa Lab plugin module owns gateway child process lifecycle behavior.
import type { ChildProcess } from "node:child_process";
import type { WriteStream } from "node:fs";
import { finished } from "node:stream/promises";
import { StringDecoder } from "node:string_decoder";
import { setTimeout as sleep } from "node:timers/promises";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
import { QaSuiteInfraError } from "./errors.js";
import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js";
import {
inspectLinuxProcessGroup,
type QaLinuxProcessGroupInspector,
} from "./posix-process-group.js";
import { runQaWindowsTaskkill } from "./windows-system-tools.js";
const QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 30_000;
const QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS = 10_000;
const QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS = 5_000;
const QA_GATEWAY_CHILD_RECENT_LOG_CHARS = 64 * 1_024;
const QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER = "[qa-lab] older gateway logs truncated\n";
const QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS = 8_192;
export type QaChildFailure = {
source: "process" | "stdout" | "stderr";
error: unknown;
};
type QaGatewayChildLogSource = "internal" | "stderr" | "stdout";
export function hasQaGatewayChildExited(child: Pick<ChildProcess, "exitCode" | "signalCode">) {
return child.exitCode !== null || child.signalCode !== null;
}
export function monitorQaChildFailure(
child: ChildProcess,
onFailure: (failure: QaChildFailure) => void,
) {
let reported = false;
const report = (source: QaChildFailure["source"]) => (error: unknown) => {
if (reported) {
return;
}
reported = true;
onFailure({ source, error });
};
child.once("error", report("process"));
child.stdout?.once("error", report("stdout"));
child.stderr?.once("error", report("stderr"));
}
export async function closeQaGatewayLogStream(
stream: WriteStream,
label: "stderr" | "stdout",
timeoutMs = QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS,
) {
if (stream.destroyed) {
return;
}
stream.end();
const signal = AbortSignal.timeout(timeoutMs);
try {
await finished(stream, { cleanup: true, signal });
} catch (error) {
if (!signal.aborted) {
throw error;
}
// Gateway logs are diagnostic only. Never let a stuck filesystem flush
// retain the stopped child runtime and its live transport credentials.
process.stderr.write(
`[qa-suite] ${label} gateway log flush exceeded ${timeoutMs}ms; forcing close\n`,
);
stream.destroy();
}
}
export function createQaGatewayChildLogCollector() {
const decoders: Record<QaGatewayChildLogSource, StringDecoder> = {
internal: new StringDecoder("utf8"),
stderr: new StringDecoder("utf8"),
stdout: new StringDecoder("utf8"),
};
let recent = "";
let end = 0;
const readFrom = (mark: number) => {
const start = end - recent.length;
const wasTruncated = mark < start;
const text = recent.slice(Math.max(0, mark - start));
return `${wasTruncated ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${text}`;
};
return {
push(source: QaGatewayChildLogSource, chunk: Buffer) {
const text = decoders[source].write(chunk);
end += text.length;
recent += text;
if (recent.length > QA_GATEWAY_CHILD_RECENT_LOG_CHARS) {
recent = sliceUtf16Safe(recent, -QA_GATEWAY_CHILD_RECENT_LOG_CHARS);
}
},
mark() {
return end;
},
readSince(mark: number) {
return readFrom(mark);
},
text() {
return `${end > recent.length ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${recent}`.trim();
},
};
}
function formatQaGatewayChildFailure(failure: QaChildFailure) {
return failure.source === "process"
? `gateway failed to spawn: ${formatErrorMessage(failure.error)}`
: `gateway child ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`;
}
export function throwQaGatewayChildFailure(
getChildFailure: (() => QaChildFailure | null) | undefined,
logs: () => string,
) {
const failure = getChildFailure?.();
if (!failure) {
return;
}
throw new QaSuiteInfraError(
"gateway_startup_unhealthy",
`${formatQaGatewayChildFailure(failure)}\n${logs()}`,
{ cause: failure.error },
);
}
export function monitorQaGatewayChildFailure(
child: ChildProcess,
output: { push(source: QaGatewayChildLogSource, chunk: Buffer): void },
) {
let childFailure: QaChildFailure | null = null;
monitorQaChildFailure(child, (failure) => {
childFailure = failure;
const description =
failure.source === "process"
? `gateway child process error: ${formatErrorMessage(failure.error)}`
: formatQaGatewayChildFailure(failure);
output.push("internal", Buffer.from(`[qa-lab] ${description}\n`));
if (failure.source !== "process" && !hasQaGatewayChildExited(child)) {
// A broken parent-side pipe means QA can no longer observe the Gateway.
// Stop the detached process tree so the existing lifecycle reports the failure.
signalQaGatewayChildProcessTree(child, "SIGTERM");
}
});
return () => childFailure;
}
export function formatQaGatewayProcessBoundaryStartupFailure(error: unknown, logs: string) {
const logTail = sliceUtf16Safe(
redactQaGatewayDebugText(logs),
-QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS,
);
return `${formatErrorMessage(error)}${formatQaGatewayLogsForError(logTail)}`;
}
function isProcessAlreadyExitedError(error: unknown): boolean {
return (error as NodeJS.ErrnoException | undefined)?.code === "ESRCH";
}
function boundQaGatewayProcessTreeDiagnostics(details: string) {
if (details.length <= 2_048) {
return details;
}
return `${sliceUtf16Safe(details, 0, 2_045)}...`;
}
function isQaGatewayChildProcessTreeAlive(
child: ChildProcess,
inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector = inspectLinuxProcessGroup,
) {
if (!child.pid) {
return false;
}
if (process.platform === "win32") {
return !hasQaGatewayChildExited(child);
}
try {
process.kill(-child.pid, 0);
if (process.platform === "linux") {
// Linux can retain zombie-only process groups after SIGKILL while Node's
// child metadata is still unsettled. Runnable /proc members are the owner.
return inspectLinuxProcessGroupFn(child.pid)?.alive ?? true;
}
return true;
} catch (error) {
if (!isProcessAlreadyExitedError(error) && !hasQaGatewayChildExited(child)) {
return true;
}
}
return false;
}
function signalQaGatewayChildProcessTree(child: ChildProcess, signal: NodeJS.Signals) {
if (!child.pid) {
return;
}
try {
if (process.platform === "win32") {
if (runQaWindowsTaskkill({ pid: child.pid, signal })) {
return;
}
child.kill(signal);
return;
}
process.kill(-child.pid, signal);
} catch {
try {
child.kill(signal);
} catch {
// The child already exited.
}
}
}
async function waitForQaGatewayChildExit(
child: ChildProcess,
timeoutMs: number,
inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector,
) {
const deadline = Date.now() + timeoutMs;
while (Date.now() <= deadline) {
if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) {
return true;
}
await sleep(Math.min(25, Math.max(0, deadline - Date.now())));
}
return !isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn);
}
type QaGatewayChildStopOptions = {
gracefulTimeoutMs?: number;
forceTimeoutMs?: number;
inspectLinuxProcessGroup?: QaLinuxProcessGroupInspector;
};
function resolveQaGatewayChildStopTimeouts(opts?: QaGatewayChildStopOptions) {
return {
gracefulTimeoutMs: opts?.gracefulTimeoutMs ?? QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS,
forceTimeoutMs: opts?.forceTimeoutMs ?? QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS,
};
}
function formatQaGatewayProcessTreeDiagnostics(
child: ChildProcess,
inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector,
) {
const childExitRecorded = hasQaGatewayChildExited(child);
if (process.platform !== "linux" || !child.pid) {
return `pid=${child.pid ?? "unknown"} childExitRecorded=${childExitRecorded}`;
}
const inspection = inspectLinuxProcessGroupFn(child.pid);
const processGroupDetails =
inspection?.diagnostics ?? `pgid=${child.pid} members=unknown (/proc unavailable)`;
return boundQaGatewayProcessTreeDiagnostics(
`${processGroupDetails} childExitRecorded=${childExitRecorded}`,
);
}
export async function stopQaGatewayChildProcessTree(
child: ChildProcess,
opts?: QaGatewayChildStopOptions,
) {
const inspectLinuxProcessGroupFn = opts?.inspectLinuxProcessGroup ?? inspectLinuxProcessGroup;
if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) {
return;
}
const timeouts = resolveQaGatewayChildStopTimeouts(opts);
signalQaGatewayChildProcessTree(child, "SIGTERM");
if (
await waitForQaGatewayChildExit(child, timeouts.gracefulTimeoutMs, inspectLinuxProcessGroupFn)
) {
return;
}
signalQaGatewayChildProcessTree(child, "SIGKILL");
const stopped = await waitForQaGatewayChildExit(
child,
timeouts.forceTimeoutMs,
inspectLinuxProcessGroupFn,
);
if (!stopped) {
throw new Error(
`qa gateway process tree remained alive after forced shutdown: ${formatQaGatewayProcessTreeDiagnostics(
child,
inspectLinuxProcessGroupFn,
)}`,
);
}
}
@@ -0,0 +1,269 @@
// Qa Lab plugin module owns gateway readiness and retry behavior.
import { setTimeout as sleep } from "node:timers/promises";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
import { QaSuiteInfraError } from "./errors.js";
import {
hasQaGatewayChildExited,
type QaChildFailure,
throwQaGatewayChildFailure,
} from "./gateway-child-process.js";
import { formatQaGatewayLogsForError } from "./gateway-log-redaction.js";
export const QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS = 5;
const QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS = 90_000;
const QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX =
"OpenClaw plugin migration inputs changed during startup convergence;";
type QaGatewayStartupRetryKind = "bind-collision" | "migration-convergence-restart";
type QaGatewayHealthChild = {
exitCode: number | null;
signalCode: NodeJS.Signals | null;
};
function classifyQaGatewayStartupRetry(details: string): QaGatewayStartupRetryKind | null {
if (details.includes(QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX)) {
return "migration-convergence-restart";
}
if (
details.includes("another gateway instance is already listening on ws://") ||
details.includes("failed to bind gateway socket on ws://") ||
details.includes("EADDRINUSE") ||
details.includes("address already in use")
) {
return "bind-collision";
}
return null;
}
export function resolveQaGatewayStartupRetry(params: {
attempt: number;
details: string;
migrationConvergenceRestartUsed: boolean;
}) {
if (params.attempt >= QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS) {
return null;
}
const kind = classifyQaGatewayStartupRetry(params.details);
if (
!kind ||
(kind === "migration-convergence-restart" && params.migrationConvergenceRestartUsed)
) {
return null;
}
return {
kind,
reuseLaunchState: kind === "migration-convergence-restart",
migrationConvergenceRestartUsed:
params.migrationConvergenceRestartUsed || kind === "migration-convergence-restart",
};
}
function isRetryableGatewayCallError(details: string): boolean {
return (
details.includes("handshake timeout") ||
details.includes("gateway closed (1000") ||
details.includes("gateway closed (1012)") ||
details.includes("gateway closed (1006") ||
details.includes("abnormal closure") ||
details.includes("service restart")
);
}
export async function callQaGatewayWithRetry<T>(params: {
deadlineMs?: number;
logs: () => string;
request: (options: { deadlineMs?: number; timeoutMs: number }) => Promise<T>;
throwChildFailure: () => void;
timeoutMs: number;
waitForReady: (timeoutMs: number) => Promise<void>;
}) {
const remainingMs = () =>
params.deadlineMs === undefined ? undefined : params.deadlineMs - Date.now();
const deadlineError = () =>
new Error(`gateway call deadline exceeded${formatQaGatewayLogsForError(params.logs())}`);
let lastDetails = "";
for (let attempt = 1; attempt <= 3; attempt += 1) {
params.throwChildFailure();
const requestRemainingMs = remainingMs();
if (requestRemainingMs !== undefined && requestRemainingMs <= 0) {
throw deadlineError();
}
try {
return await params.request({
...(params.deadlineMs === undefined ? {} : { deadlineMs: params.deadlineMs }),
timeoutMs:
requestRemainingMs === undefined
? params.timeoutMs
: Math.min(params.timeoutMs, requestRemainingMs),
});
} catch (error) {
params.throwChildFailure();
const details = formatErrorMessage(error);
lastDetails = details;
if (attempt >= 3 || !isRetryableGatewayCallError(details)) {
throw new Error(`${details}${formatQaGatewayLogsForError(params.logs())}`, {
cause: error,
});
}
const readinessRemainingMs = remainingMs();
if (readinessRemainingMs !== undefined && readinessRemainingMs <= 0) {
throw deadlineError();
}
await params.waitForReady(
readinessRemainingMs === undefined
? Math.max(10_000, params.timeoutMs)
: Math.min(Math.max(10_000, params.timeoutMs), readinessRemainingMs),
);
}
}
throw new Error(`${lastDetails}${formatQaGatewayLogsForError(params.logs())}`);
}
async function fetchLocalGatewayHealth(params: {
baseUrl: string;
healthPath: "/readyz" | "/healthz";
timeoutMs?: number;
}): Promise<boolean> {
const { response, release } = await fetchWithSsrFGuard({
url: `${params.baseUrl}${params.healthPath}`,
init: {
method: "HEAD",
headers: {
connection: "close",
},
signal: AbortSignal.timeout(params.timeoutMs ?? 2_000),
},
policy: { allowPrivateNetwork: true },
auditContext: "qa-lab-gateway-child-health",
});
try {
return response.ok;
} finally {
await release();
}
}
async function fetchLocalGatewayListening(baseUrl: string): Promise<boolean> {
const { release } = await fetchWithSsrFGuard({
url: `${baseUrl}/healthz`,
init: {
method: "HEAD",
headers: {
connection: "close",
},
signal: AbortSignal.timeout(2_000),
},
policy: { allowPrivateNetwork: true },
auditContext: "qa-lab-gateway-child-listening",
});
await release();
return true;
}
export async function waitForQaGatewayRestartBoundary(params: {
readLogsSince: (mark: number) => string;
mark: number;
pollMs?: number;
timeoutMs?: number;
}) {
const timeoutMs = params.timeoutMs ?? QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS;
const pollMs = resolveTimerTimeoutMs(params.pollMs ?? 100, 100, 0);
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
if (params.readLogsSince(params.mark).includes("restart mode:")) {
return;
}
const remainingMs = timeoutMs - (Date.now() - startedAt);
if (remainingMs <= 0) {
break;
}
await sleep(Math.min(pollMs, remainingMs));
}
throw new Error(`qa gateway child did not reach restart boundary within ${timeoutMs}ms`);
}
export async function waitForGatewayReady(params: {
baseUrl: string;
logs: () => string;
child: QaGatewayHealthChild;
getChildFailure?: () => QaChildFailure | null;
timeoutMs?: number;
}) {
const deadline = Date.now() + (params.timeoutMs ?? 60_000);
let remainingMs: number;
while ((remainingMs = deadline - Date.now()) > 0) {
throwQaGatewayChildFailure(params.getChildFailure, params.logs);
if (hasQaGatewayChildExited(params.child)) {
throw new QaSuiteInfraError(
"gateway_startup_unhealthy",
`gateway exited before becoming healthy (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
);
}
// Listener liveness can turn green before the Gateway can admit startup or restart work.
try {
if (
await fetchLocalGatewayHealth({
baseUrl: params.baseUrl,
healthPath: "/readyz",
timeoutMs: Math.min(2_000, remainingMs),
})
) {
return;
}
} catch {
// retry until timeout
}
await sleep(Math.min(250, Math.max(0, deadline - Date.now())));
}
throw new QaSuiteInfraError(
"gateway_startup_unhealthy",
`gateway failed to become healthy:\n${params.logs()}`,
);
}
export async function waitForGatewayListening(params: {
baseUrl: string;
logs: () => string;
child: QaGatewayHealthChild;
getChildFailure?: () => QaChildFailure | null;
timeoutMs?: number;
}) {
const startedAt = Date.now();
while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
throwQaGatewayChildFailure(params.getChildFailure, params.logs);
if (params.child.exitCode !== null || params.child.signalCode !== null) {
throw new QaSuiteInfraError(
"gateway_startup_unhealthy",
`gateway exited before listening (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
);
}
try {
if (await fetchLocalGatewayListening(params.baseUrl)) {
return;
}
} catch {
// retry until the HTTP listener accepts requests
}
await sleep(100);
}
throw new QaSuiteInfraError(
"gateway_startup_unhealthy",
`gateway failed to listen before timeout:\n${params.logs()}`,
);
}
export function isRetryableRpcStartupError(error: unknown) {
const details = formatErrorMessage(error);
return (
details.includes("gateway timeout after") ||
details.includes("handshake timeout") ||
details.includes("gateway token mismatch") ||
details.includes("token mismatch") ||
details.includes("gateway closed (1000") ||
details.includes("gateway closed (1006") ||
details.includes("gateway closed (1012)")
);
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,9 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
inspectLinuxProcessGroupStats,
isQaPosixProcessGroupAlive,
signalQaPosixProcessGroup,
} from "./posix-process-group.js";
import { isQaPosixProcessGroupAlive, signalQaPosixProcessGroup } from "./posix-process-group.js";
import { inspectLinuxProcessGroupStats } from "./posix-process-stat.js";
afterEach(() => {
vi.restoreAllMocks();
@@ -37,6 +34,19 @@ describe("POSIX process group inspection", () => {
});
});
it("bounds process group diagnostics", () => {
const stats = Array.from(
{ length: 300 },
(_, index) => `${index + 1} (${`worker-${index}`.padEnd(32, "x")}) S 1 123 123 0 -1 0`,
);
const inspection = inspectLinuxProcessGroupStats(123, stats);
expect(inspection.alive).toBe(true);
expect(inspection.diagnostics.length).toBeLessThanOrEqual(2_048);
expect(inspection.diagnostics).toMatch(/\.\.\.$/u);
});
it("fails closed when the Linux member snapshot is unavailable", () => {
const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
const processKill = vi.spyOn(process, "kill").mockImplementation(() => true);
+1 -61
View File
@@ -1,66 +1,6 @@
import { readFileSync, readdirSync } from "node:fs";
import path from "node:path";
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
function parseLinuxProcessStat(raw: string) {
const commandStart = raw.indexOf("(");
const commandEnd = raw.lastIndexOf(")");
if (commandStart <= 0 || commandEnd <= commandStart) {
return null;
}
const pid = Number.parseInt(raw.slice(0, commandStart).trim(), 10);
const fields = raw
.slice(commandEnd + 1)
.trim()
.split(/\s+/u);
const state = fields[0];
const processGroupId = Number.parseInt(fields[2] ?? "", 10);
if (
!Number.isSafeInteger(pid) ||
pid <= 0 ||
!state ||
!Number.isSafeInteger(processGroupId) ||
processGroupId <= 0
) {
return null;
}
return {
command: raw.slice(commandStart + 1, commandEnd),
pid,
processGroupId,
state,
};
}
function boundProcessGroupDiagnostics(details: string) {
if (details.length <= 2_048) {
return details;
}
return `${sliceUtf16Safe(details, 0, 2_045)}...`;
}
export function inspectLinuxProcessGroupStats(processGroupId: number, stats: readonly string[]) {
const members = stats
.map((raw) => parseLinuxProcessStat(raw))
.filter(
(entry): entry is NonNullable<ReturnType<typeof parseLinuxProcessStat>> =>
entry?.processGroupId === processGroupId,
)
.toSorted((left, right) => left.pid - right.pid);
const diagnostics = members
.map(
(member) =>
`pid=${member.pid} state=${member.state} command=${JSON.stringify(member.command)}`,
)
.join(", ");
return {
alive:
members.length === 0
? null
: members.some((entry) => entry.state !== "Z" && entry.state !== "X"),
diagnostics: boundProcessGroupDiagnostics(`pgid=${processGroupId} members=[${diagnostics}]`),
};
}
import { inspectLinuxProcessGroupStats } from "./posix-process-stat.js";
type QaLinuxProcessGroupInspection = ReturnType<typeof inspectLinuxProcessGroupStats>;
export type QaLinuxProcessGroupInspector = (
@@ -0,0 +1,62 @@
// Qa Lab parses Linux process stat snapshots for process-group ownership.
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
function parseLinuxProcessStat(raw: string) {
const commandStart = raw.indexOf("(");
const commandEnd = raw.lastIndexOf(")");
if (commandStart <= 0 || commandEnd <= commandStart) {
return null;
}
const pid = Number.parseInt(raw.slice(0, commandStart).trim(), 10);
const fields = raw
.slice(commandEnd + 1)
.trim()
.split(/\s+/u);
const state = fields[0];
const processGroupId = Number.parseInt(fields[2] ?? "", 10);
if (
!Number.isSafeInteger(pid) ||
pid <= 0 ||
!state ||
!Number.isSafeInteger(processGroupId) ||
processGroupId <= 0
) {
return null;
}
return {
command: raw.slice(commandStart + 1, commandEnd),
pid,
processGroupId,
state,
};
}
function boundProcessGroupDiagnostics(details: string) {
if (details.length <= 2_048) {
return details;
}
return `${sliceUtf16Safe(details, 0, 2_045)}...`;
}
export function inspectLinuxProcessGroupStats(processGroupId: number, stats: readonly string[]) {
const members = stats
.map((raw) => parseLinuxProcessStat(raw))
.filter(
(entry): entry is NonNullable<ReturnType<typeof parseLinuxProcessStat>> =>
entry?.processGroupId === processGroupId,
)
.toSorted((left, right) => left.pid - right.pid);
const diagnostics = members
.map(
(member) =>
`pid=${member.pid} state=${member.state} command=${JSON.stringify(member.command)}`,
)
.join(", ");
return {
alive:
members.length === 0
? null
: members.some((entry) => entry.state !== "Z" && entry.state !== "X"),
diagnostics: boundProcessGroupDiagnostics(`pgid=${processGroupId} members=[${diagnostics}]`),
};
}
@@ -0,0 +1,70 @@
// Qa Lab plugin module owns host live-provider config projection.
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import {
isRecord,
normalizeOptionalString,
normalizeStringEntries,
uniqueStrings,
} from "openclaw/plugin-sdk/string-coerce-runtime";
import { QA_LIVE_PROVIDER_CONFIG_PATH_ENV, resolveQaLiveProviderConfigPath } from "./env.js";
function isQaModelProviderConfig(value: unknown): value is ModelProviderConfig {
return isRecord(value) && typeof value.baseUrl === "string" && Array.isArray(value.models);
}
function normalizeQaLiveProviderConfig(value: unknown): ModelProviderConfig | null {
if (!isQaModelProviderConfig(value) && (!isRecord(value) || !Object.hasOwn(value, "apiKey"))) {
return null;
}
const { baseUrl: rawBaseUrl, ...providerConfig } = value;
const baseUrl = normalizeOptionalString(rawBaseUrl);
return {
...providerConfig,
...(baseUrl ? { baseUrl } : {}),
models: Array.isArray(value.models) ? value.models : [],
} as ModelProviderConfig;
}
export async function readQaLiveProviderConfigOverrides(params: {
providerIds: readonly string[];
env?: NodeJS.ProcessEnv;
}) {
const providerIds = uniqueStrings(normalizeStringEntries(params.providerIds));
if (providerIds.length === 0) {
return {};
}
const configPath = resolveQaLiveProviderConfigPath(params.env);
if (!existsSync(configPath.path)) {
return {};
}
try {
const raw = await fs.readFile(configPath.path, "utf8");
const parsed = JSON.parse(raw) as unknown;
const providers = isRecord(parsed)
? isRecord(parsed.models)
? isRecord(parsed.models.providers)
? parsed.models.providers
: {}
: {}
: {};
const selected: Record<string, ModelProviderConfig> = {};
for (const providerId of providerIds) {
const providerConfig = normalizeQaLiveProviderConfig(providers[providerId]);
if (providerConfig) {
selected[providerId] = providerConfig;
}
}
return selected;
} catch (error) {
if (configPath.explicit) {
throw new Error(
`failed to read ${QA_LIVE_PROVIDER_CONFIG_PATH_ENV} provider config: ${formatErrorMessage(error)}`,
{ cause: error },
);
}
return {};
}
}
@@ -159,7 +159,7 @@ export function listMockCodexModelInfos(selectedModelRefs: readonly string[] = [
upgrade: null,
base_instructions: "You are Codex, a coding agent based on GPT-5.",
include_skills_usage_instructions: false,
supports_reasoning_summaries: true,
supports_reasoning_summary_parameter: true,
default_reasoning_summary: "none",
support_verbosity: true,
default_verbosity: "low",
@@ -1,8 +1,9 @@
// Qa Lab tests cover Windows system tool path resolution.
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import {
resolveQaWindowsPowerShellExePath,
resolveQaWindowsSystem32ExePath,
runQaWindowsTaskkill,
} from "./windows-system-tools.js";
describe("qa-lab windows system tools", () => {
@@ -15,6 +16,34 @@ describe("qa-lab windows system tools", () => {
);
});
it("force-kills a process tree when graceful taskkill fails", () => {
const runCommand = vi
.fn()
.mockReturnValueOnce({ status: 1 })
.mockReturnValueOnce({ status: 0 });
expect(
runQaWindowsTaskkill({
pid: 12345,
signal: "SIGTERM",
env: { SystemRoot: "D:\\Windows" },
runCommand,
}),
).toBe(true);
expect(runCommand).toHaveBeenNthCalledWith(
1,
"D:\\Windows\\System32\\taskkill.exe",
["/PID", "12345", "/T"],
{ stdio: "ignore", windowsHide: true, timeout: 5_000 },
);
expect(runCommand).toHaveBeenNthCalledWith(
2,
"D:\\Windows\\System32\\taskkill.exe",
["/PID", "12345", "/T", "/F"],
{ stdio: "ignore", windowsHide: true, timeout: 5_000 },
);
});
it("falls back to the default Windows root when env roots are unsafe", () => {
expect(resolveQaWindowsSystem32ExePath("taskkill.exe", { SystemRoot: "C:\\tmp;C:\\bad" })).toBe(
"C:\\Windows\\System32\\taskkill.exe",
@@ -1,4 +1,5 @@
// Qa Lab resolves Windows system tools without trusting PATH.
import { spawnSync } from "node:child_process";
import path from "node:path";
const DEFAULT_WINDOWS_SYSTEM_ROOT = "C:\\Windows";
@@ -59,6 +60,37 @@ export function resolveQaWindowsSystem32ExePath(
return path.win32.join(resolveQaWindowsSystemRoot(env), "System32", executableName);
}
export function runQaWindowsTaskkill(params: {
pid: number;
signal: NodeJS.Signals;
env?: Record<string, string | undefined>;
runCommand?: typeof spawnSync;
}) {
const runCommand = params.runCommand ?? spawnSync;
const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe", params.env);
const args = ["/PID", String(params.pid), "/T"];
if (params.signal === "SIGKILL") {
args.push("/F");
}
const result = runCommand(taskkillPath, args, {
stdio: "ignore",
windowsHide: true,
timeout: 5_000,
});
if (!result.error && result.status === 0) {
return true;
}
if (params.signal !== "SIGKILL") {
const forceResult = runCommand(taskkillPath, [...args, "/F"], {
stdio: "ignore",
windowsHide: true,
timeout: 5_000,
});
return !forceResult.error && forceResult.status === 0;
}
return false;
}
export function resolveQaWindowsPowerShellExePath(
env: Record<string, string | undefined> = process.env,
): string {
@@ -20,7 +20,8 @@ scenario:
- docs/channels/qa-channel.md
codeRefs:
- src/agents/system-prompt.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
- extensions/qa-lab/src/providers/mock-openai/server.ts
execution:
kind: flow
@@ -18,7 +18,8 @@ scenario:
- The failed tool call is never replayed or described as successful.
- The model receives one tools-disabled finalization and qa-channel delivers exactly one visible reply.
codeRefs:
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
- src/agents/embedded-agent-runner/run/code-mode-repair.ts
- extensions/qa-lab/src/providers/mock-openai/server.ts
execution:
@@ -15,7 +15,8 @@ scenario:
- The runtime continues from settled tool results without replaying the write.
- Telegram receives the exact recovery marker.
codeRefs:
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
- src/agents/embedded-agent-runner/run/terminal-resolution.ts
- extensions/qa-lab/src/providers/mock-openai/server.ts
execution:
@@ -27,7 +27,8 @@ scenario:
- docs/help/testing.md
codeRefs:
- extensions/qa-lab/src/providers/mock-openai/server.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Verify Anthropic stream errors after signed thinking recover after a replay-safe read.
@@ -18,7 +18,8 @@ scenario:
codeRefs:
- extensions/qa-lab/src/suite.ts
- extensions/qa-lab/src/providers/mock-openai/server.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Verify a short approval like "ok do it" triggers immediate tool use instead of fake-progress narration.
@@ -21,7 +21,8 @@ scenario:
- docs/help/testing.md
codeRefs:
- extensions/qa-lab/src/providers/mock-openai/server.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Verify empty OpenAI turns recover after a replay-safe read.
@@ -19,7 +19,8 @@ scenario:
- docs/help/testing.md
codeRefs:
- extensions/qa-lab/src/providers/mock-openai/server.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Verify empty-response retry exhaustion still surfaces a visible failure.
@@ -20,7 +20,8 @@ scenario:
- docs/help/testing.md
codeRefs:
- extensions/qa-lab/src/providers/mock-openai/server.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Verify reasoning-only turns after a write do not auto-retry.
@@ -18,7 +18,8 @@ scenario:
- docs/help/testing.md
codeRefs:
- extensions/qa-lab/src/providers/mock-openai/server.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Verify reasoning-only OpenAI turns recover after a replay-safe read.
@@ -19,7 +19,8 @@ scenario:
- docs/concepts/streaming.md
- docs/channels/qa-channel.md
codeRefs:
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
- extensions/qa-lab/src/bus-state.ts
- extensions/qa-lab/src/suite-runtime-transport.ts
execution:
@@ -20,7 +20,8 @@ scenario:
codeRefs:
- extensions/qa-channel/src/inbound.ts
- extensions/qa-lab/src/providers/mock-openai/server.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Complete a proof-backed QA-channel task and verify artifact-before-reply ordering plus honest terminal status.
@@ -19,7 +19,8 @@ scenario:
- The failed tool never replays and qa-channel receives exactly one final reply.
codeRefs:
- src/cron/isolated-agent/run-executor.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
- extensions/qa-lab/src/providers/mock-openai/server.ts
execution:
kind: flow
@@ -20,7 +20,8 @@ scenario:
codeRefs:
- extensions/qa-lab/src/suite-runtime-agent-process.ts
- extensions/qa-lab/src/suite-runtime-transport.ts
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
execution:
kind: flow
summary: Verify fake secret fixtures are not echoed into channel-visible output.
@@ -27,7 +27,7 @@ scenario:
- docs/gateway/protocol.md
codeRefs:
- ui/src/e2e/config-safe-write.e2e.test.ts
- ui/src/lib/config/index.ts
- ui/src/lib/config/config-write-coordinator.ts
- ui/src/components/settings-save-indicator.ts
execution:
kind: playwright