mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor(qa): remove gateway child test facade (#122693)
* refactor(qa): remove gateway child test facade * test(qa): refresh scenario source references * fix(qa): preserve packaged auth redaction boundary
This commit is contained in:
committed by
GitHub
parent
28a5889e9b
commit
71e8cc033b
@@ -99,14 +99,10 @@ export {
|
||||
} from "./src/self-check.js";
|
||||
export { runQaE2eSelfCheck, runQaLabSelfCheck } from "./src/self-check-runner.js";
|
||||
export {
|
||||
testing,
|
||||
buildQaRuntimeEnv,
|
||||
type QaCliBackendAuthMode,
|
||||
type QaGatewayChildListeningContext,
|
||||
type QaGatewayChildCommand,
|
||||
type QaGatewayChildStateMutationContext,
|
||||
resolveQaControlUiRoot,
|
||||
resolveQaGatewayChildProviderMode,
|
||||
startQaGatewayChild,
|
||||
} from "./src/gateway-child.js";
|
||||
export {
|
||||
|
||||
@@ -36,7 +36,7 @@ function isQaOpenAiResponsesProviderConfig(config: ModelProviderConfig) {
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) {
|
||||
function resolveQaBundledPluginSourceDir(params: { repoRoot: string; pluginId: string }) {
|
||||
assertSafeQaBundledPluginId(params.pluginId);
|
||||
const candidates = [
|
||||
path.join(params.repoRoot, "dist", "extensions", params.pluginId),
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
// Qa Lab plugin module implements cli paths behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { assertNoSymlinkParents, pathScope } from "openclaw/plugin-sdk/security-runtime";
|
||||
|
||||
@@ -29,25 +28,6 @@ export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: strin
|
||||
return resolved.path;
|
||||
}
|
||||
|
||||
async function resolveNearestExistingPath(targetPath: string) {
|
||||
let current = path.resolve(targetPath);
|
||||
while (true) {
|
||||
try {
|
||||
await fs.lstat(current);
|
||||
return current;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const parent = path.dirname(current);
|
||||
if (parent === current) {
|
||||
throw new Error(`failed to resolve existing path for ${targetPath}`);
|
||||
}
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function assertRepoRelativePath(repoRoot: string, targetPath: string, label: string) {
|
||||
const relative = path.relative(repoRoot, targetPath);
|
||||
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
||||
@@ -72,18 +52,6 @@ async function assertNoSymlinkSegments(repoRoot: string, targetPath: string, lab
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertRepoBoundPath(repoRoot: string, targetPath: string, label: string) {
|
||||
const repoRootResolved = path.resolve(repoRoot);
|
||||
const targetResolved = path.resolve(targetPath);
|
||||
assertRepoRelativePath(repoRootResolved, targetResolved, label);
|
||||
await assertNoSymlinkSegments(repoRootResolved, targetResolved, label);
|
||||
const repoRootReal = await fs.realpath(repoRootResolved);
|
||||
const nearestExistingPath = await resolveNearestExistingPath(targetResolved);
|
||||
const nearestExistingReal = await fs.realpath(nearestExistingPath);
|
||||
assertRepoRelativePath(repoRootReal, nearestExistingReal, label);
|
||||
return targetResolved;
|
||||
}
|
||||
|
||||
export async function ensureRepoBoundDirectory(
|
||||
repoRoot: string,
|
||||
targetDir: string,
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
// Qa Lab plugin module owns sanitized gateway debug artifacts and temp cleanup.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { ensureRepoBoundDirectory } from "./cli-paths.js";
|
||||
import { redactQaGatewayDebugText } from "./gateway-log-redaction.js";
|
||||
|
||||
async function writeSanitizedQaGatewayDebugLog(params: { sourcePath: string; targetPath: string }) {
|
||||
const contents = await fs.readFile(params.sourcePath, "utf8").catch((error: unknown) => {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
await fs.writeFile(params.targetPath, redactQaGatewayDebugText(contents), "utf8");
|
||||
}
|
||||
|
||||
async function clearQaGatewayArtifactDir(dir: string) {
|
||||
for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
|
||||
await fs.rm(path.join(dir, entry.name), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function cleanupQaGatewayTempRoots(params: {
|
||||
tempRoot: string;
|
||||
stagedBundledPluginsRoot?: string | null;
|
||||
}) {
|
||||
await fs.rm(params.tempRoot, { recursive: true, force: true }).catch(() => {});
|
||||
if (params.stagedBundledPluginsRoot) {
|
||||
await fs.rm(params.stagedBundledPluginsRoot, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
export async function preserveQaGatewayDebugArtifacts(params: {
|
||||
preserveToDir: string;
|
||||
stdoutLogPath: string;
|
||||
stderrLogPath: string;
|
||||
tempRoot: string;
|
||||
repoRoot?: string;
|
||||
}) {
|
||||
const preserveToDir = params.repoRoot
|
||||
? await ensureRepoBoundDirectory(
|
||||
params.repoRoot,
|
||||
params.preserveToDir,
|
||||
"QA gateway artifact directory",
|
||||
{
|
||||
mode: 0o700,
|
||||
},
|
||||
)
|
||||
: params.preserveToDir;
|
||||
await fs.mkdir(preserveToDir, { recursive: true, mode: 0o700 });
|
||||
await clearQaGatewayArtifactDir(preserveToDir);
|
||||
await Promise.all([
|
||||
writeSanitizedQaGatewayDebugLog({
|
||||
sourcePath: params.stdoutLogPath,
|
||||
targetPath: path.join(preserveToDir, "gateway.stdout.log"),
|
||||
}),
|
||||
writeSanitizedQaGatewayDebugLog({
|
||||
sourcePath: params.stderrLogPath,
|
||||
targetPath: path.join(preserveToDir, "gateway.stderr.log"),
|
||||
}),
|
||||
]);
|
||||
await fs.writeFile(
|
||||
path.join(preserveToDir, "README.txt"),
|
||||
[
|
||||
"Only sanitized gateway debug artifacts are preserved here.",
|
||||
"The full QA gateway runtime was not copied because it may contain credentials or auth tokens.",
|
||||
"Original runtime temp root omitted because local temp paths can identify the runner.",
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
// Qa Lab plugin module owns gateway child command bootstrap behavior.
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { formatErrorMessage, toErrorObject } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
appendQaChildOutput,
|
||||
appendQaChildOutputTail,
|
||||
createQaChildOutputCapture,
|
||||
createQaChildOutputTail,
|
||||
formatQaChildOutputTail,
|
||||
readQaChildOutput,
|
||||
} from "./child-output.js";
|
||||
import { hasQaGatewayChildExited, monitorQaChildFailure } from "./gateway-child-process.js";
|
||||
import type { QaGatewayProcessBoundaryConfig } from "./gateway-process-boundary.js";
|
||||
|
||||
type QaGatewayChildDirectCommand = {
|
||||
executablePath: string;
|
||||
argsPrefix?: string[];
|
||||
argsSuffix?: string[];
|
||||
cwd?: string;
|
||||
tempParentDir?: string;
|
||||
usePackagedPlugins?: boolean;
|
||||
processBoundary?: undefined;
|
||||
};
|
||||
|
||||
type QaGatewayChildVerifiedCommand = Omit<QaGatewayChildDirectCommand, "processBoundary"> & {
|
||||
processBoundary: QaGatewayProcessBoundaryConfig;
|
||||
};
|
||||
|
||||
export type QaGatewayChildCommand = QaGatewayChildDirectCommand | QaGatewayChildVerifiedCommand;
|
||||
|
||||
export function resolveQaGatewayChildCommand(repoRoot: string): QaGatewayChildCommand {
|
||||
for (const relativePath of ["scripts/run-node.mjs", "dist/index.mjs", "dist/index.js"]) {
|
||||
const entryPath = path.join(repoRoot, relativePath);
|
||||
if (existsSync(entryPath)) {
|
||||
return {
|
||||
executablePath: process.execPath,
|
||||
argsPrefix: [entryPath],
|
||||
cwd: repoRoot,
|
||||
usePackagedPlugins: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"OpenClaw CLI entry not found: expected scripts/run-node.mjs or dist/index.(m)js",
|
||||
);
|
||||
}
|
||||
|
||||
export async function runQaGatewayCliCommand(params: {
|
||||
executablePath: string;
|
||||
argsPrefix: readonly string[];
|
||||
args: readonly string[];
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
stdin?: string;
|
||||
}): Promise<string> {
|
||||
const hasStdin = params.stdin !== undefined;
|
||||
const child = spawn(params.executablePath, [...params.argsPrefix, ...params.args], {
|
||||
cwd: params.cwd,
|
||||
env: { ...params.env, OPENCLAW_CLI: "1" },
|
||||
stdio: [hasStdin ? "pipe" : "ignore", "pipe", "pipe"],
|
||||
});
|
||||
const result = readQaGatewayCliCommand(child);
|
||||
if (hasStdin) {
|
||||
child.stdin?.once("error", () => {});
|
||||
child.stdin?.end(params.stdin);
|
||||
}
|
||||
return await result;
|
||||
}
|
||||
|
||||
async function readQaGatewayCliCommand(child: ChildProcess): Promise<string> {
|
||||
const stdout = createQaChildOutputCapture();
|
||||
const stderr = createQaChildOutputTail();
|
||||
child.stdout?.on("data", (chunk) => appendQaChildOutput(stdout, chunk));
|
||||
child.stderr?.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk));
|
||||
const exitCode = await new Promise<number>((resolve, reject) => {
|
||||
monitorQaChildFailure(child, (failure) => {
|
||||
if (failure.source === "process") {
|
||||
reject(toErrorObject(failure.error, "OpenClaw CLI process failed"));
|
||||
return;
|
||||
}
|
||||
if (!hasQaGatewayChildExited(child) && !child.killed) {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// The child exited between the state check and signal.
|
||||
}
|
||||
}
|
||||
reject(
|
||||
new Error(
|
||||
`qa gateway cli ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`,
|
||||
{ cause: failure.error },
|
||||
),
|
||||
);
|
||||
});
|
||||
child.once("close", (code) => resolve(code ?? 1));
|
||||
});
|
||||
const stdoutText = readQaChildOutput(stdout);
|
||||
if (exitCode !== 0) {
|
||||
const stderrText = formatQaChildOutputTail(stderr, "stderr");
|
||||
throw new Error(`OpenClaw CLI exited ${exitCode}: ${stderrText || stdoutText}`);
|
||||
}
|
||||
return stdoutText;
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// Qa Lab plugin module owns gateway child runtime environment behavior.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { buildQaCodexAppServerArgs } from "./codex-app-server-args.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import {
|
||||
normalizeQaProviderModeEnv,
|
||||
resolveQaLiveCliAuthEnv,
|
||||
type QaCliBackendAuthMode,
|
||||
} from "./providers/env.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
import {
|
||||
QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV,
|
||||
QA_LIVE_SETUP_TOKEN_VALUE_ENV,
|
||||
} from "./providers/live-frontier/auth.js";
|
||||
import { listMockCodexModelInfos } from "./providers/shared/mock-model-config.js";
|
||||
import type { RuntimeId } from "./runtime-parity.js";
|
||||
|
||||
const QA_MOCK_OPENAI_API_KEY = ["qa", "mock", "openai", "key"].join("-");
|
||||
const QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS = Object.freeze([
|
||||
"OPENCLAW_QA_CONVEX_SECRET_CI",
|
||||
"OPENCLAW_QA_CONVEX_SECRET_MAINTAINER",
|
||||
"OPENCLAW_QA_SUT_FORBIDDEN_SENTINEL",
|
||||
"OPENCLAW_QA_TELEGRAM_GROUP_ID",
|
||||
"OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN",
|
||||
"OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN",
|
||||
]);
|
||||
|
||||
function scrubQaGatewayChildSecretEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
||||
for (const envKey of QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS) {
|
||||
delete env[envKey];
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
function scrubQaGatewayChildTestRunnerEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
||||
// The Gateway is a product child, not a nested Vitest worker. Leaking runner
|
||||
// markers makes the dist launcher select test-only startup behavior.
|
||||
delete env.VITEST;
|
||||
delete env.VITEST_POOL_ID;
|
||||
delete env.VITEST_WORKER_ID;
|
||||
if (env.NODE_ENV === "test") {
|
||||
delete env.NODE_ENV;
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
export function buildQaRuntimeEnv(params: {
|
||||
configPath: string;
|
||||
gatewayToken: string;
|
||||
homeDir: string;
|
||||
forwardHostHome?: boolean;
|
||||
stateDir: string;
|
||||
tempRoot: string;
|
||||
xdgConfigHome: string;
|
||||
xdgDataHome: string;
|
||||
xdgCacheHome: string;
|
||||
bundledPluginsDir?: string;
|
||||
stagedBundledPluginsRoot?: string | null;
|
||||
compatibilityHostVersion?: string;
|
||||
providerMode?: QaProviderMode;
|
||||
baseEnv?: NodeJS.ProcessEnv;
|
||||
runtimeEnvPatch?: NodeJS.ProcessEnv;
|
||||
forwardHostHomeForClaudeCli?: boolean;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
}) {
|
||||
const baseEnv = params.baseEnv ?? process.env;
|
||||
const provider = params.providerMode ? getQaProvider(params.providerMode) : null;
|
||||
const forwardedHostHome = params.forwardHostHome
|
||||
? baseEnv.HOME?.trim() || os.homedir()
|
||||
: undefined;
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...baseEnv,
|
||||
HOME: forwardedHostHome ?? params.homeDir,
|
||||
...(provider?.appliesLiveEnvAliases
|
||||
? resolveQaLiveCliAuthEnv(baseEnv, {
|
||||
forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
})
|
||||
: {}),
|
||||
OPENCLAW_HOME: params.homeDir,
|
||||
OPENCLAW_CONFIG_PATH: params.configPath,
|
||||
OPENCLAW_STATE_DIR: params.stateDir,
|
||||
OPENCLAW_OAUTH_DIR: path.join(params.stateDir, "credentials"),
|
||||
OPENCLAW_GATEWAY_TOKEN: params.gatewayToken,
|
||||
OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1",
|
||||
OPENCLAW_SKIP_GMAIL_WATCHER: "1",
|
||||
OPENCLAW_SKIP_CANVAS_HOST: "1",
|
||||
OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1",
|
||||
OPENCLAW_NO_RESPAWN: "1",
|
||||
OPENCLAW_TEST_FAST: "1",
|
||||
OPENCLAW_EMBEDDED_ABORT_SETTLE_TIMEOUT_MS: "2000",
|
||||
OPENCLAW_QA_PARENT_PID: String(process.pid),
|
||||
OPENCLAW_QA_TEMP_ROOT: params.tempRoot,
|
||||
...(params.stagedBundledPluginsRoot
|
||||
? { OPENCLAW_QA_STAGED_RUNTIME_ROOT: params.stagedBundledPluginsRoot }
|
||||
: {}),
|
||||
OPENCLAW_QA_ALLOW_LOCAL_IMAGE_PROVIDER: "1",
|
||||
// QA uses the fast runtime envelope for speed, but it still exercises
|
||||
// normal config-driven heartbeats and runtime config writes.
|
||||
OPENCLAW_ALLOW_SLOW_REPLY_TESTS: "1",
|
||||
XDG_CONFIG_HOME: params.xdgConfigHome,
|
||||
XDG_DATA_HOME: params.xdgDataHome,
|
||||
XDG_CACHE_HOME: params.xdgCacheHome,
|
||||
...(params.bundledPluginsDir ? { OPENCLAW_BUNDLED_PLUGINS_DIR: params.bundledPluginsDir } : {}),
|
||||
...(params.compatibilityHostVersion
|
||||
? { OPENCLAW_COMPATIBILITY_HOST_VERSION: params.compatibilityHostVersion }
|
||||
: {}),
|
||||
};
|
||||
const normalizedEnv = normalizeQaProviderModeEnv(env, params.providerMode);
|
||||
// Test-runner skip flags are parent controls; each QA child declares its own runtime needs.
|
||||
delete normalizedEnv.OPENCLAW_SKIP_CHANNELS;
|
||||
delete normalizedEnv.OPENCLAW_SKIP_PROVIDERS;
|
||||
Object.assign(normalizedEnv, params.runtimeEnvPatch);
|
||||
normalizedEnv.OPENCLAW_BUILD_PRIVATE_QA = "1";
|
||||
delete normalizedEnv[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV];
|
||||
delete normalizedEnv[QA_LIVE_SETUP_TOKEN_VALUE_ENV];
|
||||
return scrubQaGatewayChildSecretEnv(scrubQaGatewayChildTestRunnerEnv(normalizedEnv));
|
||||
}
|
||||
|
||||
export async function stageQaCodexMockModelCatalog(params: {
|
||||
tempRoot: string;
|
||||
forcedRuntime?: RuntimeId;
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
}): Promise<string | undefined> {
|
||||
if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") {
|
||||
return undefined;
|
||||
}
|
||||
const modelCatalogPath = path.join(params.tempRoot, "codex-model-catalog.json");
|
||||
const selectedModelRefs = [params.primaryModel, params.alternateModel].filter(
|
||||
(model): model is string => typeof model === "string" && model.length > 0,
|
||||
);
|
||||
await fs.writeFile(
|
||||
modelCatalogPath,
|
||||
`${JSON.stringify({ models: listMockCodexModelInfos(selectedModelRefs) }, null, 2)}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
return modelCatalogPath;
|
||||
}
|
||||
|
||||
export function buildQaForcedRuntimeEnvPatch(params: {
|
||||
forcedRuntime?: RuntimeId;
|
||||
providerMode: QaProviderMode;
|
||||
providerBaseUrl?: string;
|
||||
codexModelCatalogPath?: string;
|
||||
nativeAppServerArgs?: string;
|
||||
}): NodeJS.ProcessEnv | undefined {
|
||||
if (!params.forcedRuntime) {
|
||||
return undefined;
|
||||
}
|
||||
const patch: NodeJS.ProcessEnv = {
|
||||
OPENCLAW_BUILD_PRIVATE_QA: "1",
|
||||
OPENCLAW_QA_FORCE_RUNTIME: params.forcedRuntime,
|
||||
};
|
||||
if (params.forcedRuntime !== "codex") {
|
||||
return patch;
|
||||
}
|
||||
if (params.providerMode !== "mock-openai") {
|
||||
patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({
|
||||
existingArgs: params.nativeAppServerArgs,
|
||||
});
|
||||
return patch;
|
||||
}
|
||||
const providerBaseUrl = params.providerBaseUrl?.trim().replace(/\/+$/u, "");
|
||||
if (!providerBaseUrl) {
|
||||
throw new Error("forced Codex mock QA requires the managed mock provider URL");
|
||||
}
|
||||
if (!params.codexModelCatalogPath) {
|
||||
throw new Error("forced Codex mock QA requires the staged native model catalog");
|
||||
}
|
||||
patch.OPENCLAW_CODEX_APP_SERVER_ARGS = buildQaCodexAppServerArgs({
|
||||
providerBaseUrl,
|
||||
modelCatalogPath: params.codexModelCatalogPath,
|
||||
});
|
||||
patch.OPENAI_API_KEY = QA_MOCK_OPENAI_API_KEY;
|
||||
patch.CODEX_API_KEY = QA_MOCK_OPENAI_API_KEY;
|
||||
return patch;
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
// Qa Lab plugin module owns gateway child process lifecycle behavior.
|
||||
import type { ChildProcess } from "node:child_process";
|
||||
import type { WriteStream } from "node:fs";
|
||||
import { finished } from "node:stream/promises";
|
||||
import { StringDecoder } from "node:string_decoder";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
import { QaSuiteInfraError } from "./errors.js";
|
||||
import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js";
|
||||
import {
|
||||
inspectLinuxProcessGroup,
|
||||
type QaLinuxProcessGroupInspector,
|
||||
} from "./posix-process-group.js";
|
||||
import { runQaWindowsTaskkill } from "./windows-system-tools.js";
|
||||
|
||||
const QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 30_000;
|
||||
const QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS = 10_000;
|
||||
const QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS = 5_000;
|
||||
const QA_GATEWAY_CHILD_RECENT_LOG_CHARS = 64 * 1_024;
|
||||
const QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER = "[qa-lab] older gateway logs truncated\n";
|
||||
const QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS = 8_192;
|
||||
|
||||
export type QaChildFailure = {
|
||||
source: "process" | "stdout" | "stderr";
|
||||
error: unknown;
|
||||
};
|
||||
|
||||
type QaGatewayChildLogSource = "internal" | "stderr" | "stdout";
|
||||
|
||||
export function hasQaGatewayChildExited(child: Pick<ChildProcess, "exitCode" | "signalCode">) {
|
||||
return child.exitCode !== null || child.signalCode !== null;
|
||||
}
|
||||
|
||||
export function monitorQaChildFailure(
|
||||
child: ChildProcess,
|
||||
onFailure: (failure: QaChildFailure) => void,
|
||||
) {
|
||||
let reported = false;
|
||||
const report = (source: QaChildFailure["source"]) => (error: unknown) => {
|
||||
if (reported) {
|
||||
return;
|
||||
}
|
||||
reported = true;
|
||||
onFailure({ source, error });
|
||||
};
|
||||
child.once("error", report("process"));
|
||||
child.stdout?.once("error", report("stdout"));
|
||||
child.stderr?.once("error", report("stderr"));
|
||||
}
|
||||
|
||||
export async function closeQaGatewayLogStream(
|
||||
stream: WriteStream,
|
||||
label: "stderr" | "stdout",
|
||||
timeoutMs = QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS,
|
||||
) {
|
||||
if (stream.destroyed) {
|
||||
return;
|
||||
}
|
||||
stream.end();
|
||||
const signal = AbortSignal.timeout(timeoutMs);
|
||||
try {
|
||||
await finished(stream, { cleanup: true, signal });
|
||||
} catch (error) {
|
||||
if (!signal.aborted) {
|
||||
throw error;
|
||||
}
|
||||
// Gateway logs are diagnostic only. Never let a stuck filesystem flush
|
||||
// retain the stopped child runtime and its live transport credentials.
|
||||
process.stderr.write(
|
||||
`[qa-suite] ${label} gateway log flush exceeded ${timeoutMs}ms; forcing close\n`,
|
||||
);
|
||||
stream.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
export function createQaGatewayChildLogCollector() {
|
||||
const decoders: Record<QaGatewayChildLogSource, StringDecoder> = {
|
||||
internal: new StringDecoder("utf8"),
|
||||
stderr: new StringDecoder("utf8"),
|
||||
stdout: new StringDecoder("utf8"),
|
||||
};
|
||||
let recent = "";
|
||||
let end = 0;
|
||||
|
||||
const readFrom = (mark: number) => {
|
||||
const start = end - recent.length;
|
||||
const wasTruncated = mark < start;
|
||||
const text = recent.slice(Math.max(0, mark - start));
|
||||
return `${wasTruncated ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${text}`;
|
||||
};
|
||||
return {
|
||||
push(source: QaGatewayChildLogSource, chunk: Buffer) {
|
||||
const text = decoders[source].write(chunk);
|
||||
end += text.length;
|
||||
recent += text;
|
||||
if (recent.length > QA_GATEWAY_CHILD_RECENT_LOG_CHARS) {
|
||||
recent = sliceUtf16Safe(recent, -QA_GATEWAY_CHILD_RECENT_LOG_CHARS);
|
||||
}
|
||||
},
|
||||
mark() {
|
||||
return end;
|
||||
},
|
||||
readSince(mark: number) {
|
||||
return readFrom(mark);
|
||||
},
|
||||
text() {
|
||||
return `${end > recent.length ? QA_GATEWAY_CHILD_LOG_TRUNCATION_MARKER : ""}${recent}`.trim();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatQaGatewayChildFailure(failure: QaChildFailure) {
|
||||
return failure.source === "process"
|
||||
? `gateway failed to spawn: ${formatErrorMessage(failure.error)}`
|
||||
: `gateway child ${failure.source} stream failed: ${formatErrorMessage(failure.error)}`;
|
||||
}
|
||||
|
||||
export function throwQaGatewayChildFailure(
|
||||
getChildFailure: (() => QaChildFailure | null) | undefined,
|
||||
logs: () => string,
|
||||
) {
|
||||
const failure = getChildFailure?.();
|
||||
if (!failure) {
|
||||
return;
|
||||
}
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`${formatQaGatewayChildFailure(failure)}\n${logs()}`,
|
||||
{ cause: failure.error },
|
||||
);
|
||||
}
|
||||
|
||||
export function monitorQaGatewayChildFailure(
|
||||
child: ChildProcess,
|
||||
output: { push(source: QaGatewayChildLogSource, chunk: Buffer): void },
|
||||
) {
|
||||
let childFailure: QaChildFailure | null = null;
|
||||
monitorQaChildFailure(child, (failure) => {
|
||||
childFailure = failure;
|
||||
const description =
|
||||
failure.source === "process"
|
||||
? `gateway child process error: ${formatErrorMessage(failure.error)}`
|
||||
: formatQaGatewayChildFailure(failure);
|
||||
output.push("internal", Buffer.from(`[qa-lab] ${description}\n`));
|
||||
if (failure.source !== "process" && !hasQaGatewayChildExited(child)) {
|
||||
// A broken parent-side pipe means QA can no longer observe the Gateway.
|
||||
// Stop the detached process tree so the existing lifecycle reports the failure.
|
||||
signalQaGatewayChildProcessTree(child, "SIGTERM");
|
||||
}
|
||||
});
|
||||
return () => childFailure;
|
||||
}
|
||||
|
||||
export function formatQaGatewayProcessBoundaryStartupFailure(error: unknown, logs: string) {
|
||||
const logTail = sliceUtf16Safe(
|
||||
redactQaGatewayDebugText(logs),
|
||||
-QA_GATEWAY_PROCESS_BOUNDARY_LOG_TAIL_CHARS,
|
||||
);
|
||||
return `${formatErrorMessage(error)}${formatQaGatewayLogsForError(logTail)}`;
|
||||
}
|
||||
|
||||
function isProcessAlreadyExitedError(error: unknown): boolean {
|
||||
return (error as NodeJS.ErrnoException | undefined)?.code === "ESRCH";
|
||||
}
|
||||
|
||||
function boundQaGatewayProcessTreeDiagnostics(details: string) {
|
||||
if (details.length <= 2_048) {
|
||||
return details;
|
||||
}
|
||||
return `${sliceUtf16Safe(details, 0, 2_045)}...`;
|
||||
}
|
||||
|
||||
function isQaGatewayChildProcessTreeAlive(
|
||||
child: ChildProcess,
|
||||
inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector = inspectLinuxProcessGroup,
|
||||
) {
|
||||
if (!child.pid) {
|
||||
return false;
|
||||
}
|
||||
if (process.platform === "win32") {
|
||||
return !hasQaGatewayChildExited(child);
|
||||
}
|
||||
try {
|
||||
process.kill(-child.pid, 0);
|
||||
if (process.platform === "linux") {
|
||||
// Linux can retain zombie-only process groups after SIGKILL while Node's
|
||||
// child metadata is still unsettled. Runnable /proc members are the owner.
|
||||
return inspectLinuxProcessGroupFn(child.pid)?.alive ?? true;
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (!isProcessAlreadyExitedError(error) && !hasQaGatewayChildExited(child)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function signalQaGatewayChildProcessTree(child: ChildProcess, signal: NodeJS.Signals) {
|
||||
if (!child.pid) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (process.platform === "win32") {
|
||||
if (runQaWindowsTaskkill({ pid: child.pid, signal })) {
|
||||
return;
|
||||
}
|
||||
child.kill(signal);
|
||||
return;
|
||||
}
|
||||
process.kill(-child.pid, signal);
|
||||
} catch {
|
||||
try {
|
||||
child.kill(signal);
|
||||
} catch {
|
||||
// The child already exited.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForQaGatewayChildExit(
|
||||
child: ChildProcess,
|
||||
timeoutMs: number,
|
||||
inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector,
|
||||
) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) {
|
||||
return true;
|
||||
}
|
||||
await sleep(Math.min(25, Math.max(0, deadline - Date.now())));
|
||||
}
|
||||
return !isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn);
|
||||
}
|
||||
|
||||
type QaGatewayChildStopOptions = {
|
||||
gracefulTimeoutMs?: number;
|
||||
forceTimeoutMs?: number;
|
||||
inspectLinuxProcessGroup?: QaLinuxProcessGroupInspector;
|
||||
};
|
||||
|
||||
function resolveQaGatewayChildStopTimeouts(opts?: QaGatewayChildStopOptions) {
|
||||
return {
|
||||
gracefulTimeoutMs: opts?.gracefulTimeoutMs ?? QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS,
|
||||
forceTimeoutMs: opts?.forceTimeoutMs ?? QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS,
|
||||
};
|
||||
}
|
||||
|
||||
function formatQaGatewayProcessTreeDiagnostics(
|
||||
child: ChildProcess,
|
||||
inspectLinuxProcessGroupFn: QaLinuxProcessGroupInspector,
|
||||
) {
|
||||
const childExitRecorded = hasQaGatewayChildExited(child);
|
||||
if (process.platform !== "linux" || !child.pid) {
|
||||
return `pid=${child.pid ?? "unknown"} childExitRecorded=${childExitRecorded}`;
|
||||
}
|
||||
const inspection = inspectLinuxProcessGroupFn(child.pid);
|
||||
const processGroupDetails =
|
||||
inspection?.diagnostics ?? `pgid=${child.pid} members=unknown (/proc unavailable)`;
|
||||
return boundQaGatewayProcessTreeDiagnostics(
|
||||
`${processGroupDetails} childExitRecorded=${childExitRecorded}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function stopQaGatewayChildProcessTree(
|
||||
child: ChildProcess,
|
||||
opts?: QaGatewayChildStopOptions,
|
||||
) {
|
||||
const inspectLinuxProcessGroupFn = opts?.inspectLinuxProcessGroup ?? inspectLinuxProcessGroup;
|
||||
if (!isQaGatewayChildProcessTreeAlive(child, inspectLinuxProcessGroupFn)) {
|
||||
return;
|
||||
}
|
||||
const timeouts = resolveQaGatewayChildStopTimeouts(opts);
|
||||
signalQaGatewayChildProcessTree(child, "SIGTERM");
|
||||
if (
|
||||
await waitForQaGatewayChildExit(child, timeouts.gracefulTimeoutMs, inspectLinuxProcessGroupFn)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
signalQaGatewayChildProcessTree(child, "SIGKILL");
|
||||
const stopped = await waitForQaGatewayChildExit(
|
||||
child,
|
||||
timeouts.forceTimeoutMs,
|
||||
inspectLinuxProcessGroupFn,
|
||||
);
|
||||
if (!stopped) {
|
||||
throw new Error(
|
||||
`qa gateway process tree remained alive after forced shutdown: ${formatQaGatewayProcessTreeDiagnostics(
|
||||
child,
|
||||
inspectLinuxProcessGroupFn,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// Qa Lab plugin module owns gateway readiness and retry behavior.
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { QaSuiteInfraError } from "./errors.js";
|
||||
import {
|
||||
hasQaGatewayChildExited,
|
||||
type QaChildFailure,
|
||||
throwQaGatewayChildFailure,
|
||||
} from "./gateway-child-process.js";
|
||||
import { formatQaGatewayLogsForError } from "./gateway-log-redaction.js";
|
||||
|
||||
export const QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS = 5;
|
||||
const QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS = 90_000;
|
||||
const QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX =
|
||||
"OpenClaw plugin migration inputs changed during startup convergence;";
|
||||
|
||||
type QaGatewayStartupRetryKind = "bind-collision" | "migration-convergence-restart";
|
||||
|
||||
type QaGatewayHealthChild = {
|
||||
exitCode: number | null;
|
||||
signalCode: NodeJS.Signals | null;
|
||||
};
|
||||
|
||||
function classifyQaGatewayStartupRetry(details: string): QaGatewayStartupRetryKind | null {
|
||||
if (details.includes(QA_GATEWAY_MIGRATION_CONVERGENCE_RESTART_PREFIX)) {
|
||||
return "migration-convergence-restart";
|
||||
}
|
||||
if (
|
||||
details.includes("another gateway instance is already listening on ws://") ||
|
||||
details.includes("failed to bind gateway socket on ws://") ||
|
||||
details.includes("EADDRINUSE") ||
|
||||
details.includes("address already in use")
|
||||
) {
|
||||
return "bind-collision";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveQaGatewayStartupRetry(params: {
|
||||
attempt: number;
|
||||
details: string;
|
||||
migrationConvergenceRestartUsed: boolean;
|
||||
}) {
|
||||
if (params.attempt >= QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS) {
|
||||
return null;
|
||||
}
|
||||
const kind = classifyQaGatewayStartupRetry(params.details);
|
||||
if (
|
||||
!kind ||
|
||||
(kind === "migration-convergence-restart" && params.migrationConvergenceRestartUsed)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
kind,
|
||||
reuseLaunchState: kind === "migration-convergence-restart",
|
||||
migrationConvergenceRestartUsed:
|
||||
params.migrationConvergenceRestartUsed || kind === "migration-convergence-restart",
|
||||
};
|
||||
}
|
||||
|
||||
function isRetryableGatewayCallError(details: string): boolean {
|
||||
return (
|
||||
details.includes("handshake timeout") ||
|
||||
details.includes("gateway closed (1000") ||
|
||||
details.includes("gateway closed (1012)") ||
|
||||
details.includes("gateway closed (1006") ||
|
||||
details.includes("abnormal closure") ||
|
||||
details.includes("service restart")
|
||||
);
|
||||
}
|
||||
|
||||
export async function callQaGatewayWithRetry<T>(params: {
|
||||
deadlineMs?: number;
|
||||
logs: () => string;
|
||||
request: (options: { deadlineMs?: number; timeoutMs: number }) => Promise<T>;
|
||||
throwChildFailure: () => void;
|
||||
timeoutMs: number;
|
||||
waitForReady: (timeoutMs: number) => Promise<void>;
|
||||
}) {
|
||||
const remainingMs = () =>
|
||||
params.deadlineMs === undefined ? undefined : params.deadlineMs - Date.now();
|
||||
const deadlineError = () =>
|
||||
new Error(`gateway call deadline exceeded${formatQaGatewayLogsForError(params.logs())}`);
|
||||
let lastDetails = "";
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
params.throwChildFailure();
|
||||
const requestRemainingMs = remainingMs();
|
||||
if (requestRemainingMs !== undefined && requestRemainingMs <= 0) {
|
||||
throw deadlineError();
|
||||
}
|
||||
try {
|
||||
return await params.request({
|
||||
...(params.deadlineMs === undefined ? {} : { deadlineMs: params.deadlineMs }),
|
||||
timeoutMs:
|
||||
requestRemainingMs === undefined
|
||||
? params.timeoutMs
|
||||
: Math.min(params.timeoutMs, requestRemainingMs),
|
||||
});
|
||||
} catch (error) {
|
||||
params.throwChildFailure();
|
||||
const details = formatErrorMessage(error);
|
||||
lastDetails = details;
|
||||
if (attempt >= 3 || !isRetryableGatewayCallError(details)) {
|
||||
throw new Error(`${details}${formatQaGatewayLogsForError(params.logs())}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
const readinessRemainingMs = remainingMs();
|
||||
if (readinessRemainingMs !== undefined && readinessRemainingMs <= 0) {
|
||||
throw deadlineError();
|
||||
}
|
||||
await params.waitForReady(
|
||||
readinessRemainingMs === undefined
|
||||
? Math.max(10_000, params.timeoutMs)
|
||||
: Math.min(Math.max(10_000, params.timeoutMs), readinessRemainingMs),
|
||||
);
|
||||
}
|
||||
}
|
||||
throw new Error(`${lastDetails}${formatQaGatewayLogsForError(params.logs())}`);
|
||||
}
|
||||
|
||||
async function fetchLocalGatewayHealth(params: {
|
||||
baseUrl: string;
|
||||
healthPath: "/readyz" | "/healthz";
|
||||
timeoutMs?: number;
|
||||
}): Promise<boolean> {
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: `${params.baseUrl}${params.healthPath}`,
|
||||
init: {
|
||||
method: "HEAD",
|
||||
headers: {
|
||||
connection: "close",
|
||||
},
|
||||
signal: AbortSignal.timeout(params.timeoutMs ?? 2_000),
|
||||
},
|
||||
policy: { allowPrivateNetwork: true },
|
||||
auditContext: "qa-lab-gateway-child-health",
|
||||
});
|
||||
try {
|
||||
return response.ok;
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchLocalGatewayListening(baseUrl: string): Promise<boolean> {
|
||||
const { release } = await fetchWithSsrFGuard({
|
||||
url: `${baseUrl}/healthz`,
|
||||
init: {
|
||||
method: "HEAD",
|
||||
headers: {
|
||||
connection: "close",
|
||||
},
|
||||
signal: AbortSignal.timeout(2_000),
|
||||
},
|
||||
policy: { allowPrivateNetwork: true },
|
||||
auditContext: "qa-lab-gateway-child-listening",
|
||||
});
|
||||
await release();
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function waitForQaGatewayRestartBoundary(params: {
|
||||
readLogsSince: (mark: number) => string;
|
||||
mark: number;
|
||||
pollMs?: number;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const timeoutMs = params.timeoutMs ?? QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS;
|
||||
const pollMs = resolveTimerTimeoutMs(params.pollMs ?? 100, 100, 0);
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (params.readLogsSince(params.mark).includes("restart mode:")) {
|
||||
return;
|
||||
}
|
||||
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
||||
if (remainingMs <= 0) {
|
||||
break;
|
||||
}
|
||||
await sleep(Math.min(pollMs, remainingMs));
|
||||
}
|
||||
throw new Error(`qa gateway child did not reach restart boundary within ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
export async function waitForGatewayReady(params: {
|
||||
baseUrl: string;
|
||||
logs: () => string;
|
||||
child: QaGatewayHealthChild;
|
||||
getChildFailure?: () => QaChildFailure | null;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const deadline = Date.now() + (params.timeoutMs ?? 60_000);
|
||||
let remainingMs: number;
|
||||
while ((remainingMs = deadline - Date.now()) > 0) {
|
||||
throwQaGatewayChildFailure(params.getChildFailure, params.logs);
|
||||
if (hasQaGatewayChildExited(params.child)) {
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway exited before becoming healthy (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
|
||||
);
|
||||
}
|
||||
// Listener liveness can turn green before the Gateway can admit startup or restart work.
|
||||
try {
|
||||
if (
|
||||
await fetchLocalGatewayHealth({
|
||||
baseUrl: params.baseUrl,
|
||||
healthPath: "/readyz",
|
||||
timeoutMs: Math.min(2_000, remainingMs),
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// retry until timeout
|
||||
}
|
||||
await sleep(Math.min(250, Math.max(0, deadline - Date.now())));
|
||||
}
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway failed to become healthy:\n${params.logs()}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function waitForGatewayListening(params: {
|
||||
baseUrl: string;
|
||||
logs: () => string;
|
||||
child: QaGatewayHealthChild;
|
||||
getChildFailure?: () => QaChildFailure | null;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
|
||||
throwQaGatewayChildFailure(params.getChildFailure, params.logs);
|
||||
if (params.child.exitCode !== null || params.child.signalCode !== null) {
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway exited before listening (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
if (await fetchLocalGatewayListening(params.baseUrl)) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// retry until the HTTP listener accepts requests
|
||||
}
|
||||
await sleep(100);
|
||||
}
|
||||
throw new QaSuiteInfraError(
|
||||
"gateway_startup_unhealthy",
|
||||
`gateway failed to listen before timeout:\n${params.logs()}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function isRetryableRpcStartupError(error: unknown) {
|
||||
const details = formatErrorMessage(error);
|
||||
return (
|
||||
details.includes("gateway timeout after") ||
|
||||
details.includes("handshake timeout") ||
|
||||
details.includes("gateway token mismatch") ||
|
||||
details.includes("token mismatch") ||
|
||||
details.includes("gateway closed (1000") ||
|
||||
details.includes("gateway closed (1006") ||
|
||||
details.includes("gateway closed (1012)")
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+114
-1119
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
inspectLinuxProcessGroupStats,
|
||||
isQaPosixProcessGroupAlive,
|
||||
signalQaPosixProcessGroup,
|
||||
} from "./posix-process-group.js";
|
||||
import { isQaPosixProcessGroupAlive, signalQaPosixProcessGroup } from "./posix-process-group.js";
|
||||
import { inspectLinuxProcessGroupStats } from "./posix-process-stat.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
@@ -37,6 +34,19 @@ describe("POSIX process group inspection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds process group diagnostics", () => {
|
||||
const stats = Array.from(
|
||||
{ length: 300 },
|
||||
(_, index) => `${index + 1} (${`worker-${index}`.padEnd(32, "x")}) S 1 123 123 0 -1 0`,
|
||||
);
|
||||
|
||||
const inspection = inspectLinuxProcessGroupStats(123, stats);
|
||||
|
||||
expect(inspection.alive).toBe(true);
|
||||
expect(inspection.diagnostics.length).toBeLessThanOrEqual(2_048);
|
||||
expect(inspection.diagnostics).toMatch(/\.\.\.$/u);
|
||||
});
|
||||
|
||||
it("fails closed when the Linux member snapshot is unavailable", () => {
|
||||
const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
|
||||
const processKill = vi.spyOn(process, "kill").mockImplementation(() => true);
|
||||
|
||||
@@ -1,66 +1,6 @@
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
|
||||
function parseLinuxProcessStat(raw: string) {
|
||||
const commandStart = raw.indexOf("(");
|
||||
const commandEnd = raw.lastIndexOf(")");
|
||||
if (commandStart <= 0 || commandEnd <= commandStart) {
|
||||
return null;
|
||||
}
|
||||
const pid = Number.parseInt(raw.slice(0, commandStart).trim(), 10);
|
||||
const fields = raw
|
||||
.slice(commandEnd + 1)
|
||||
.trim()
|
||||
.split(/\s+/u);
|
||||
const state = fields[0];
|
||||
const processGroupId = Number.parseInt(fields[2] ?? "", 10);
|
||||
if (
|
||||
!Number.isSafeInteger(pid) ||
|
||||
pid <= 0 ||
|
||||
!state ||
|
||||
!Number.isSafeInteger(processGroupId) ||
|
||||
processGroupId <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
command: raw.slice(commandStart + 1, commandEnd),
|
||||
pid,
|
||||
processGroupId,
|
||||
state,
|
||||
};
|
||||
}
|
||||
|
||||
function boundProcessGroupDiagnostics(details: string) {
|
||||
if (details.length <= 2_048) {
|
||||
return details;
|
||||
}
|
||||
return `${sliceUtf16Safe(details, 0, 2_045)}...`;
|
||||
}
|
||||
|
||||
export function inspectLinuxProcessGroupStats(processGroupId: number, stats: readonly string[]) {
|
||||
const members = stats
|
||||
.map((raw) => parseLinuxProcessStat(raw))
|
||||
.filter(
|
||||
(entry): entry is NonNullable<ReturnType<typeof parseLinuxProcessStat>> =>
|
||||
entry?.processGroupId === processGroupId,
|
||||
)
|
||||
.toSorted((left, right) => left.pid - right.pid);
|
||||
const diagnostics = members
|
||||
.map(
|
||||
(member) =>
|
||||
`pid=${member.pid} state=${member.state} command=${JSON.stringify(member.command)}`,
|
||||
)
|
||||
.join(", ");
|
||||
return {
|
||||
alive:
|
||||
members.length === 0
|
||||
? null
|
||||
: members.some((entry) => entry.state !== "Z" && entry.state !== "X"),
|
||||
diagnostics: boundProcessGroupDiagnostics(`pgid=${processGroupId} members=[${diagnostics}]`),
|
||||
};
|
||||
}
|
||||
import { inspectLinuxProcessGroupStats } from "./posix-process-stat.js";
|
||||
|
||||
type QaLinuxProcessGroupInspection = ReturnType<typeof inspectLinuxProcessGroupStats>;
|
||||
export type QaLinuxProcessGroupInspector = (
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// Qa Lab parses Linux process stat snapshots for process-group ownership.
|
||||
import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
|
||||
function parseLinuxProcessStat(raw: string) {
|
||||
const commandStart = raw.indexOf("(");
|
||||
const commandEnd = raw.lastIndexOf(")");
|
||||
if (commandStart <= 0 || commandEnd <= commandStart) {
|
||||
return null;
|
||||
}
|
||||
const pid = Number.parseInt(raw.slice(0, commandStart).trim(), 10);
|
||||
const fields = raw
|
||||
.slice(commandEnd + 1)
|
||||
.trim()
|
||||
.split(/\s+/u);
|
||||
const state = fields[0];
|
||||
const processGroupId = Number.parseInt(fields[2] ?? "", 10);
|
||||
if (
|
||||
!Number.isSafeInteger(pid) ||
|
||||
pid <= 0 ||
|
||||
!state ||
|
||||
!Number.isSafeInteger(processGroupId) ||
|
||||
processGroupId <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
command: raw.slice(commandStart + 1, commandEnd),
|
||||
pid,
|
||||
processGroupId,
|
||||
state,
|
||||
};
|
||||
}
|
||||
|
||||
function boundProcessGroupDiagnostics(details: string) {
|
||||
if (details.length <= 2_048) {
|
||||
return details;
|
||||
}
|
||||
return `${sliceUtf16Safe(details, 0, 2_045)}...`;
|
||||
}
|
||||
|
||||
export function inspectLinuxProcessGroupStats(processGroupId: number, stats: readonly string[]) {
|
||||
const members = stats
|
||||
.map((raw) => parseLinuxProcessStat(raw))
|
||||
.filter(
|
||||
(entry): entry is NonNullable<ReturnType<typeof parseLinuxProcessStat>> =>
|
||||
entry?.processGroupId === processGroupId,
|
||||
)
|
||||
.toSorted((left, right) => left.pid - right.pid);
|
||||
const diagnostics = members
|
||||
.map(
|
||||
(member) =>
|
||||
`pid=${member.pid} state=${member.state} command=${JSON.stringify(member.command)}`,
|
||||
)
|
||||
.join(", ");
|
||||
return {
|
||||
alive:
|
||||
members.length === 0
|
||||
? null
|
||||
: members.some((entry) => entry.state !== "Z" && entry.state !== "X"),
|
||||
diagnostics: boundProcessGroupDiagnostics(`pgid=${processGroupId} members=[${diagnostics}]`),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
// Qa Lab plugin module owns host live-provider config projection.
|
||||
import { existsSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import {
|
||||
isRecord,
|
||||
normalizeOptionalString,
|
||||
normalizeStringEntries,
|
||||
uniqueStrings,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { QA_LIVE_PROVIDER_CONFIG_PATH_ENV, resolveQaLiveProviderConfigPath } from "./env.js";
|
||||
|
||||
function isQaModelProviderConfig(value: unknown): value is ModelProviderConfig {
|
||||
return isRecord(value) && typeof value.baseUrl === "string" && Array.isArray(value.models);
|
||||
}
|
||||
|
||||
function normalizeQaLiveProviderConfig(value: unknown): ModelProviderConfig | null {
|
||||
if (!isQaModelProviderConfig(value) && (!isRecord(value) || !Object.hasOwn(value, "apiKey"))) {
|
||||
return null;
|
||||
}
|
||||
const { baseUrl: rawBaseUrl, ...providerConfig } = value;
|
||||
const baseUrl = normalizeOptionalString(rawBaseUrl);
|
||||
return {
|
||||
...providerConfig,
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
models: Array.isArray(value.models) ? value.models : [],
|
||||
} as ModelProviderConfig;
|
||||
}
|
||||
|
||||
export async function readQaLiveProviderConfigOverrides(params: {
|
||||
providerIds: readonly string[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}) {
|
||||
const providerIds = uniqueStrings(normalizeStringEntries(params.providerIds));
|
||||
if (providerIds.length === 0) {
|
||||
return {};
|
||||
}
|
||||
const configPath = resolveQaLiveProviderConfigPath(params.env);
|
||||
if (!existsSync(configPath.path)) {
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
const raw = await fs.readFile(configPath.path, "utf8");
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
const providers = isRecord(parsed)
|
||||
? isRecord(parsed.models)
|
||||
? isRecord(parsed.models.providers)
|
||||
? parsed.models.providers
|
||||
: {}
|
||||
: {}
|
||||
: {};
|
||||
const selected: Record<string, ModelProviderConfig> = {};
|
||||
for (const providerId of providerIds) {
|
||||
const providerConfig = normalizeQaLiveProviderConfig(providers[providerId]);
|
||||
if (providerConfig) {
|
||||
selected[providerId] = providerConfig;
|
||||
}
|
||||
}
|
||||
return selected;
|
||||
} catch (error) {
|
||||
if (configPath.explicit) {
|
||||
throw new Error(
|
||||
`failed to read ${QA_LIVE_PROVIDER_CONFIG_PATH_ENV} provider config: ${formatErrorMessage(error)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
}
|
||||
@@ -159,7 +159,7 @@ export function listMockCodexModelInfos(selectedModelRefs: readonly string[] = [
|
||||
upgrade: null,
|
||||
base_instructions: "You are Codex, a coding agent based on GPT-5.",
|
||||
include_skills_usage_instructions: false,
|
||||
supports_reasoning_summaries: true,
|
||||
supports_reasoning_summary_parameter: true,
|
||||
default_reasoning_summary: "none",
|
||||
support_verbosity: true,
|
||||
default_verbosity: "low",
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
// Qa Lab tests cover Windows system tool path resolution.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
resolveQaWindowsPowerShellExePath,
|
||||
resolveQaWindowsSystem32ExePath,
|
||||
runQaWindowsTaskkill,
|
||||
} from "./windows-system-tools.js";
|
||||
|
||||
describe("qa-lab windows system tools", () => {
|
||||
@@ -15,6 +16,34 @@ describe("qa-lab windows system tools", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("force-kills a process tree when graceful taskkill fails", () => {
|
||||
const runCommand = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce({ status: 1 })
|
||||
.mockReturnValueOnce({ status: 0 });
|
||||
|
||||
expect(
|
||||
runQaWindowsTaskkill({
|
||||
pid: 12345,
|
||||
signal: "SIGTERM",
|
||||
env: { SystemRoot: "D:\\Windows" },
|
||||
runCommand,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(runCommand).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
"D:\\Windows\\System32\\taskkill.exe",
|
||||
["/PID", "12345", "/T"],
|
||||
{ stdio: "ignore", windowsHide: true, timeout: 5_000 },
|
||||
);
|
||||
expect(runCommand).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"D:\\Windows\\System32\\taskkill.exe",
|
||||
["/PID", "12345", "/T", "/F"],
|
||||
{ stdio: "ignore", windowsHide: true, timeout: 5_000 },
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the default Windows root when env roots are unsafe", () => {
|
||||
expect(resolveQaWindowsSystem32ExePath("taskkill.exe", { SystemRoot: "C:\\tmp;C:\\bad" })).toBe(
|
||||
"C:\\Windows\\System32\\taskkill.exe",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Qa Lab resolves Windows system tools without trusting PATH.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
|
||||
const DEFAULT_WINDOWS_SYSTEM_ROOT = "C:\\Windows";
|
||||
@@ -59,6 +60,37 @@ export function resolveQaWindowsSystem32ExePath(
|
||||
return path.win32.join(resolveQaWindowsSystemRoot(env), "System32", executableName);
|
||||
}
|
||||
|
||||
export function runQaWindowsTaskkill(params: {
|
||||
pid: number;
|
||||
signal: NodeJS.Signals;
|
||||
env?: Record<string, string | undefined>;
|
||||
runCommand?: typeof spawnSync;
|
||||
}) {
|
||||
const runCommand = params.runCommand ?? spawnSync;
|
||||
const taskkillPath = resolveQaWindowsSystem32ExePath("taskkill.exe", params.env);
|
||||
const args = ["/PID", String(params.pid), "/T"];
|
||||
if (params.signal === "SIGKILL") {
|
||||
args.push("/F");
|
||||
}
|
||||
const result = runCommand(taskkillPath, args, {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
timeout: 5_000,
|
||||
});
|
||||
if (!result.error && result.status === 0) {
|
||||
return true;
|
||||
}
|
||||
if (params.signal !== "SIGKILL") {
|
||||
const forceResult = runCommand(taskkillPath, [...args, "/F"], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
timeout: 5_000,
|
||||
});
|
||||
return !forceResult.error && forceResult.status === 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function resolveQaWindowsPowerShellExePath(
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): string {
|
||||
|
||||
@@ -20,7 +20,8 @@ scenario:
|
||||
- docs/channels/qa-channel.md
|
||||
codeRefs:
|
||||
- src/agents/system-prompt.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -18,7 +18,8 @@ scenario:
|
||||
- The failed tool call is never replayed or described as successful.
|
||||
- The model receives one tools-disabled finalization and qa-channel delivers exactly one visible reply.
|
||||
codeRefs:
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
- src/agents/embedded-agent-runner/run/code-mode-repair.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
|
||||
@@ -15,7 +15,8 @@ scenario:
|
||||
- The runtime continues from settled tool results without replaying the write.
|
||||
- Telegram receives the exact recovery marker.
|
||||
codeRefs:
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
- src/agents/embedded-agent-runner/run/terminal-resolution.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
|
||||
@@ -27,7 +27,8 @@ scenario:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify Anthropic stream errors after signed thinking recover after a replay-safe read.
|
||||
|
||||
@@ -18,7 +18,8 @@ scenario:
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify a short approval like "ok do it" triggers immediate tool use instead of fake-progress narration.
|
||||
|
||||
@@ -21,7 +21,8 @@ scenario:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify empty OpenAI turns recover after a replay-safe read.
|
||||
|
||||
@@ -19,7 +19,8 @@ scenario:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify empty-response retry exhaustion still surfaces a visible failure.
|
||||
|
||||
@@ -20,7 +20,8 @@ scenario:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify reasoning-only turns after a write do not auto-retry.
|
||||
|
||||
@@ -18,7 +18,8 @@ scenario:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify reasoning-only OpenAI turns recover after a replay-safe read.
|
||||
|
||||
@@ -19,7 +19,8 @@ scenario:
|
||||
- docs/concepts/streaming.md
|
||||
- docs/channels/qa-channel.md
|
||||
codeRefs:
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
- extensions/qa-lab/src/bus-state.ts
|
||||
- extensions/qa-lab/src/suite-runtime-transport.ts
|
||||
execution:
|
||||
|
||||
@@ -20,7 +20,8 @@ scenario:
|
||||
codeRefs:
|
||||
- extensions/qa-channel/src/inbound.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Complete a proof-backed QA-channel task and verify artifact-before-reply ordering plus honest terminal status.
|
||||
|
||||
@@ -19,7 +19,8 @@ scenario:
|
||||
- The failed tool never replays and qa-channel receives exactly one final reply.
|
||||
codeRefs:
|
||||
- src/cron/isolated-agent/run-executor.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -20,7 +20,8 @@ scenario:
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/suite-runtime-agent-process.ts
|
||||
- extensions/qa-lab/src/suite-runtime-transport.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify fake secret fixtures are not echoed into channel-visible output.
|
||||
|
||||
@@ -27,7 +27,7 @@ scenario:
|
||||
- docs/gateway/protocol.md
|
||||
codeRefs:
|
||||
- ui/src/e2e/config-safe-write.e2e.test.ts
|
||||
- ui/src/lib/config/index.ts
|
||||
- ui/src/lib/config/config-write-coordinator.ts
|
||||
- ui/src/components/settings-save-indicator.ts
|
||||
execution:
|
||||
kind: playwright
|
||||
|
||||
Reference in New Issue
Block a user