diff --git a/extensions/qa-lab/src/test-file-scenario-docker-batch.ts b/extensions/qa-lab/src/test-file-scenario-docker-batch.ts new file mode 100644 index 000000000000..e1b922e5b035 --- /dev/null +++ b/extensions/qa-lab/src/test-file-scenario-docker-batch.ts @@ -0,0 +1,161 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { shellQuote } from "./shell-quote.js"; +import type { + QaScenarioCommandExecution, + QaScenarioCommandResult, +} from "./test-file-scenario-command-lifecycle.js"; + +const QA_DOCKER_E2E_LANE_SCRIPT = "test/e2e/qa-lab/runtime/docker-e2e-lane.ts"; + +type QaDockerScenario = QaSeedScenarioWithSource & { + execution: Extract; +}; + +type QaDockerBatchResult = { + durationMs: number; + failureMessage?: string; + logPath: string; + scenario: QaDockerScenario; + status: "fail" | "pass"; +}; + +export function dockerE2eLaneName(scenario: QaSeedScenarioWithSource) { + const args = scenario.execution.kind === "script" ? scenario.execution.args : undefined; + if ( + scenario.execution.kind !== "script" || + scenario.execution.path !== QA_DOCKER_E2E_LANE_SCRIPT || + args?.length !== 2 || + args[0] !== "--lane" + ) { + return undefined; + } + const laneName = args[1]?.trim(); + return laneName || undefined; +} + +export function isDockerE2eScenario( + scenario: QaSeedScenarioWithSource, +): scenario is QaDockerScenario { + return dockerE2eLaneName(scenario) !== undefined; +} + +function laneMatches( + selectedLane: string, + resultLane: string | undefined, + resolvedLaneNames: readonly string[], +) { + // The scheduler summary records aliases as their resolved lanes. Prefix matching is + // safe only when the selected name itself disappeared during that resolution. + return ( + resultLane === selectedLane || + (!resolvedLaneNames.includes(selectedLane) && + resultLane !== undefined && + resolvedLaneNames.includes(resultLane) && + resultLane.startsWith(`${selectedLane}-`)) + ); +} + +export async function runDockerE2eBatch(params: { + commandTimeoutMs: number; + env: NodeJS.ProcessEnv; + outputDir: string; + repoRoot: string; + runCommand: (command: QaScenarioCommandExecution) => Promise; + scenarios: readonly QaDockerScenario[]; +}): Promise { + const selected = params.scenarios.map((scenario) => ({ + lane: dockerE2eLaneName(scenario)!, + scenario, + })); + const laneNames = [...new Set(selected.map(({ lane }) => lane))]; + const batchId = `${params.commandTimeoutMs}ms`; + const dockerOutputDir = path.join(params.outputDir, `docker-e2e-${batchId}`); + const logPath = path.join(params.outputDir, `docker-e2e-batch-${batchId}.log`); + await fs.mkdir(dockerOutputDir, { recursive: true }); + const summaryPath = path.join(dockerOutputDir, "summary.json"); + await fs.rm(summaryPath, { force: true }); + let commandResult: QaScenarioCommandResult; + try { + commandResult = await params.runCommand({ + command: process.execPath, + args: ["scripts/test-docker-all.mjs"], + cwd: params.repoRoot, + env: { + ...params.env, + OPENCLAW_DOCKER_ALL_BUILD: "1", + OPENCLAW_DOCKER_ALL_FAIL_FAST: "0", + OPENCLAW_DOCKER_ALL_LANES: laneNames.join(","), + OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS: String(params.commandTimeoutMs), + OPENCLAW_DOCKER_ALL_LOG_DIR: dockerOutputDir, + OPENCLAW_DOCKER_ALL_PROFILE: "all", + OPENCLAW_DOCKER_ALL_TIMINGS_FILE: path.join(dockerOutputDir, "lane-timings.json"), + }, + // The scheduler owns each resolved lane deadline. Parent signals and the + // enclosing QA workflow bound the aggregate run without alias-count guesses. + }); + } catch (error) { + commandResult = { + exitCode: 1, + failureMessage: formatErrorMessage(error), + stderr: `${formatErrorMessage(error)}\n`, + stdout: "", + }; + } + await fs.writeFile( + logPath, + `$ ${shellQuote(process.execPath)} scripts/test-docker-all.mjs\n${commandResult.stdout}${commandResult.stderr}`, + "utf8", + ); + + let summary: + | { + failures?: Array<{ name?: string }>; + lanes?: Array<{ elapsedSeconds?: number; name?: string; status?: number }>; + selectedLanes?: string[]; + } + | undefined; + try { + summary = JSON.parse(await fs.readFile(summaryPath, "utf8")); + } catch { + // The command-level failure below owns missing or incomplete scheduler output. + } + const lanes = summary?.lanes ?? []; + const failures = summary?.failures ?? []; + const resolvedLaneNames = summary?.selectedLanes ?? []; + const unexplainedFailure = + commandResult.exitCode !== 0 && + (failures.length === 0 || + failures.some( + (failure) => + !laneNames.some((laneName) => laneMatches(laneName, failure.name, resolvedLaneNames)), + )); + return selected.map(({ lane, scenario }) => { + const matchingLanes = lanes.filter((result) => + laneMatches(lane, result.name, resolvedLaneNames), + ); + const failedLane = matchingLanes.find((result) => result.status !== 0); + const failureMessage = unexplainedFailure + ? commandResult.failureMessage || "Docker E2E scheduler failed before reporting lane results" + : failedLane + ? `${failedLane.name ?? lane} exited with ${String(failedLane.status ?? 1)}` + : matchingLanes.length === 0 + ? `Docker E2E scheduler returned no result for ${lane}` + : undefined; + const result: QaDockerBatchResult = { + durationMs: Math.max( + 1, + ...matchingLanes.map((entry) => Math.max(0, entry.elapsedSeconds ?? 0) * 1000), + ), + logPath, + scenario, + status: failureMessage ? "fail" : "pass", + }; + if (failureMessage) { + result.failureMessage = failureMessage; + } + return result; + }); +} diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts index d9e42f8ebd5c..efa875ccf222 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.test.ts @@ -6,6 +6,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest import { validateQaEvidenceSummaryJson } from "./evidence-summary.js"; import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; import { createTempDirHarness } from "./temp-dir.test-helper.js"; +import { dockerE2eLaneName } from "./test-file-scenario-docker-batch.js"; import { qaTestFileScenarioRunnerTesting, runQaTestFileScenarios, @@ -81,6 +82,35 @@ function makeTestFileScenario( }; } +function makeDockerE2eScenario(id: string, lane: string): QaSeedScenarioWithSource { + const scenario = makeTestFileScenario("script", "test/e2e/qa-lab/runtime/docker-e2e-lane.ts"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + return { + ...scenario, + id, + execution: { + ...scenario.execution, + args: ["--lane", lane], + }, + }; +} + +it("only batches the canonical Docker lane argument shape", () => { + const scenario = makeDockerE2eScenario("docker-lane", "gateway-network"); + if (scenario.execution.kind !== "script") { + throw new Error("expected script scenario"); + } + expect(dockerE2eLaneName(scenario)).toBe("gateway-network"); + expect( + dockerE2eLaneName({ + ...scenario, + execution: { ...scenario.execution, args: ["--lane", "gateway-network", "--extra"] }, + }), + ).toBeUndefined(); +}); + async function makeTempRepo(prefix: string) { const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); tempRoots.push(repoRoot); @@ -514,6 +544,79 @@ describe("qa test file scenario runner", () => { }); }); + it("runs Docker script scenarios through one aggregate scheduler invocation", async () => { + const repoRoot = await makeTempRepo("qa-script-docker-batch-"); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "docker-batch"); + const staleSummaryPath = path.join(outputDir, "docker-e2e-1800000ms", "summary.json"); + await fs.mkdir(path.dirname(staleSummaryPath), { recursive: true }); + await fs.writeFile(staleSummaryPath, '{"status":"passed"}\n', "utf8"); + const commands: QaScenarioCommandExecution[] = []; + const scenarios = [ + makeDockerE2eScenario("openai-tools", "openai-chat-tools"), + makeDockerE2eScenario("bundled-plugins", "bundled-plugin-install-uninstall"), + makeDockerE2eScenario("prefix-lane", "gateway"), + makeDockerE2eScenario("failing-lane", "gateway-network"), + ]; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + scenarios, + runCommand: async (command) => { + commands.push(command); + await expect(fs.access(staleSummaryPath)).rejects.toThrow(); + const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR; + if (!logDir) { + throw new Error("missing Docker scheduler log dir"); + } + await fs.mkdir(logDir, { recursive: true }); + const failedLane = { elapsedSeconds: 2, name: "gateway-network", status: 1 }; + await fs.writeFile( + path.join(logDir, "summary.json"), + `${JSON.stringify({ + failures: [failedLane], + lanes: [ + { elapsedSeconds: 4, name: "openai-chat-tools", status: 0 }, + { elapsedSeconds: 7, name: "bundled-plugin-install-uninstall-0", status: 0 }, + { elapsedSeconds: 6, name: "bundled-plugin-install-uninstall-1", status: 0 }, + { elapsedSeconds: 1, name: "gateway", status: 0 }, + failedLane, + ], + selectedLanes: [ + "openai-chat-tools", + "bundled-plugin-install-uninstall-0", + "bundled-plugin-install-uninstall-1", + "gateway", + "gateway-network", + ], + })}\n`, + "utf8", + ); + return { exitCode: 1, stdout: "", stderr: "scheduler failed\n" }; + }, + }); + + expect(commands).toHaveLength(1); + expect(commands[0]).toMatchObject({ + args: ["scripts/test-docker-all.mjs"], + command: process.execPath, + env: { + OPENCLAW_DOCKER_ALL_FAIL_FAST: "0", + OPENCLAW_DOCKER_ALL_LANES: + "openai-chat-tools,bundled-plugin-install-uninstall,gateway,gateway-network", + OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS: "1800000", + }, + }); + expect(result.results).toMatchObject([ + { scenario: { id: "openai-tools" }, status: "pass" }, + { scenario: { id: "bundled-plugins" }, status: "pass" }, + { scenario: { id: "prefix-lane" }, status: "pass" }, + { scenario: { id: "failing-lane" }, status: "fail" }, + ]); + expect(result.results[3]?.failureMessage).toBe("gateway-network exited with 1"); + }); + it("uses script scenario timeout overrides when running producer commands", async () => { const repoRoot = await makeTempRepo("qa-script-scenario-timeout-"); const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-timeout"); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts index bdb7ff1a3511..31037028d6d6 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.ts @@ -28,6 +28,7 @@ import { type QaScenarioCommandExecution, type QaScenarioCommandResult, } from "./test-file-scenario-command-lifecycle.js"; +import { isDockerE2eScenario, runDockerE2eBatch } from "./test-file-scenario-docker-batch.js"; export type { QaScenarioCommandExecution } from "./test-file-scenario-command-lifecycle.js"; export type QaTestFileScenario = QaSeedScenarioWithSource & { @@ -580,7 +581,37 @@ export async function runQaTestFileScenarios( ...params.env, }; const results: QaTestFileScenarioResult[] = []; + const dockerBatchScenarios = + kind === "script" && !params.failFast ? scenarios.filter(isDockerE2eScenario) : []; + const dockerBatchGroups = new Map(); + for (const scenario of dockerBatchScenarios) { + const scenarioTimeoutMs = resolvePositiveTimerTimeoutMs( + scenario.execution.timeoutMs, + commandTimeoutMs, + ); + const group = dockerBatchGroups.get(scenarioTimeoutMs) ?? []; + group.push(scenario); + dockerBatchGroups.set(scenarioTimeoutMs, group); + } + for (const [scenarioTimeoutMs, group] of dockerBatchGroups) { + // A scheduler invocation shares one fallback lane timeout, so timeout overrides + // stay in separate batches instead of borrowing another scenario's budget. + results.push( + ...(await runDockerE2eBatch({ + commandTimeoutMs: scenarioTimeoutMs, + env, + outputDir: params.outputDir, + repoRoot: params.repoRoot, + runCommand, + scenarios: group, + })), + ); + } + const dockerBatchScenarioIds = new Set(dockerBatchScenarios.map((scenario) => scenario.id)); for (const scenario of scenarios) { + if (dockerBatchScenarioIds.has(scenario.id)) { + continue; + } const result = await runQaTestFileScenario({ env, commandTimeoutMs, @@ -594,6 +625,11 @@ export async function runQaTestFileScenarios( break; } } + const scenarioOrder = new Map(scenarios.map((scenario, index) => [scenario, index])); + results.sort( + (left, right) => + (scenarioOrder.get(left.scenario) ?? 0) - (scenarioOrder.get(right.scenario) ?? 0), + ); const generatedAt = new Date().toISOString(); const artifactPaths = buildScenarioArtifactPaths({ repoRoot: params.repoRoot,