fix(qa): run maturity Docker scenarios through the aggregate scheduler (#114447)

* fix(qa): batch Docker scorecard scenarios

* fix(qa): keep Docker batch types private

extensions/qa-lab/src/test-file-scenario-docker-batch.ts

@@ -0,0 +1,161 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
+import { shellQuote } from "./shell-quote.js";
+import type {
+  QaScenarioCommandExecution,
+  QaScenarioCommandResult,
+} from "./test-file-scenario-command-lifecycle.js";
+
+const QA_DOCKER_E2E_LANE_SCRIPT = "test/e2e/qa-lab/runtime/docker-e2e-lane.ts";
+
+type QaDockerScenario = QaSeedScenarioWithSource & {
+  execution: Extract<QaSeedScenarioWithSource["execution"], { kind: "script" }>;
+};
+
+type QaDockerBatchResult = {
+  durationMs: number;
+  failureMessage?: string;
+  logPath: string;
+  scenario: QaDockerScenario;
+  status: "fail" | "pass";
+};
+
+export function dockerE2eLaneName(scenario: QaSeedScenarioWithSource) {
+  const args = scenario.execution.kind === "script" ? scenario.execution.args : undefined;
+  if (
+    scenario.execution.kind !== "script" ||
+    scenario.execution.path !== QA_DOCKER_E2E_LANE_SCRIPT ||
+    args?.length !== 2 ||
+    args[0] !== "--lane"
+  ) {
+    return undefined;
+  }
+  const laneName = args[1]?.trim();
+  return laneName || undefined;
+}
+
+export function isDockerE2eScenario(
+  scenario: QaSeedScenarioWithSource,
+): scenario is QaDockerScenario {
+  return dockerE2eLaneName(scenario) !== undefined;
+}
+
+function laneMatches(
+  selectedLane: string,
+  resultLane: string | undefined,
+  resolvedLaneNames: readonly string[],
+) {
+  // The scheduler summary records aliases as their resolved lanes. Prefix matching is
+  // safe only when the selected name itself disappeared during that resolution.
+  return (
+    resultLane === selectedLane ||
+    (!resolvedLaneNames.includes(selectedLane) &&
+      resultLane !== undefined &&
+      resolvedLaneNames.includes(resultLane) &&
+      resultLane.startsWith(`${selectedLane}-`))
+  );
+}
+
+export async function runDockerE2eBatch(params: {
+  commandTimeoutMs: number;
+  env: NodeJS.ProcessEnv;
+  outputDir: string;
+  repoRoot: string;
+  runCommand: (command: QaScenarioCommandExecution) => Promise<QaScenarioCommandResult>;
+  scenarios: readonly QaDockerScenario[];
+}): Promise<QaDockerBatchResult[]> {
+  const selected = params.scenarios.map((scenario) => ({
+    lane: dockerE2eLaneName(scenario)!,
+    scenario,
+  }));
+  const laneNames = [...new Set(selected.map(({ lane }) => lane))];
+  const batchId = `${params.commandTimeoutMs}ms`;
+  const dockerOutputDir = path.join(params.outputDir, `docker-e2e-${batchId}`);
+  const logPath = path.join(params.outputDir, `docker-e2e-batch-${batchId}.log`);
+  await fs.mkdir(dockerOutputDir, { recursive: true });
+  const summaryPath = path.join(dockerOutputDir, "summary.json");
+  await fs.rm(summaryPath, { force: true });
+  let commandResult: QaScenarioCommandResult;
+  try {
+    commandResult = await params.runCommand({
+      command: process.execPath,
+      args: ["scripts/test-docker-all.mjs"],
+      cwd: params.repoRoot,
+      env: {
+        ...params.env,
+        OPENCLAW_DOCKER_ALL_BUILD: "1",
+        OPENCLAW_DOCKER_ALL_FAIL_FAST: "0",
+        OPENCLAW_DOCKER_ALL_LANES: laneNames.join(","),
+        OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS: String(params.commandTimeoutMs),
+        OPENCLAW_DOCKER_ALL_LOG_DIR: dockerOutputDir,
+        OPENCLAW_DOCKER_ALL_PROFILE: "all",
+        OPENCLAW_DOCKER_ALL_TIMINGS_FILE: path.join(dockerOutputDir, "lane-timings.json"),
+      },
+      // The scheduler owns each resolved lane deadline. Parent signals and the
+      // enclosing QA workflow bound the aggregate run without alias-count guesses.
+    });
+  } catch (error) {
+    commandResult = {
+      exitCode: 1,
+      failureMessage: formatErrorMessage(error),
+      stderr: `${formatErrorMessage(error)}\n`,
+      stdout: "",
+    };
+  }
+  await fs.writeFile(
+    logPath,
+    `$ ${shellQuote(process.execPath)} scripts/test-docker-all.mjs\n${commandResult.stdout}${commandResult.stderr}`,
+    "utf8",
+  );
+
+  let summary:
+    | {
+        failures?: Array<{ name?: string }>;
+        lanes?: Array<{ elapsedSeconds?: number; name?: string; status?: number }>;
+        selectedLanes?: string[];
+      }
+    | undefined;
+  try {
+    summary = JSON.parse(await fs.readFile(summaryPath, "utf8"));
+  } catch {
+    // The command-level failure below owns missing or incomplete scheduler output.
+  }
+  const lanes = summary?.lanes ?? [];
+  const failures = summary?.failures ?? [];
+  const resolvedLaneNames = summary?.selectedLanes ?? [];
+  const unexplainedFailure =
+    commandResult.exitCode !== 0 &&
+    (failures.length === 0 ||
+      failures.some(
+        (failure) =>
+          !laneNames.some((laneName) => laneMatches(laneName, failure.name, resolvedLaneNames)),
+      ));
+  return selected.map(({ lane, scenario }) => {
+    const matchingLanes = lanes.filter((result) =>
+      laneMatches(lane, result.name, resolvedLaneNames),
+    );
+    const failedLane = matchingLanes.find((result) => result.status !== 0);
+    const failureMessage = unexplainedFailure
+      ? commandResult.failureMessage || "Docker E2E scheduler failed before reporting lane results"
+      : failedLane
+        ? `${failedLane.name ?? lane} exited with ${String(failedLane.status ?? 1)}`
+        : matchingLanes.length === 0
+          ? `Docker E2E scheduler returned no result for ${lane}`
+          : undefined;
+    const result: QaDockerBatchResult = {
+      durationMs: Math.max(
+        1,
+        ...matchingLanes.map((entry) => Math.max(0, entry.elapsedSeconds ?? 0) * 1000),
+      ),
+      logPath,
+      scenario,
+      status: failureMessage ? "fail" : "pass",
+    };
+    if (failureMessage) {
+      result.failureMessage = failureMessage;
+    }
+    return result;
+  });
+}

extensions/qa-lab/src/test-file-scenario-runner.test.ts

@@ -6,6 +6,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
 import { validateQaEvidenceSummaryJson } from "./evidence-summary.js";
 import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
 import { createTempDirHarness } from "./temp-dir.test-helper.js";
+import { dockerE2eLaneName } from "./test-file-scenario-docker-batch.js";
 import {
   qaTestFileScenarioRunnerTesting,
   runQaTestFileScenarios,
@@ -81,6 +82,35 @@ function makeTestFileScenario(
   };
 }
 
+function makeDockerE2eScenario(id: string, lane: string): QaSeedScenarioWithSource {
+  const scenario = makeTestFileScenario("script", "test/e2e/qa-lab/runtime/docker-e2e-lane.ts");
+  if (scenario.execution.kind !== "script") {
+    throw new Error("expected script scenario");
+  }
+  return {
+    ...scenario,
+    id,
+    execution: {
+      ...scenario.execution,
+      args: ["--lane", lane],
+    },
+  };
+}
+
+it("only batches the canonical Docker lane argument shape", () => {
+  const scenario = makeDockerE2eScenario("docker-lane", "gateway-network");
+  if (scenario.execution.kind !== "script") {
+    throw new Error("expected script scenario");
+  }
+  expect(dockerE2eLaneName(scenario)).toBe("gateway-network");
+  expect(
+    dockerE2eLaneName({
+      ...scenario,
+      execution: { ...scenario.execution, args: ["--lane", "gateway-network", "--extra"] },
+    }),
+  ).toBeUndefined();
+});
+
 async function makeTempRepo(prefix: string) {
   const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
   tempRoots.push(repoRoot);
@@ -514,6 +544,79 @@ describe("qa test file scenario runner", () => {
     });
   });
 
+  it("runs Docker script scenarios through one aggregate scheduler invocation", async () => {
+    const repoRoot = await makeTempRepo("qa-script-docker-batch-");
+    const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "docker-batch");
+    const staleSummaryPath = path.join(outputDir, "docker-e2e-1800000ms", "summary.json");
+    await fs.mkdir(path.dirname(staleSummaryPath), { recursive: true });
+    await fs.writeFile(staleSummaryPath, '{"status":"passed"}\n', "utf8");
+    const commands: QaScenarioCommandExecution[] = [];
+    const scenarios = [
+      makeDockerE2eScenario("openai-tools", "openai-chat-tools"),
+      makeDockerE2eScenario("bundled-plugins", "bundled-plugin-install-uninstall"),
+      makeDockerE2eScenario("prefix-lane", "gateway"),
+      makeDockerE2eScenario("failing-lane", "gateway-network"),
+    ];
+    const result = await runQaTestFileScenarios({
+      repoRoot,
+      outputDir,
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.6-luna",
+      scenarios,
+      runCommand: async (command) => {
+        commands.push(command);
+        await expect(fs.access(staleSummaryPath)).rejects.toThrow();
+        const logDir = command.env.OPENCLAW_DOCKER_ALL_LOG_DIR;
+        if (!logDir) {
+          throw new Error("missing Docker scheduler log dir");
+        }
+        await fs.mkdir(logDir, { recursive: true });
+        const failedLane = { elapsedSeconds: 2, name: "gateway-network", status: 1 };
+        await fs.writeFile(
+          path.join(logDir, "summary.json"),
+          `${JSON.stringify({
+            failures: [failedLane],
+            lanes: [
+              { elapsedSeconds: 4, name: "openai-chat-tools", status: 0 },
+              { elapsedSeconds: 7, name: "bundled-plugin-install-uninstall-0", status: 0 },
+              { elapsedSeconds: 6, name: "bundled-plugin-install-uninstall-1", status: 0 },
+              { elapsedSeconds: 1, name: "gateway", status: 0 },
+              failedLane,
+            ],
+            selectedLanes: [
+              "openai-chat-tools",
+              "bundled-plugin-install-uninstall-0",
+              "bundled-plugin-install-uninstall-1",
+              "gateway",
+              "gateway-network",
+            ],
+          })}\n`,
+          "utf8",
+        );
+        return { exitCode: 1, stdout: "", stderr: "scheduler failed\n" };
+      },
+    });
+
+    expect(commands).toHaveLength(1);
+    expect(commands[0]).toMatchObject({
+      args: ["scripts/test-docker-all.mjs"],
+      command: process.execPath,
+      env: {
+        OPENCLAW_DOCKER_ALL_FAIL_FAST: "0",
+        OPENCLAW_DOCKER_ALL_LANES:
+          "openai-chat-tools,bundled-plugin-install-uninstall,gateway,gateway-network",
+        OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS: "1800000",
+      },
+    });
+    expect(result.results).toMatchObject([
+      { scenario: { id: "openai-tools" }, status: "pass" },
+      { scenario: { id: "bundled-plugins" }, status: "pass" },
+      { scenario: { id: "prefix-lane" }, status: "pass" },
+      { scenario: { id: "failing-lane" }, status: "fail" },
+    ]);
+    expect(result.results[3]?.failureMessage).toBe("gateway-network exited with 1");
+  });
+
   it("uses script scenario timeout overrides when running producer commands", async () => {
     const repoRoot = await makeTempRepo("qa-script-scenario-timeout-");
     const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-timeout");

extensions/qa-lab/src/test-file-scenario-runner.ts

@@ -28,6 +28,7 @@ import {
   type QaScenarioCommandExecution,
   type QaScenarioCommandResult,
 } from "./test-file-scenario-command-lifecycle.js";
+import { isDockerE2eScenario, runDockerE2eBatch } from "./test-file-scenario-docker-batch.js";
 export type { QaScenarioCommandExecution } from "./test-file-scenario-command-lifecycle.js";
 
 export type QaTestFileScenario = QaSeedScenarioWithSource & {
@@ -580,7 +581,37 @@ export async function runQaTestFileScenarios(
     ...params.env,
   };
   const results: QaTestFileScenarioResult[] = [];
+  const dockerBatchScenarios =
+    kind === "script" && !params.failFast ? scenarios.filter(isDockerE2eScenario) : [];
+  const dockerBatchGroups = new Map<number, typeof dockerBatchScenarios>();
+  for (const scenario of dockerBatchScenarios) {
+    const scenarioTimeoutMs = resolvePositiveTimerTimeoutMs(
+      scenario.execution.timeoutMs,
+      commandTimeoutMs,
+    );
+    const group = dockerBatchGroups.get(scenarioTimeoutMs) ?? [];
+    group.push(scenario);
+    dockerBatchGroups.set(scenarioTimeoutMs, group);
+  }
+  for (const [scenarioTimeoutMs, group] of dockerBatchGroups) {
+    // A scheduler invocation shares one fallback lane timeout, so timeout overrides
+    // stay in separate batches instead of borrowing another scenario's budget.
+    results.push(
+      ...(await runDockerE2eBatch({
+        commandTimeoutMs: scenarioTimeoutMs,
+        env,
+        outputDir: params.outputDir,
+        repoRoot: params.repoRoot,
+        runCommand,
+        scenarios: group,
+      })),
+    );
+  }
+  const dockerBatchScenarioIds = new Set(dockerBatchScenarios.map((scenario) => scenario.id));
   for (const scenario of scenarios) {
+    if (dockerBatchScenarioIds.has(scenario.id)) {
+      continue;
+    }
     const result = await runQaTestFileScenario({
       env,
       commandTimeoutMs,
@@ -594,6 +625,11 @@ export async function runQaTestFileScenarios(
       break;
     }
   }
+  const scenarioOrder = new Map(scenarios.map((scenario, index) => [scenario, index]));
+  results.sort(
+    (left, right) =>
+      (scenarioOrder.get(left.scenario) ?? 0) - (scenarioOrder.get(right.scenario) ?? 0),
+  );
   const generatedAt = new Date().toISOString();
   const artifactPaths = buildScenarioArtifactPaths({
     repoRoot: params.repoRoot,