fix(qa): keep runtime parity lanes reproducible

extensions/qa-lab/src/cli.runtime.test.ts

@@ -1530,7 +1530,6 @@ describe("qa cli runtime", () => {
       scenarioIds: [
         "runtime-long-context-cache-stability",
         "runtime-soak-100-turn",
-        "runtime-tool-image-generate",
         "runtime-tool-memory-add",
         "runtime-tool-memory-recall",
         "runtime-tool-message-tool",
@@ -1540,6 +1539,76 @@ describe("qa cli runtime", () => {
         "runtime-tool-tts",
       ],
     });
+    expectWriteContains(
+      stderrWrite,
+      "excluded lane-incompatible scenario(s): runtime-tool-image-generate",
+    );
+  });
+
+  it("keeps runtime-pair tier selection on flow scenarios and reports exclusions", async () => {
+    await runQaSuiteCommand({
+      repoRoot: "/tmp/openclaw-repo",
+      runtimePair: "openclaw,codex",
+      runtimeParityTier: ["standard", "live-only"],
+    });
+
+    const scenarioIds = mockFirstObjectArg(runQaSuite).scenarioIds as string[];
+    expect(scenarioIds).toContain("runtime-first-hour-20-turn");
+    expect(scenarioIds).toContain("streaming-final-integrity");
+    expect(scenarioIds).not.toContain("gateway-restart-inflight-run");
+    expect(scenarioIds).not.toContain("hosted-image-generation-providers-live");
+    expect(scenarioIds).not.toContain("hosted-video-generation-providers-live");
+    expectFields(mockFirstObjectArg(runQaSuite), {
+      runtimePair: ["openclaw", "codex"],
+    });
+    expectWriteContains(
+      stderrWrite,
+      "excluded incompatible non-flow scenario(s): hosted-image-generation-providers-live (script), hosted-video-generation-providers-live (script)",
+    );
+    expectWriteContains(
+      stderrWrite,
+      "excluded lane-incompatible scenario(s): gateway-restart-inflight-run",
+    );
+  });
+
+  it("rejects explicit runtime-pair scenarios with no compatible flow execution", async () => {
+    await expect(
+      runQaSuiteCommand({
+        repoRoot: "/tmp/openclaw-repo",
+        runtimePair: "openclaw,codex",
+        scenarioIds: ["hosted-image-generation-providers-live"],
+      }),
+    ).rejects.toThrow(
+      "--runtime-pair requires execution.kind: flow scenarios; unsupported scenario(s): hosted-image-generation-providers-live (script)",
+    );
+
+    expect(runQaSuite).not.toHaveBeenCalled();
+  });
+
+  it("rejects runtime-pair tiers with no compatible flow scenarios", async () => {
+    const catalog = readQaScenarioPack();
+    const hostedImageScenario = catalog.scenarios.find(
+      (scenario) => scenario.id === "hosted-image-generation-providers-live",
+    );
+    if (!hostedImageScenario) {
+      throw new Error("missing hosted image scenario fixture");
+    }
+    readQaScenarioPack.mockReturnValueOnce({
+      ...catalog,
+      scenarios: [hostedImageScenario],
+    });
+
+    await expect(
+      runQaSuiteCommand({
+        repoRoot: "/tmp/openclaw-repo",
+        runtimePair: "openclaw,codex",
+        runtimeParityTier: ["live-only"],
+      }),
+    ).rejects.toThrow(
+      "--runtime-parity-tier matched no execution.kind: flow scenarios for live-only; incompatible scenario(s): hosted-image-generation-providers-live (script).",
+    );
+
+    expect(runQaSuite).not.toHaveBeenCalled();
   });
 
   it("rejects unknown runtime parity tier filters", async () => {

extensions/qa-lab/src/cli.runtime.ts

@@ -312,34 +312,85 @@ function parseQaRuntimeParityTierFilters(input: string[] | undefined): QaRuntime
 }
 
 function resolveQaRuntimeParityTierScenarioIds(params: {
+  channelDriver?: QaScorecardChannelDriver | null;
+  claudeCliAuthMode?: QaCliBackendAuthMode;
+  primaryModel: string;
+  providerMode: QaProviderMode;
   scenarioIds: string[];
   runtimeParityTiers: readonly QaRuntimeParityTier[];
-}): string[] {
+  runtimePair: boolean;
+}): {
+  scenarioIds: string[];
+  excludedLaneScenarios: string[];
+  excludedNonFlowScenarios: string[];
+} {
   if (params.runtimeParityTiers.length === 0) {
-    return params.scenarioIds;
+    return {
+      scenarioIds: params.scenarioIds,
+      excludedLaneScenarios: [],
+      excludedNonFlowScenarios: [],
+    };
   }
   const tierSet = new Set(params.runtimeParityTiers);
-  const matchingScenarioIds = readQaScenarioPack()
-    .scenarios.filter(
-      (scenario) => scenario.runtimeParityTier && tierSet.has(scenario.runtimeParityTier),
-    )
-    .map((scenario) => scenario.id);
-  if (matchingScenarioIds.length === 0) {
+  const matchingScenarios = readQaScenarioPack().scenarios.filter(
+    (scenario) => scenario.runtimeParityTier && tierSet.has(scenario.runtimeParityTier),
+  );
+  if (matchingScenarios.length === 0) {
     throw new Error(
       `--runtime-parity-tier matched no scenarios for ${params.runtimeParityTiers.join(", ")}.`,
     );
   }
-  return uniqueStrings([...params.scenarioIds, ...matchingScenarioIds]);
+  const compatibleScenarios = params.runtimePair
+    ? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow")
+    : matchingScenarios;
+  const laneCompatibleScenarios = compatibleScenarios.filter((scenario) =>
+    scenarioMatchesQaProviderLane({
+      scenario,
+      providerMode: params.providerMode,
+      primaryModel: params.primaryModel,
+      channelDriver: params.channelDriver,
+      claudeCliAuthMode: params.claudeCliAuthMode,
+    }),
+  );
+  const excludedLaneScenarios = compatibleScenarios
+    .filter((scenario) => !laneCompatibleScenarios.includes(scenario))
+    .map((scenario) => scenario.id);
+  const excludedNonFlowScenarios = params.runtimePair
+    ? matchingScenarios
+        .filter((scenario) => scenario.execution.kind !== "flow")
+        .map((scenario) => `${scenario.id} (${scenario.execution.kind})`)
+    : [];
+  if (compatibleScenarios.length === 0) {
+    throw new Error(
+      `--runtime-parity-tier matched no execution.kind: flow scenarios for ${params.runtimeParityTiers.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.join(", ")}.`,
+    );
+  }
+  if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) {
+    throw new Error(
+      `--runtime-parity-tier matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.join(", ")}.`,
+    );
+  }
+  return {
+    scenarioIds: uniqueStrings([
+      ...params.scenarioIds,
+      ...laneCompatibleScenarios.map((scenario) => scenario.id),
+    ]),
+    excludedLaneScenarios,
+    excludedNonFlowScenarios,
+  };
 }
 
-function rejectNonFlowScenarioIdsForMultipass(scenarioIds: readonly string[]) {
-  if (scenarioIds.length === 0) {
+function rejectNonFlowScenarioIds(params: {
+  option: "--runner multipass" | "--runtime-pair";
+  scenarioIds: readonly string[];
+}) {
+  if (params.scenarioIds.length === 0) {
     return;
   }
   const scenarioById = new Map(
     readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]),
   );
-  const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => {
+  const nonFlowScenarios = params.scenarioIds.flatMap((scenarioId) => {
     const scenario = scenarioById.get(scenarioId);
     return scenario && scenario.execution.kind !== "flow"
       ? [`${scenario.id} (${scenario.execution.kind})`]
@@ -347,7 +398,7 @@ function rejectNonFlowScenarioIdsForMultipass(scenarioIds: readonly string[]) {
   });
   if (nonFlowScenarios.length > 0) {
     throw new Error(
-      `--runner multipass requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`,
+      `${params.option} requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`,
     );
   }
 }
@@ -881,6 +932,12 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
   const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
   const transportId = normalizeQaTransportId(opts.transportId);
   const runner = (opts.runner ?? "host").trim().toLowerCase();
+  const runtimePair = parseQaRuntimePair(opts.runtimePair);
+  const providerMode = normalizeQaProviderMode(opts.providerMode);
+  const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
+  const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
+  const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
+  const channelDriver = normalizeQaSuiteChannelDriver(opts.channelDriver);
   const explicitScenarioIds = resolveQaScenarioPackScenarioIds({
     pack: opts.pack,
     scenarioIds: resolveQaParityPackScenarioIds({
@@ -889,17 +946,30 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
     }),
   });
   const runtimeParityTiers = parseQaRuntimeParityTierFilters(opts.runtimeParityTier);
-  const scenarioIds = resolveQaRuntimeParityTierScenarioIds({
+  const runtimeParityTierSelection = resolveQaRuntimeParityTierScenarioIds({
+    channelDriver,
+    claudeCliAuthMode,
+    primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
+    providerMode,
     scenarioIds: explicitScenarioIds,
     runtimeParityTiers,
+    runtimePair: runtimePair !== undefined,
   });
+  const scenarioIds = runtimeParityTierSelection.scenarioIds;
+  if (runtimePair) {
+    rejectNonFlowScenarioIds({ option: "--runtime-pair", scenarioIds });
+  }
+  if (runtimeParityTierSelection.excludedNonFlowScenarios.length > 0) {
+    process.stderr.write(
+      `QA runtime-pair tier selection excluded incompatible non-flow scenario(s): ${runtimeParityTierSelection.excludedNonFlowScenarios.join(", ")}\n`,
+    );
+  }
+  if (runtimeParityTierSelection.excludedLaneScenarios.length > 0) {
+    process.stderr.write(
+      `QA runtime-pair tier selection excluded lane-incompatible scenario(s): ${runtimeParityTierSelection.excludedLaneScenarios.join(", ")}\n`,
+    );
+  }
   const allowFailures = opts.allowFailures === true;
-  const providerMode = normalizeQaProviderMode(opts.providerMode);
-  const runtimePair = parseQaRuntimePair(opts.runtimePair);
-  const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
-  const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
-  const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
-  const channelDriver = normalizeQaSuiteChannelDriver(opts.channelDriver);
   if (opts.channel?.trim() && channelDriver !== "crabline" && channelDriver !== "live") {
     throw new Error("--channel override requires --channel-driver crabline or live.");
   }
@@ -959,7 +1029,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
     throw new Error("--runtime-pair is not supported with a live QA adapter.");
   }
   if (runner === "multipass") {
-    rejectNonFlowScenarioIdsForMultipass(scenarioIds);
+    rejectNonFlowScenarioIds({ option: "--runner multipass", scenarioIds });
     const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
     const result = await runQaMultipass({
       repoRoot,

extensions/qa-lab/src/gateway-child.ts

@@ -58,6 +58,7 @@ import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js";
 import { seedQaAgentWorkspace } from "./qa-agent-workspace.js";
 import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js";
 import type { QaTransportAdapter } from "./qa-transport.js";
+import type { RuntimeId } from "./runtime-parity.js";
 import { resolveQaWindowsSystem32ExePath } from "./windows-system-tools.js";
 
 export type { QaCliBackendAuthMode } from "./providers/env.js";
@@ -780,6 +781,7 @@ export async function startQaGatewayChild(params: {
   alternateModel?: string;
   fastMode?: boolean;
   thinkingDefault?: QaThinkingLevel;
+  forcedRuntime?: RuntimeId;
   claudeCliAuthMode?: QaCliBackendAuthMode;
   controlUiEnabled?: boolean;
   enabledPluginIds?: string[];

extensions/qa-lab/src/qa-gateway-config.ts

@@ -12,6 +12,7 @@ import { getQaProvider } from "./providers/index.js";
 import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
 import type { QaThinkingLevel } from "./qa-thinking.js";
 import type { QaTransportGatewayConfig } from "./qa-transport.js";
+import type { RuntimeId } from "./runtime-parity.js";
 
 export { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-thinking.js";
 
@@ -68,6 +69,7 @@ export function buildQaGatewayConfig(params: {
   liveProviderConfigs?: Record<string, ModelProviderConfig>;
   fastMode?: boolean;
   thinkingDefault?: QaThinkingLevel;
+  forcedRuntime?: RuntimeId;
 }): OpenClawConfig {
   const providerBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1";
   const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
@@ -135,12 +137,20 @@ export function buildQaGatewayConfig(params: {
       ...transportPluginIds,
     ]),
   ];
-  const resolveModelParams = (modelRef: string) =>
-    provider.resolveModelParams({
-      modelRef,
-      fastMode: params.fastMode,
-      thinkingDefault: params.thinkingDefault,
-    });
+  const resolveModelEntry = (modelRef: string) => {
+    // Codex owns its app-server transport. OpenClaw provider params would make
+    // the forced parity cell an authored route that Codex correctly rejects.
+    if (params.forcedRuntime === "codex") {
+      return {};
+    }
+    return {
+      params: provider.resolveModelParams({
+        modelRef,
+        fastMode: params.fastMode,
+        thinkingDefault: params.thinkingDefault,
+      }),
+    };
+  };
   const allowedOrigins = mergeQaControlUiAllowedOrigins(params.controlUiAllowedOrigins);
   const providerGatewayModels = provider.buildGatewayModels({
     providerBaseUrl,
@@ -210,12 +220,8 @@ export function buildQaGatewayConfig(params: {
           },
         },
         models: {
-          [primaryModel]: {
-            params: resolveModelParams(primaryModel),
-          },
-          [alternateModel]: {
-            params: resolveModelParams(alternateModel),
-          },
+          [primaryModel]: resolveModelEntry(primaryModel),
+          [alternateModel]: resolveModelEntry(alternateModel),
         },
         subagents: {
           allowAgents: ["*"],
@@ -227,6 +233,9 @@ export function buildQaGatewayConfig(params: {
           id: "qa",
           default: true,
           model: buildQaModelSelection(primaryModel, alternateModel),
+          ...(params.forcedRuntime === "codex" && params.fastMode !== undefined
+            ? { fastModeDefault: params.fastMode }
+            : {}),
           identity: {
             name: "C-3PO QA",
             theme: "Flustered Protocol Droid",

extensions/qa-lab/src/suite.ts

@@ -1649,6 +1649,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
       alternateModel,
       fastMode,
       thinkingDefault: params?.thinkingDefault,
+      forcedRuntime: params?.forcedRuntime,
       claudeCliAuthMode: params?.claudeCliAuthMode,
       controlUiEnabled: params?.controlUiEnabled ?? true,
       enabledPluginIds,