diff --git a/extensions/qa-lab/src/agentic-parity-report.test.ts b/extensions/qa-lab/src/agentic-parity-report.test.ts index 9fc206a118e8..666091ff542c 100644 --- a/extensions/qa-lab/src/agentic-parity-report.test.ts +++ b/extensions/qa-lab/src/agentic-parity-report.test.ts @@ -55,6 +55,7 @@ function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }], finalText: "done", @@ -64,6 +65,7 @@ function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }], finalText: "done", @@ -85,6 +87,7 @@ function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }], finalText: "done", @@ -94,6 +97,7 @@ function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [{ tool: "read_file", argsHash: "b", resultHash: "r" }], finalText: "done", @@ -202,6 +206,7 @@ describe("qa agentic parity report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [], finalText: "done", @@ -211,6 +216,7 @@ describe("qa agentic parity report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [], finalText: "done", diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 6db70c841417..c43fbd2e61eb 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -1916,6 +1916,7 @@ describe("qa cli runtime", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [{ tool: "read_file", argsHash: "a", resultHash: "r" }], finalText: "done", @@ -1925,6 +1926,7 @@ describe("qa cli runtime", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [{ tool: "read_file", argsHash: "b", resultHash: "r" }], finalText: "done", @@ -1986,6 +1988,7 @@ describe("qa cli runtime", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: [{ tool: "fs.read", argsHash: "a", resultHash: "r" }], finalText: "done", @@ -1995,6 +1998,7 @@ describe("qa cli runtime", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: '{"role":"assistant"}\n', toolCalls: Array.from({ length: 40 }, (_, index) => ({ tool: "fs.read", @@ -2162,6 +2166,7 @@ describe("qa cli runtime", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r" }], finalText: "", @@ -2171,6 +2176,7 @@ describe("qa cli runtime", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [], finalText: "", @@ -2195,7 +2201,10 @@ describe("qa cli runtime", () => { expect(process.exitCode).toBe(1); expectWriteContains(stdoutWrite, "- Verdict: fail"); - expectWriteContains(stdoutWrite, "web_search missing codex tool call web_search"); + expectWriteContains( + stdoutWrite, + "web_search missing successful codex tool call/result web_search", + ); } finally { process.exitCode = priorExitCode; await fs.rm(repoRoot, { recursive: true, force: true }); diff --git a/extensions/qa-lab/src/confidence-report.test.ts b/extensions/qa-lab/src/confidence-report.test.ts index 95e56b85ef1c..416ea67a0bc3 100644 --- a/extensions/qa-lab/src/confidence-report.test.ts +++ b/extensions/qa-lab/src/confidence-report.test.ts @@ -544,8 +544,9 @@ describe("qa confidence report", () => { scenarioId: "plugin-hook-health-sentinel", drift: "none", cells: { - pi: { sentinelFindings: [] }, + openclaw: { status: "pass", sentinelFindings: [] }, codex: { + status: "pass", sentinelFindings: [ { kind: "plugin-hook-failure", diff --git a/extensions/qa-lab/src/confidence-report.ts b/extensions/qa-lab/src/confidence-report.ts index 2537a77827ce..b2c7e5e48bcb 100644 --- a/extensions/qa-lab/src/confidence-report.ts +++ b/extensions/qa-lab/src/confidence-report.ts @@ -1028,7 +1028,7 @@ async function detectRuntimeDrift(params: { const result = await runRuntimeParityScenario({ scenarioId: params.scenarioId, runCell: async (runtime) => ({ - scenarioStatus: "pass", + status: "pass", cell: runtime === "openclaw" ? params.openclaw : params.codex, }), }); @@ -1100,7 +1100,10 @@ function detectTokenEfficiencyRegression(): boolean { }); const runtimeParity: RuntimeParityResult = { scenarioId: "token-efficiency-regression", - cells: { openclaw, codex }, + cells: { + openclaw: { ...openclaw, status: "pass" }, + codex: { ...codex, status: "pass" }, + }, drift: "none", }; const report = buildTokenEfficiencyReport({ diff --git a/extensions/qa-lab/src/jsonl-replay.test.ts b/extensions/qa-lab/src/jsonl-replay.test.ts index edb2d79f0c7b..56f02493bdd4 100644 --- a/extensions/qa-lab/src/jsonl-replay.test.ts +++ b/extensions/qa-lab/src/jsonl-replay.test.ts @@ -121,7 +121,7 @@ describe("jsonl replay", () => { const runCell: JsonlReplayCellRunner = async ({ runtime, turn }) => { if (turn.turn === 2) { return { - scenarioStatus: "pass", + status: "pass", cell: makeCell(runtime, { toolCalls: [makeToolCall(runtime === "openclaw" ? {} : { argsHash: "args-codex" })], }), @@ -129,14 +129,14 @@ describe("jsonl replay", () => { } if (turn.turn === 3) { return { - scenarioStatus: "pass", + status: "pass", cell: makeCell(runtime, { finalText: runtime === "openclaw" ? "openclaw wording" : "codex wording", }), }; } return { - scenarioStatus: "pass", + status: "pass", cell: makeCell(runtime), }; }; diff --git a/extensions/qa-lab/src/jsonl-replay.ts b/extensions/qa-lab/src/jsonl-replay.ts index f6ecd9638119..a0c9daa47bd4 100644 --- a/extensions/qa-lab/src/jsonl-replay.ts +++ b/extensions/qa-lab/src/jsonl-replay.ts @@ -171,7 +171,7 @@ function assertSupportedRuntimePair(runtimePair: JsonlReplayInput["runtimePair"] export function createMockJsonlReplayCellRunner(): JsonlReplayCellRunner { return async ({ runtime, turn }) => ({ - scenarioStatus: "pass", + status: "pass", cell: { runtime, transcriptBytes: turn.transcriptPrefix, diff --git a/extensions/qa-lab/src/runtime-parity-outcomes.test.ts b/extensions/qa-lab/src/runtime-parity-outcomes.test.ts new file mode 100644 index 000000000000..06bc3cb5fa8a --- /dev/null +++ b/extensions/qa-lab/src/runtime-parity-outcomes.test.ts @@ -0,0 +1,111 @@ +// Qa Lab tests cover runtime parity outcome precedence and skip preservation. +import { describe, expect, it } from "vitest"; +import { + captureRuntimeParityCell, + isRuntimeParityResultPass, + runRuntimeParityScenario, + type RuntimeId, + type RuntimeParityCell, +} from "./runtime-parity.js"; + +function makeRuntimeParityCell(runtime: RuntimeId): RuntimeParityCell { + return { + runtime, + transcriptBytes: '{"message":{"role":"assistant","content":"done"}}\n', + toolCalls: [], + finalText: "done", + usage: { + inputTokens: 1, + outputTokens: 1, + totalTokens: 2, + }, + wallClockMs: 10, + bootStateLines: [], + }; +} + +describe("runtime parity outcomes", () => { + it("keeps a skip diagnostic out of runtime error classification", async () => { + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { + tempRoot: `/tmp/openclaw-qa-runtime-parity-missing-${process.pid}`, + }, + scenarioResult: { + status: "skip", + details: "expected-unavailable tool: this fixture is report-only", + }, + wallClockMs: 10, + }); + + expect(cell.runtimeErrorClass).toBeUndefined(); + }); + + it("does not claim both canonical runtime-pair cells failed for a one-sided failure", async () => { + const result = await runRuntimeParityScenario({ + scenarioId: "one-sided-cell-failure", + runCell: async (runtime) => ({ + status: "pass", + cell: { + ...makeRuntimeParityCell(runtime), + ...(runtime === "codex" ? { runtimeErrorClass: "scenario-failure" } : {}), + }, + }), + }); + + expect(result).toMatchObject({ + drift: "failure-mode", + driftDetails: "at least one runtime failed", + }); + }); + + it("preserves skipped cell outcomes instead of promoting matching empty evidence", async () => { + const result = await runRuntimeParityScenario({ + scenarioId: "skipped-cells", + runCell: async (runtime) => ({ + status: "skip", + details: "implementation unavailable", + cell: makeRuntimeParityCell(runtime), + }), + }); + + expect(result).toMatchObject({ + cells: { + openclaw: { status: "skip", details: "implementation unavailable" }, + codex: { status: "skip", details: "implementation unavailable" }, + }, + drift: "failure-mode", + driftDetails: "both canonical runtime-pair cells skipped", + }); + expect(isRuntimeParityResultPass(result)).toBe(false); + }); + + it("requires every canonical runtime-pair cell status to pass", async () => { + const result = await runRuntimeParityScenario({ + scenarioId: "complete-result-cells", + runCell: async (runtime) => ({ + status: "pass", + cell: makeRuntimeParityCell(runtime), + }), + }); + + expect(isRuntimeParityResultPass(result)).toBe(true); + + expect( + isRuntimeParityResultPass({ + ...result, + cells: { + ...result.cells, + codex: { ...result.cells.codex, status: "skip" }, + }, + }), + ).toBe(false); + + expect( + isRuntimeParityResultPass({ + ...result, + cells: { openclaw: result.cells.openclaw } as typeof result.cells, + }), + ).toBe(false); + }); +}); diff --git a/extensions/qa-lab/src/runtime-parity-session-selection.test.ts b/extensions/qa-lab/src/runtime-parity-session-selection.test.ts index 43ba26cab10d..833e5a45d037 100644 --- a/extensions/qa-lab/src/runtime-parity-session-selection.test.ts +++ b/extensions/qa-lab/src/runtime-parity-session-selection.test.ts @@ -125,4 +125,63 @@ describe("runtime parity session selection", () => { expect(cell.toolCalls).toEqual([expect.objectContaining({ tool: "web_fetch" })]); }); + + it("retains parent sessions_spawn evidence when the spawned child is newer", async () => { + const now = Date.now(); + const parentSessionKey = "agent:qa:runtime-tool:sessions_spawn:happy"; + const tempRoot = await seedSession({ + sessionId: "sessions-spawn-parent", + sessionKey: parentSessionKey, + messages: [{ role: "user", content: "tool search qa check target=sessions_spawn" }], + updatedAt: now - 1_000, + trajectoryEvents: [ + { + type: "tool.call", + data: { + toolCallId: "sessions-spawn-1", + name: "sessions_spawn", + arguments: { task: "reply exactly RUNTIME-TOOL-FIXTURE", mode: "run" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "sessions-spawn-1", + name: "sessions_spawn", + status: "completed", + success: true, + result: { status: "accepted", childSessionKey: "agent:qa:subagent:child" }, + }, + }, + ], + }); + await seedSession({ + tempRoot, + sessionId: "sessions-spawn-child", + sessionKey: "agent:qa:subagent:child", + parentSessionKey, + messages: [{ role: "assistant", content: "RUNTIME-TOOL-FIXTURE" }], + updatedAt: now, + }); + + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { tempRoot }, + scenarioResult: { + status: "pass", + details: `RUNTIME_PARITY_SESSION_KEY=${parentSessionKey}`, + }, + wallClockMs: 10, + }); + + expect(cell.transcriptBytes).toContain("target=sessions_spawn"); + expect(cell.transcriptBytes).not.toContain("RUNTIME-TOOL-FIXTURE"); + expect(cell.toolCalls).toEqual([ + expect.objectContaining({ + tool: "sessions_spawn", + }), + ]); + expect(cell.toolCalls[0]).not.toHaveProperty("errorClass"); + expect(cell.toolCalls[0]?.resultHash).not.toBe(""); + }); }); diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts index fc6bb12dc145..9c2828fbbeec 100644 --- a/extensions/qa-lab/src/runtime-parity.test.ts +++ b/extensions/qa-lab/src/runtime-parity.test.ts @@ -644,7 +644,7 @@ describe("runtime parity", () => { const result = await runRuntimeParityScenario({ scenarioId: "resolved-tool", runCell: async (runtime) => ({ - scenarioStatus: "pass", + status: "pass", cell: { ...cell, runtime }, }), }); @@ -663,7 +663,7 @@ describe("runtime parity", () => { reason: " Local fixture only; no assistant turn runs. ", }, runCell: async (runtime) => ({ - scenarioStatus: "pass", + status: "pass", cell: makeRuntimeParityCell(runtime, []), }), }); @@ -691,7 +691,7 @@ describe("runtime parity", () => { const result = await runRuntimeParityScenario({ scenarioId: "planned-only-tool", runCell: async (runtime) => ({ - scenarioStatus: "pass", + status: "pass", cell: { ...cell, runtime }, }), }); @@ -704,7 +704,7 @@ describe("runtime parity", () => { const result = await runRuntimeParityScenario({ scenarioId: "matching-tool-errors", runCell: async (runtime) => ({ - scenarioStatus: "pass", + status: "pass", cell: { ...makeRuntimeParityCell(runtime, [ { @@ -727,7 +727,7 @@ describe("runtime parity", () => { const result = await runRuntimeParityScenario({ scenarioId: "failed-cell-with-drift", runCell: async (runtime) => ({ - scenarioStatus: runtime === "codex" ? "fail" : "pass", + status: runtime === "codex" ? "fail" : "pass", cell: makeRuntimeParityCell(runtime, [ { tool: "web_search", @@ -740,7 +740,7 @@ describe("runtime parity", () => { expect(result).toMatchObject({ drift: "failure-mode", - driftDetails: "scenario status differs (pass vs fail)", + driftDetails: "runtime-pair cell status differs (pass vs fail)", }); expect(isRuntimeParityResultPass(result)).toBe(false); }); diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index b3c70073fce1..558d35d352be 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -22,8 +22,14 @@ import { discardIgnoredResponseBody } from "./ignored-response-body.js"; import * as parity from "./parity-shared.js"; import { readRawQaSessionStore } from "./suite-runtime-agent-session.js"; +// These are the canonical QA comparison cells, not the extensible product +// AgentHarness registry. Broader harness coverage needs its own explicit lane. export type RuntimeId = "openclaw" | "codex"; +type RuntimeParityStatus = "pass" | "fail" | "skip"; + +const CANONICAL_RUNTIME_IDS = ["openclaw", "codex"] as const satisfies readonly RuntimeId[]; + export type RuntimeParityToolCall = { tool: string; argsHash: string; @@ -57,6 +63,11 @@ export type RuntimeParityCell = { sentinelFindings?: GatewayLogSentinelFinding[]; }; +type RuntimeParityResultCell = RuntimeParityCell & { + status: RuntimeParityStatus; + details?: string; +}; + export type RuntimeParityDrift = | "none" | "text-only" @@ -68,10 +79,7 @@ export type RuntimeParityDrift = export type RuntimeParityResult = { scenarioId: string; runtimeParityUsage?: RuntimeParityUsagePolicy; - cells: { - openclaw: RuntimeParityCell; - codex: RuntimeParityCell; - }; + cells: Record; drift: RuntimeParityDrift; driftDetails?: string; }; @@ -93,8 +101,8 @@ export function resolveRuntimeParityUsagePolicy(value: unknown): RuntimeParityUs } export type RuntimeParityScenarioExecution = { - scenarioStatus: "pass" | "fail"; - scenarioDetails?: string; + status: RuntimeParityStatus; + details?: string; cell: RuntimeParityCell; }; @@ -110,8 +118,10 @@ export function runtimeParityCellStatus( export function isRuntimeParityResultPass(result: RuntimeParityResult) { return ( result.drift !== "failure-mode" && - isRuntimeParityCellPassable(result.cells.openclaw) && - isRuntimeParityCellPassable(result.cells.codex) + CANONICAL_RUNTIME_IDS.every((runtime) => { + const cell = result.cells[runtime]; + return cell?.status === "pass" && isRuntimeParityCellPassable(cell); + }) ); } @@ -1099,8 +1109,8 @@ function summarizeSentinelErrorClass(findings: readonly GatewayLogSentinelFindin function classifyRuntimeParityCells(params: { openclaw: RuntimeParityCell; codex: RuntimeParityCell; - openclawScenarioStatus: "pass" | "fail"; - codexScenarioStatus: "pass" | "fail"; + openclawStatus: RuntimeParityStatus; + codexStatus: RuntimeParityStatus; }): Pick { if ( isHardFailureRuntimeError(params.openclaw.runtimeErrorClass) || @@ -1128,17 +1138,21 @@ function classifyRuntimeParityCells(params: { } if ( - params.openclawScenarioStatus === "fail" || - params.codexScenarioStatus === "fail" || + params.openclawStatus !== "pass" || + params.codexStatus !== "pass" || !isRuntimeParityCellPassable(params.openclaw) || !isRuntimeParityCellPassable(params.codex) ) { return { drift: "failure-mode", driftDetails: - params.openclawScenarioStatus === params.codexScenarioStatus - ? "at least one runtime failed" - : `scenario status differs (${params.openclawScenarioStatus} vs ${params.codexScenarioStatus})`, + params.openclawStatus === params.codexStatus + ? params.openclawStatus === "skip" + ? "both canonical runtime-pair cells skipped" + : params.openclawStatus === "fail" + ? "both canonical runtime-pair cells failed" + : "at least one runtime failed" + : `runtime-pair cell status differs (${params.openclawStatus} vs ${params.codexStatus})`, }; } @@ -1398,9 +1412,9 @@ export async function captureRuntimeParityCell( // Retry passes retain first-attempt diagnostics; only terminal failures may // classify that historical text as the cell's runtime error. const scenarioErrorClass = - params.scenarioResult.status === "pass" - ? undefined - : classifyScenarioError(params.scenarioResult.details); + params.scenarioResult.status === "fail" + ? classifyScenarioError(params.scenarioResult.details) + : undefined; const sentinelErrorClass = summarizeSentinelErrorClass(sentinelFindings); const terminalImageResultProven = hasProvenTerminalImageResult(params.scenarioResult); return { @@ -1432,15 +1446,23 @@ export async function runRuntimeParityScenario(params: { const drift = classifyRuntimeParityCells({ openclaw: openclaw.cell, codex: codex.cell, - openclawScenarioStatus: openclaw.scenarioStatus, - codexScenarioStatus: codex.scenarioStatus, + openclawStatus: openclaw.status, + codexStatus: codex.status, }); return { scenarioId: params.scenarioId, runtimeParityUsage: resolveRuntimeParityUsagePolicy(params.runtimeParityUsage), cells: { - openclaw: openclaw.cell, - codex: codex.cell, + openclaw: { + ...openclaw.cell, + status: openclaw.status, + ...(openclaw.details ? { details: openclaw.details } : {}), + }, + codex: { + ...codex.cell, + status: codex.status, + ...(codex.details ? { details: codex.details } : {}), + }, }, drift: drift.drift, ...(drift.driftDetails ? { driftDetails: drift.driftDetails } : {}), diff --git a/extensions/qa-lab/src/runtime-tool-fixture.test.ts b/extensions/qa-lab/src/runtime-tool-fixture.test.ts index 97ca88c58639..54e3aa78b6f4 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.test.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.test.ts @@ -324,7 +324,7 @@ describe("runtime tool fixture", () => { expect(details).toContain("read live provider failure planned args"); }); - it("allows async live runtime tool fixtures to prove the happy path with the planned call", async () => { + it("skips async live runtime tool fixtures when the happy path has no result", async () => { const env = await makeEnv(); await writeQaSessionTranscript(env, "agent:qa:runtime-tool:image_generate:happy", [ { @@ -360,29 +360,26 @@ describe("runtime tool fixture", () => { }, ]); - const details = await runRuntimeToolFixture( - env, - { - toolName: "image_generate", - toolCoverage: { - bucket: "openclaw-dynamic-integration", - expectedLayer: "openclaw-dynamic", + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + }, + happyPathOutputRequired: false, }, - happyPathOutputRequired: false, - }, - { - createSession: vi.fn(async (_env, _label, key) => key!), - readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), - runAgentPrompt: vi.fn(async () => ({})), - fetchJson: vi.fn(), - ensureImageGenerationConfigured: vi.fn(), - }, - ); - - expect(details).toContain( - "image_generate live provider happy direct output not required for this async fixture", - ); - expect(details).toContain("image_generate live provider failure planned args"); + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson: vi.fn(), + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("planned call without a linked successful result"); }); it("still requires async live runtime tool fixtures to call the happy-path tool", async () => { @@ -559,7 +556,7 @@ describe("runtime tool fixture", () => { ).rejects.toThrow("expected live happy-path successful tool output for read"); }); - it("does not fail Codex-native fixtures solely because OpenClaw dynamic exposure is absent", async () => { + it("skips Codex-native fixtures when only OpenClaw dynamic exposure evidence is absent", async () => { const env = await makeEnv({ mock: { baseUrl: "http://127.0.0.1:9999" }, gateway: { @@ -589,35 +586,38 @@ describe("runtime tool fixture", () => { transcriptToolNames.push(params.transcriptToolName); return {}; }); - const details = await runRuntimeToolFixture( - env, - { - toolName: "read", - toolCoverage: { - bucket: "codex-native-workspace", - expectedLayer: "codex-native-workspace", - reason: "Codex owns read natively.", + await expect( + runRuntimeToolFixture( + env, + { + toolName: "read", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + reason: "Codex owns read natively.", + }, + promptSnippet: "target=read", + failurePromptSnippet: "failure target=read", }, - promptSnippet: "target=read", - failurePromptSnippet: "failure target=read", - }, - { - createSession: vi.fn(async (_env, _label, key) => key!), - readEffectiveTools: vi.fn(async () => new Set()), - runAgentPrompt, - fetchJson, - ensureImageGenerationConfigured: vi.fn(), - }, - ); - - expect(details).toContain("codex-native-workspace read"); - expect(details).toContain("OpenClaw dynamic exposure is intentionally omitted"); - expect(details).toContain("mock provider happy planned args (diagnostic only)"); + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt, + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toMatchObject({ + name: "QaSuiteScenarioSkipError", + message: expect.stringMatching( + /codex-native-workspace read[\s\S]*RUNTIME_PARITY_SESSION_KEY=agent:qa:runtime-tool:read:happy[\s\S]*RUNTIME_PARITY_SESSION_KEY=agent:qa:runtime-tool:read:failure/u, + ), + }); expect(runAgentPrompt).toHaveBeenCalledTimes(2); expect(transcriptToolNames).toEqual([undefined, undefined]); }); - it("reports Codex-native async planned-only happy fixtures without dereferencing missing output", async () => { + it("skips Codex-native async planned-only fixtures without treating the plan as proof", async () => { const env = await makeEnv({ mock: { baseUrl: "http://127.0.0.1:9999" }, gateway: { @@ -654,31 +654,29 @@ describe("runtime tool fixture", () => { }, ]); - const details = await runRuntimeToolFixture( - env, - { - toolName: "image_generate", - toolCoverage: { - bucket: "codex-native-workspace", - expectedLayer: "codex-native-workspace", - reason: "Codex owns image generation natively in this fixture.", + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "codex-native-workspace", + expectedLayer: "codex-native-workspace", + reason: "Codex owns image generation natively in this fixture.", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", + happyPathOutputRequired: false, }, - promptSnippet: "target=image_generate", - failurePromptSnippet: "failure target=image_generate", - happyPathOutputRequired: false, - }, - { - createSession: vi.fn(async (_env, _label, key) => key!), - readEffectiveTools: vi.fn(async () => new Set()), - runAgentPrompt: vi.fn(async () => ({})), - fetchJson, - ensureImageGenerationConfigured: vi.fn(), - }, - ); - - expect(details).toContain("codex-native-workspace image_generate"); - expect(details).toContain('"prompt":"QA lighthouse runtime parity fixture"'); - expect(details).toContain('"__qaFailureMode":"denied-input"'); + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set()), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("image_generate mock provider report-only"); }); it("requires mock runtime tool fixtures to produce tool output", async () => { @@ -728,7 +726,7 @@ describe("runtime tool fixture", () => { ).rejects.toThrow("expected mock happy-path tool output for read"); }); - it("allows async mock runtime tool fixtures to prove the happy path with the planned call", async () => { + it("skips async mock runtime tool fixtures when the happy path has no result", async () => { const env = await makeEnv({ mock: { baseUrl: "http://127.0.0.1:9999" }, }); @@ -755,32 +753,28 @@ describe("runtime tool fixture", () => { }, ]); - const details = await runRuntimeToolFixture( - env, - { - toolName: "image_generate", - toolCoverage: { - bucket: "openclaw-dynamic-integration", - expectedLayer: "openclaw-dynamic", + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", + happyPathOutputRequired: false, }, - promptSnippet: "target=image_generate", - failurePromptSnippet: "failure target=image_generate", - happyPathOutputRequired: false, - }, - { - createSession: vi.fn(async (_env, _label, key) => key!), - readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), - runAgentPrompt: vi.fn(async () => ({})), - fetchJson, - ensureImageGenerationConfigured: vi.fn(), - }, - ); - - expect(details).toContain( - "image_generate mock provider happy direct output not required for this async fixture", - ); - expect(details).toContain('"prompt":"QA lighthouse runtime parity fixture"'); - expect(details).toContain('"__qaFailureMode":"denied-input"'); + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("planned call without a linked successful result"); }); it("accepts mock runtime tool fixtures only after planned calls return output", async () => { @@ -839,7 +833,7 @@ describe("runtime tool fixture", () => { expect(details).toContain("read mock provider failure planned args"); }); - it("accepts non-required mock fixtures when both paths are planned without direct output", async () => { + it("skips non-required mock fixtures when both paths are only planned", async () => { const env = await makeEnv({ mock: { baseUrl: "http://127.0.0.1:9999" }, }); @@ -861,31 +855,29 @@ describe("runtime tool fixture", () => { }, ]); - const details = await runRuntimeToolFixture( - env, - { - toolName: "image_generate", - toolCoverage: { - bucket: "openclaw-dynamic-integration", - expectedLayer: "openclaw-dynamic", - required: false, - action: "optional runtime parity gate with async image completion coverage", + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + required: false, + action: "optional runtime parity gate with async image completion coverage", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", }, - promptSnippet: "target=image_generate", - failurePromptSnippet: "failure target=image_generate", - }, - { - createSession: vi.fn(async (_env, _label, key) => key!), - readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), - runAgentPrompt: vi.fn(async () => ({})), - fetchJson, - ensureImageGenerationConfigured: vi.fn(), - }, - ); - - expect(details).toContain("image_generate mock provider report-only"); - expect(details).toContain("image_generate mock provider happy planned args"); - expect(details).toContain("image_generate mock provider failure planned args"); + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("image_generate mock provider report-only"); }); it("still rejects failed happy output for non-required mock fixtures", async () => { diff --git a/extensions/qa-lab/src/runtime-tool-fixture.ts b/extensions/qa-lab/src/runtime-tool-fixture.ts index ccd4a8390fc7..359168ede4fa 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { loadTranscriptEventsSync } from "openclaw/plugin-sdk/session-store-runtime"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { QaSuiteInfraError } from "./errors.js"; +import { QaSuiteInfraError, QaSuiteScenarioSkipError } from "./errors.js"; import { qaMockRequestCursorUrl, qaMockRequestsAfterUrl, @@ -660,6 +660,9 @@ export async function runRuntimeToolFixture( const sessionKeys = [happySessionKey, failureSessionKey] as const; const withSessionDetails = (details: string) => runtimeToolFixtureDetails(details, ...sessionKeys); + const skipFixture = (details: string): never => { + throw new QaSuiteScenarioSkipError(withSessionDetails(details)); + }; const fixtureError = (error: unknown) => runtimeToolFixtureError(error, ...sessionKeys); const runFixtureOperation = async (operation: () => Promise): Promise => { try { @@ -678,13 +681,13 @@ export async function runRuntimeToolFixture( const expectedAvailable = readBoolean(config.expectedAvailable, true); if (!tools.has(toolName) && !dynamicExposureIntentionallyExcluded) { if (!expectedAvailable) { - return withSessionDetails(formatExpectedUnavailableDetails(toolName, tools)); + skipFixture(formatExpectedUnavailableDetails(toolName, tools)); } if (isKnownBroken(config.knownBroken)) { - return withSessionDetails(formatKnownBrokenDetails(toolName, tools, config)); + skipFixture(formatKnownBrokenDetails(toolName, tools, config)); } if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error( @@ -748,11 +751,12 @@ export async function runRuntimeToolFixture( if (!happyRequest.outputRequest) { const happyPlannedOnly = happyRequest.plannedRequest && !happyPathOutputRequired; if (happyPlannedOnly) { - // Async runtime tools prove the start call here; completion is covered - // by their task lifecycle scenarios. + skipFixture( + `${toolName} live provider report-only: a planned call without a linked successful result is not product execution evidence`, + ); } else { if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error( @@ -763,9 +767,9 @@ export async function runRuntimeToolFixture( ); } } - if (happyRequest.outputRequest?.structuredFailure) { + if (happyRequest.outputRequest?.failure) { if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error(`expected live happy-path successful tool output for ${toolName}`), @@ -780,7 +784,7 @@ export async function runRuntimeToolFixture( ); if (!failureRequest.outputRequest) { if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error( @@ -792,7 +796,7 @@ export async function runRuntimeToolFixture( } if (!failureRequest.failureOutputRequest) { if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error(`expected live failure-path tool failure output for ${toolName}`), @@ -856,7 +860,7 @@ export async function runRuntimeToolFixture( new Error(`expected mock failure-path tool failure output for ${toolName}`), ); } - return withSessionDetails( + skipFixture( formatReportOnlyMockDetails({ toolName, happyRequest: happyPlannedRequest, @@ -864,12 +868,15 @@ export async function runRuntimeToolFixture( }), ); } - // Async runtime tools prove the start call here; completion is covered by - // their task lifecycle scenarios. const happyPlannedOnly = Boolean(happyPlannedRequest && !happyPathOutputRequired); + if (!happyRequest && happyPlannedOnly) { + skipFixture( + `${toolName} mock provider report-only: a planned call without a linked successful result is not product execution evidence`, + ); + } if (!happyRequest && !happyPlannedOnly) { if (dynamicExposureIntentionallyExcluded) { - return withSessionDetails( + skipFixture( formatCodexNativeWorkspaceDetails({ toolName, tools, @@ -879,7 +886,7 @@ export async function runRuntimeToolFixture( ); } if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error( @@ -891,7 +898,7 @@ export async function runRuntimeToolFixture( } if (happyRequest && requestHasHappyPathFailureToolOutput(happyRequest.outputRequest)) { if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error(`expected mock happy-path successful tool output for ${toolName}`), @@ -899,7 +906,7 @@ export async function runRuntimeToolFixture( } if (!failureRequest) { if (dynamicExposureIntentionallyExcluded) { - return withSessionDetails( + skipFixture( formatCodexNativeWorkspaceDetails({ toolName, tools, @@ -910,7 +917,7 @@ export async function runRuntimeToolFixture( ); } if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError( new Error( @@ -922,13 +929,13 @@ export async function runRuntimeToolFixture( } if (!requestHasFailureLikeToolOutput(failureRequest.outputRequest)) { if (isKnownHarnessGap(config.knownHarnessGap)) { - return withSessionDetails(formatKnownHarnessGapDetails(toolName, config)); + skipFixture(formatKnownHarnessGapDetails(toolName, config)); } throw fixtureError(new Error(`expected mock failure-path tool failure output for ${toolName}`)); } if (dynamicExposureIntentionallyExcluded) { - return withSessionDetails( + skipFixture( formatCodexNativeWorkspaceDetails({ toolName, tools, diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 045c6066087e..30bc6d445090 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -342,6 +342,7 @@ describe("qa scenario catalog", () => { it("loads runtime tool fixture metadata for core and extended lanes", () => { const applyPatch = readQaScenarioById("runtime-tool-apply-patch"); + const sessionsSpawn = readQaScenarioById("runtime-tool-sessions-spawn"); const messageTool = readQaScenarioById("runtime-tool-message-tool"); const tavilySearch = readQaScenarioById("runtime-tool-tavily-search"); const webFetch = readQaScenarioById("runtime-tool-web-fetch"); @@ -349,6 +350,23 @@ describe("qa scenario catalog", () => { const imageGenerate = readQaScenarioById("runtime-tool-image-generate"); expect(applyPatch.runtimePairLane).toBe("core"); + for (const scenarioId of [ + "runtime-tool-apply-patch", + "runtime-tool-bash", + "runtime-tool-edit", + "runtime-tool-exec", + "runtime-tool-fs-list", + "runtime-tool-fs-read", + "runtime-tool-fs-write", + "runtime-tool-grep", + ]) { + const nativeWorkspaceScenario = readQaScenarioById(scenarioId); + expect(nativeWorkspaceScenario.coverage?.primary, scenarioId).toEqual([]); + expect(nativeWorkspaceScenario.coverage?.secondary?.length, scenarioId).toBeGreaterThan(0); + } + expect(sessionsSpawn.coverage?.primary).toEqual([ + "agent-runtime.subagent-turns-sessions-spawn", + ]); expect(messageTool.runtimePairLane).toBe("extended"); expect(tavilySearch.runtimePairLane).toBe("extended"); expect(imageGenerate.runtimePairLane).toBe("extended"); @@ -377,6 +395,16 @@ describe("qa scenario catalog", () => { required: true, }, }); + expect(readQaScenarioExecutionConfig(sessionsSpawn.id)).toMatchObject({ + toolName: "sessions_spawn", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + capabilityLayer: "openclaw-dynamic-direct", + required: true, + }, + }); + expect(readQaScenarioExecutionConfig(sessionsSpawn.id)).not.toHaveProperty("knownHarnessGap"); const webFetchConfig = readQaScenarioExecutionConfig(webFetch.id); expect(webFetchConfig?.happyPrompt).toContain("Call web_fetch exactly once"); expect(webFetchConfig?.happyPrompt).toContain("call it directly without tool_search"); diff --git a/extensions/qa-lab/src/suite-runtime-parity-result.ts b/extensions/qa-lab/src/suite-runtime-parity-result.ts new file mode 100644 index 000000000000..e601aa8f8393 --- /dev/null +++ b/extensions/qa-lab/src/suite-runtime-parity-result.ts @@ -0,0 +1,87 @@ +// QA Lab projects canonical runtime-pair results into suite scenario results. +import { + isRuntimeParityResultPass, + type RuntimeId, + type RuntimeParityCell, + type RuntimeParityResult, +} from "./runtime-parity.js"; +import type { QaSuiteScenarioResult } from "./suite-types.js"; + +function formatRuntimeParityCellDetails(cell: RuntimeParityCell) { + const errors = [cell.transportErrorClass, cell.runtimeErrorClass].filter(Boolean).join(", "); + const sentinels = cell.sentinelFindings?.map((finding) => finding.kind).join(", "); + return [ + `runtime=${cell.runtime}`, + `wallMs=${cell.wallClockMs}`, + `toolCalls=${cell.toolCalls.length}`, + `finalChars=${cell.finalText.length}`, + `tokens=${cell.usage.totalTokens}`, + ...(errors ? [`errors=${errors}`] : []), + ...(sentinels ? [`sentinels=${sentinels}`] : []), + ].join(" "); +} + +function formatRuntimeParityScenarioCellDetails(cell: RuntimeParityResult["cells"][RuntimeId]) { + return [cell.details, formatRuntimeParityCellDetails(cell)].filter(Boolean).join("\n"); +} + +function runtimeParityScenarioStepStatus( + cell: Pick< + RuntimeParityResult["cells"][RuntimeId], + "runtimeErrorClass" | "status" | "transportErrorClass" + >, +) { + if (cell.status === "fail" || cell.runtimeErrorClass || cell.transportErrorClass) { + return "fail"; + } + if (cell.status === "skip") { + return "skip"; + } + return "pass"; +} + +function runtimeParityScenarioResultStatus(result: RuntimeParityResult) { + const cellStatuses = new Set([ + runtimeParityScenarioStepStatus(result.cells.openclaw), + runtimeParityScenarioStepStatus(result.cells.codex), + ]); + if (cellStatuses.has("fail")) { + return "fail"; + } + if (cellStatuses.has("skip")) { + return "skip"; + } + return isRuntimeParityResultPass(result) ? "pass" : "fail"; +} + +export function buildRuntimeParityScenarioResult(params: { + scenarioName: string; + result: RuntimeParityResult; +}): QaSuiteScenarioResult { + const driftStepStatus = runtimeParityScenarioResultStatus(params.result); + const openclawCell = params.result.cells.openclaw; + const codexCell = params.result.cells.codex; + return { + name: params.scenarioName, + status: driftStepStatus, + details: params.result.driftDetails ?? `runtime drift classified as ${params.result.drift}`, + steps: [ + { + name: openclawCell.runtime, + status: runtimeParityScenarioStepStatus(openclawCell), + details: formatRuntimeParityScenarioCellDetails(openclawCell), + }, + { + name: codexCell.runtime, + status: runtimeParityScenarioStepStatus(codexCell), + details: formatRuntimeParityScenarioCellDetails(codexCell), + }, + { + name: "runtime drift", + status: driftStepStatus, + details: params.result.driftDetails ?? params.result.drift, + }, + ], + runtimeParity: params.result, + }; +} diff --git a/extensions/qa-lab/src/suite-runtime-parity-runner.test.ts b/extensions/qa-lab/src/suite-runtime-parity-runner.test.ts new file mode 100644 index 000000000000..a900b2371374 --- /dev/null +++ b/extensions/qa-lab/src/suite-runtime-parity-runner.test.ts @@ -0,0 +1,60 @@ +// QA Lab tests cover canonical runtime-pair result projection. +import { describe, expect, it } from "vitest"; +import { buildRuntimeParityScenarioResult } from "./suite-runtime-parity-result.js"; + +function makeCell( + runtime: "openclaw" | "codex", + status: "pass" | "fail" | "skip", + runtimeErrorClass?: string, +) { + return { + runtime, + status, + transcriptBytes: "", + toolCalls: [], + finalText: "", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + wallClockMs: 1, + bootStateLines: [], + ...(runtimeErrorClass ? { runtimeErrorClass } : {}), + }; +} + +describe("QA suite runtime parity result projection", () => { + it("combines result-cell status with execution errors without losing skip details", () => { + const failed = buildRuntimeParityScenarioResult({ + scenarioName: "Runtime tool fixture — read", + result: { + scenarioId: "runtime-tool-read", + cells: { + openclaw: { + ...makeCell("openclaw", "skip"), + details: + "implementation unavailable\nRUNTIME_PARITY_SESSION_KEY=agent:qa:runtime-tool:read:happy", + }, + codex: makeCell("codex", "pass", "scenario-failure"), + }, + drift: "none", + }, + }); + expect(failed.status).toBe("fail"); + expect(failed.steps?.map((step) => step.status)).toEqual(["skip", "fail", "fail"]); + expect(failed.steps?.[0]?.details).toContain( + "RUNTIME_PARITY_SESSION_KEY=agent:qa:runtime-tool:read:happy", + ); + + const skipped = buildRuntimeParityScenarioResult({ + scenarioName: "Runtime tool fixture — read", + result: { + scenarioId: "runtime-tool-read", + cells: { + openclaw: makeCell("openclaw", "skip"), + codex: makeCell("codex", "skip"), + }, + drift: "failure-mode", + }, + }); + expect(skipped.status).toBe("skip"); + expect(skipped.steps?.map((step) => step.status)).toEqual(["skip", "skip", "skip"]); + }); +}); diff --git a/extensions/qa-lab/src/suite-runtime-parity-runner.ts b/extensions/qa-lab/src/suite-runtime-parity-runner.ts index d7ba77669a47..8a96224d8770 100644 --- a/extensions/qa-lab/src/suite-runtime-parity-runner.ts +++ b/extensions/qa-lab/src/suite-runtime-parity-runner.ts @@ -11,11 +11,9 @@ import { sanitizeQaProgressValue as sanitizeQaSuiteProgressValue } from "./progr import type { QaThinkingLevel } from "./qa-gateway-config.js"; import type { QaTransportAdapterFactory, QaTransportId } from "./qa-transport-registry.js"; import { - isRuntimeParityResultPass, runRuntimeParityScenario, type RuntimeId, type RuntimeParityCell, - type RuntimeParityResult, } from "./runtime-parity.js"; import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; import type { QaScorecardChannelDriver, QaScorecardEvidenceMode } from "./scorecard-taxonomy.js"; @@ -26,6 +24,7 @@ import { resolveQaSuiteWorkerStartStaggerMs, scenarioRequiresControlUi, } from "./suite-planning.js"; +import { buildRuntimeParityScenarioResult } from "./suite-runtime-parity-result.js"; import { remapModelRefForForcedRuntime } from "./suite-support.js"; import type { QaSuiteRunParams, @@ -40,60 +39,6 @@ import { writeQaSuiteProgress, } from "./suite.js"; -function isRuntimeParityPass(result: RuntimeParityResult) { - return isRuntimeParityResultPass(result); -} - -function formatRuntimeParityCellDetails(cell: RuntimeParityCell) { - const errors = [cell.transportErrorClass, cell.runtimeErrorClass].filter(Boolean).join(", "); - const sentinels = cell.sentinelFindings?.map((finding) => finding.kind).join(", "); - return [ - `runtime=${cell.runtime}`, - `wallMs=${cell.wallClockMs}`, - `toolCalls=${cell.toolCalls.length}`, - `finalChars=${cell.finalText.length}`, - `tokens=${cell.usage.totalTokens}`, - ...(errors ? [`errors=${errors}`] : []), - ...(sentinels ? [`sentinels=${sentinels}`] : []), - ].join(" "); -} - -function buildRuntimeParityScenarioResult(params: { - scenarioName: string; - result: RuntimeParityResult; -}): QaSuiteScenarioResult { - const driftStepStatus = isRuntimeParityPass(params.result) ? "pass" : "fail"; - const openclawCell = params.result.cells.openclaw; - return { - name: params.scenarioName, - status: driftStepStatus, - details: params.result.driftDetails ?? `runtime drift classified as ${params.result.drift}`, - steps: [ - { - name: openclawCell.runtime, - status: - openclawCell.runtimeErrorClass || openclawCell.transportErrorClass ? "fail" : "pass", - details: formatRuntimeParityCellDetails(openclawCell), - }, - { - name: params.result.cells.codex.runtime, - status: - params.result.cells.codex.runtimeErrorClass || - params.result.cells.codex.transportErrorClass - ? "fail" - : "pass", - details: formatRuntimeParityCellDetails(params.result.cells.codex), - }, - { - name: "runtime drift", - status: driftStepStatus, - details: params.result.driftDetails ?? params.result.drift, - }, - ], - runtimeParity: params.result, - }; -} - export async function runQaRuntimeParitySuite(params: { runQaFlowSuite: QaSuiteRunner; adapterOptions?: QaSuiteRunParams["adapterOptions"]; @@ -252,8 +197,8 @@ export async function runQaRuntimeParitySuite(params: { bootStateLines: [], } satisfies RuntimeParityCell; return { - scenarioStatus: scenarioResult.status === "pass" ? "pass" : "fail", - scenarioDetails: scenarioResult.details, + status: scenarioResult.status, + details: scenarioResult.details, cell: cellResult.runtimeParityCell ?? fallbackCell, }; }, diff --git a/extensions/qa-lab/src/suite.summary-json.test.ts b/extensions/qa-lab/src/suite.summary-json.test.ts index 63c354a74ed0..01bef2a03f8c 100644 --- a/extensions/qa-lab/src/suite.summary-json.test.ts +++ b/extensions/qa-lab/src/suite.summary-json.test.ts @@ -202,6 +202,7 @@ describe("buildQaSuiteSummaryJson", () => { cells: { openclaw: { runtime: "openclaw" as const, + status: "pass" as const, transcriptBytes: "", toolCalls: [], finalText: "done", @@ -211,6 +212,7 @@ describe("buildQaSuiteSummaryJson", () => { }, codex: { runtime: "codex" as const, + status: "pass" as const, transcriptBytes: "", toolCalls: [], finalText: "done", @@ -227,6 +229,10 @@ describe("buildQaSuiteSummaryJson", () => { expect(json.scenarios[0]).toMatchObject({ runtimeParity: { scenarioId: "scenario-a", + cells: { + openclaw: { status: "pass" }, + codex: { status: "pass" }, + }, runtimeParityUsage: { expectation: "not-applicable", reason: "Local fixture only; no assistant turn runs.", diff --git a/extensions/qa-lab/src/token-efficiency-report.test.ts b/extensions/qa-lab/src/token-efficiency-report.test.ts index 641cb3be8166..b91be3aca326 100644 --- a/extensions/qa-lab/src/token-efficiency-report.test.ts +++ b/extensions/qa-lab/src/token-efficiency-report.test.ts @@ -47,7 +47,10 @@ function makeRuntimeParity( scenarioId, ...(runtimeParityUsage ? { runtimeParityUsage } : {}), drift: "none", - cells: { openclaw, codex }, + cells: { + openclaw: { ...openclaw, status: "pass" }, + codex: { ...codex, status: "pass" }, + }, }; } diff --git a/extensions/qa-lab/src/tool-coverage-report.test.ts b/extensions/qa-lab/src/tool-coverage-report.test.ts index fb339fa54f07..2a28c48be71e 100644 --- a/extensions/qa-lab/src/tool-coverage-report.test.ts +++ b/extensions/qa-lab/src/tool-coverage-report.test.ts @@ -48,6 +48,45 @@ function readToolCoverageConfig(config: Record): Record { it("derives tool fixture rows from tool coverage metadata", () => { const report = buildQaToolCoverageReport({ @@ -122,7 +161,66 @@ describe("qa tool coverage report", () => { expect(markdown).toContain("#80236 tracked \\| runtime drift"); }); - it("uses runtime parity summary rows and allows tracked known-broken drift", () => { + it("keeps tracking metadata independent from required coverage metrics", () => { + const report = buildQaToolCoverageReport({ + scenarios: [ + makeScenario("tool-read", "read", { + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + capabilityLayer: "openclaw-dynamic-direct", + required: true, + tracking: "#80236", + }, + }), + ], + summary: { + scenarios: [ + { + name: "tool read", + status: "pass", + runtimeParity: { + scenarioId: "tool-read", + drift: "none", + cells: { + openclaw: { + runtime: "openclaw", + status: "pass", + transcriptBytes: "", + toolCalls: [{ tool: "read", argsHash: "a", resultHash: "r" }], + finalText: "", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + wallClockMs: 1, + bootStateLines: [], + }, + codex: { + runtime: "codex", + status: "pass", + transcriptBytes: "", + toolCalls: [{ tool: "read", argsHash: "a", resultHash: "r" }], + finalText: "", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + wallClockMs: 1, + bootStateLines: [], + }, + }, + }, + }, + ], + }, + generatedAt: "2026-05-10T00:00:00.000Z", + }); + + expect(report).toMatchObject({ + pass: true, + requiredTools: 1, + reportOnlyTools: 0, + trackedTools: 1, + passingTools: 1, + }); + }); + + it("retains tracking metadata on accepted result-shape drift", () => { const report = buildQaToolCoverageReport({ scenarios: [ makeScenario("tool-read", "read"), @@ -149,6 +247,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "read", argsHash: "a", resultHash: "r" }], finalText: "", @@ -158,6 +257,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "read", argsHash: "a", resultHash: "r" }], finalText: "", @@ -178,6 +278,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "write", argsHash: "a", resultHash: "r1" }], finalText: "", @@ -187,6 +288,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "write", argsHash: "a", resultHash: "r2" }], finalText: "", @@ -206,7 +308,7 @@ describe("qa tool coverage report", () => { }); expect(report.pass).toBe(true); - expect(report.passingTools).toBe(1); + expect(report.passingTools).toBe(2); expect(report.trackedTools).toBe(1); expect(report.rows.find((row) => row.tool === "write")).toEqual( expect.objectContaining({ @@ -239,6 +341,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [], finalText: "", @@ -248,6 +351,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "optional", argsHash: "a", resultHash: "r" }], finalText: "", @@ -299,6 +403,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r" }], finalText: "", @@ -308,6 +413,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [], finalText: "", @@ -362,6 +468,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r1" }], finalText: "", @@ -371,6 +478,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r2" }], finalText: "", @@ -416,6 +524,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r1" }], finalText: "", @@ -425,6 +534,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r2" }], finalText: "", @@ -476,6 +586,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r" }], finalText: "", @@ -485,6 +596,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [], finalText: "", @@ -501,7 +613,149 @@ describe("qa tool coverage report", () => { }); expect(report.pass).toBe(false); - expect(report.failures).toEqual(["web-search missing codex tool call web_search"]); + expect(report.failures).toEqual([ + "web-search missing successful codex tool call/result web_search", + ]); + }); + + it("does not count error or unlinked results as successful required tool evidence", () => { + const report = buildQaToolCoverageReport({ + scenarios: [ + makeScenario("tool-web-search", "web-search", { + toolName: "web_search", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + capabilityLayer: "openclaw-dynamic-direct", + required: true, + }, + }), + ], + summary: { + scenarios: [ + { + name: "tool web_search", + status: "fail", + runtimeParity: { + scenarioId: "tool-web-search", + drift: "tool-result-shape", + cells: { + openclaw: { + runtime: "openclaw", + status: "pass", + transcriptBytes: "", + toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "ok" }], + finalText: "", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + wallClockMs: 1, + bootStateLines: [], + }, + codex: { + runtime: "codex", + status: "pass", + transcriptBytes: "", + toolCalls: [ + { + tool: "web_search", + argsHash: "a", + resultHash: "error", + errorClass: "tool-result-error", + }, + { + tool: "web_search", + argsHash: "b", + resultHash: "", + }, + ], + finalText: "", + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + wallClockMs: 1, + bootStateLines: [], + }, + }, + }, + }, + ], + }, + generatedAt: "2026-05-10T00:00:00.000Z", + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]).toMatchObject({ + codexToolCalls: 2, + codexSuccessfulToolCalls: 0, + }); + expect(report.passingTools).toBe(0); + expect(report.failures).toEqual([ + "web-search missing successful codex tool call/result web_search", + ]); + }); + + it("projects skipped runtime execution as non-passing coverage", () => { + const report = buildQaToolCoverageReport({ + scenarios: [ + makeScenario("tool-web-fetch", "web-fetch", { + toolName: "web_fetch", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + capabilityLayer: "openclaw-dynamic-direct", + required: true, + }, + }), + ], + summary: makeSkippedWebFetchSummary(), + generatedAt: "2026-05-10T00:00:00.000Z", + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]).toMatchObject({ openclaw: "pass", codex: "skip" }); + expect(report.failures).toEqual(["web-fetch status openclaw=pass codex=skip"]); + }); + + it("does not let skipped execution hide a runtime cell failure", () => { + const report = buildQaToolCoverageReport({ + scenarios: [ + makeScenario("tool-web-fetch", "web-fetch", { + toolName: "web_fetch", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + capabilityLayer: "openclaw-dynamic-direct", + required: true, + }, + }), + ], + summary: makeSkippedWebFetchSummary("capture-missing"), + generatedAt: "2026-05-10T00:00:00.000Z", + }); + + expect(report.rows[0]).toMatchObject({ openclaw: "pass", codex: "fail" }); + expect(report.failures).toEqual(["web-fetch status openclaw=pass codex=fail"]); + }); + + it("fails closed when a decoded result cell omits its required status", () => { + const summary = makeSkippedWebFetchSummary(); + delete (summary.scenarios[0]!.runtimeParity.cells.codex as { status?: string }).status; + const report = buildQaToolCoverageReport({ + scenarios: [ + makeScenario("tool-web-fetch", "web-fetch", { + toolName: "web_fetch", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + capabilityLayer: "openclaw-dynamic-direct", + required: true, + }, + }), + ], + summary, + generatedAt: "2026-05-10T00:00:00.000Z", + }); + + expect(report.pass).toBe(false); + expect(report.rows[0]).toMatchObject({ openclaw: "pass", codex: "fail" }); + expect(report.failures).toEqual(["web-fetch status openclaw=pass codex=fail"]); }); it("fails required OpenClaw dynamic tool coverage when the fixture failure mode is preserved", () => { @@ -529,6 +783,7 @@ describe("qa tool coverage report", () => { cells: { openclaw: { runtime: "openclaw", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r" }], finalText: "", @@ -538,6 +793,7 @@ describe("qa tool coverage report", () => { }, codex: { runtime: "codex", + status: "pass", transcriptBytes: "", toolCalls: [{ tool: "web_search", argsHash: "a", resultHash: "r" }], finalText: "", @@ -636,7 +892,7 @@ describe("qa tool coverage report", () => { expect.objectContaining({ bucket: "codex-native-workspace", expectedLayer: "codex-native-workspace", - required: true, + required: false, }), ); expect(applyPatchRow).toEqual( @@ -648,10 +904,10 @@ describe("qa tool coverage report", () => { expect(report.rows.find((row) => row.tool === "sessions_spawn")).toEqual( expect.objectContaining({ required: true, - tracking: expect.stringContaining("#80319"), - action: expect.stringContaining("report-only"), + action: expect.stringContaining("hard gate"), }), ); + expect(report.rows.find((row) => row.tool === "sessions_spawn")?.tracking).toBeUndefined(); expect(report.rows.find((row) => row.tool === "message-tool")).toEqual( expect.objectContaining({ bucket: "optional-profile-or-plugin", diff --git a/extensions/qa-lab/src/tool-coverage-report.ts b/extensions/qa-lab/src/tool-coverage-report.ts index 702b4b6315ea..56842bfc1913 100644 --- a/extensions/qa-lab/src/tool-coverage-report.ts +++ b/extensions/qa-lab/src/tool-coverage-report.ts @@ -7,7 +7,6 @@ import { import { isRuntimeParityCellPassable, type RuntimeId, - type RuntimeParityCell, type RuntimeParityDrift, type RuntimeParityResult, } from "./runtime-parity.js"; @@ -22,7 +21,7 @@ import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; type QaToolCoverageSuiteScenario = { name: string; - status: "pass" | "fail"; + status: "pass" | "fail" | "skip"; runtimeParity?: RuntimeParityResult; }; @@ -33,7 +32,7 @@ export type QaToolCoverageSuiteSummary = { }; }; -type QaToolCoverageStatus = "pass" | "fail" | "missing" | "not-run"; +type QaToolCoverageStatus = "pass" | "fail" | "skip" | "missing" | "not-run"; type QaToolCoverageDrift = RuntimeParityDrift | "not-run"; type QaToolCoverageBucket = QaRuntimeToolBucket; @@ -52,6 +51,8 @@ type QaToolCoverageRow = { drift: QaToolCoverageDrift; openclawToolCalls: number; codexToolCalls: number; + openclawSuccessfulToolCalls: number; + codexSuccessfulToolCalls: number; tracking?: string; codexDefaultImpact?: string; qaImpact?: string; @@ -94,11 +95,22 @@ function normalizeRuntimePair( return ["openclaw", "codex"]; } -function cellStatus(cell: RuntimeParityCell | undefined): QaToolCoverageStatus { +function cellStatus( + cell: RuntimeParityResult["cells"][RuntimeId] | undefined, +): QaToolCoverageStatus { if (!cell) { return "missing"; } - return isRuntimeParityCellPassable(cell) ? "pass" : "fail"; + if (!isRuntimeParityCellPassable(cell)) { + return "fail"; + } + if (cell.status === "skip") { + return "skip"; + } + if (cell.status === "fail") { + return "fail"; + } + return cell.status === "pass" ? "pass" : "fail"; } function toolIdsForScenario(scenario: QaSeedScenarioWithSource): string[] { @@ -179,10 +191,6 @@ function mergeScenarioResults( return failingResult; } -function isPassingToolCoverageDrift(drift: QaToolCoverageDrift, evaluated: boolean) { - return PASSING_DRIFTS.has(drift) || (!evaluated && drift === "not-run"); -} - function countRuntimeToolCalls( result: RuntimeParityResult | undefined, runtime: RuntimeId, @@ -195,6 +203,20 @@ function countRuntimeToolCalls( return cell.toolCalls.filter((call) => call.tool === toolName).length; } +function countSuccessfulRuntimeToolCalls( + result: RuntimeParityResult | undefined, + runtime: RuntimeId, + toolName: string | undefined, +) { + if (!result || !toolName) { + return 0; + } + const cell = runtime === "openclaw" ? result.cells.openclaw : result.cells.codex; + return cell.toolCalls.filter( + (call) => call.tool === toolName && !call.errorClass && call.resultHash.trim().length > 0, + ).length; +} + function buildRow(params: { group: ToolFixtureGroup; results: ReadonlyMap; @@ -226,6 +248,12 @@ function buildRow(params: { drift: result?.drift ?? "not-run", openclawToolCalls: countRuntimeToolCalls(result, "openclaw", runtimeToolName), codexToolCalls: countRuntimeToolCalls(result, "codex", runtimeToolName), + openclawSuccessfulToolCalls: countSuccessfulRuntimeToolCalls( + result, + "openclaw", + runtimeToolName, + ), + codexSuccessfulToolCalls: countSuccessfulRuntimeToolCalls(result, "codex", runtimeToolName), ...(tracking ? { tracking } : {}), ...(rowMetadata.codexDefaultImpact ? { codexDefaultImpact: rowMetadata.codexDefaultImpact } @@ -237,7 +265,7 @@ function buildRow(params: { } function coverageFailureForRow(row: QaToolCoverageRow): string | undefined { - if (!row.required || row.tracking) { + if (!row.required) { return undefined; } if (row.drift === "not-run") { @@ -249,11 +277,11 @@ function coverageFailureForRow(row: QaToolCoverageRow): string | undefined { if (row.drift === "failure-mode") { return `${row.tool} drift=failure-mode${row.details ? ` (${row.details})` : ""}`; } - if (row.runtimeToolName && row.openclawToolCalls === 0) { - return `${row.tool} missing openclaw tool call ${row.runtimeToolName}`; + if (row.runtimeToolName && row.openclawSuccessfulToolCalls === 0) { + return `${row.tool} missing successful openclaw tool call/result ${row.runtimeToolName}`; } - if (row.runtimeToolName && row.codexToolCalls === 0) { - return `${row.tool} missing codex tool call ${row.runtimeToolName}`; + if (row.runtimeToolName && row.codexSuccessfulToolCalls === 0) { + return `${row.tool} missing successful codex tool call/result ${row.runtimeToolName}`; } return undefined; } @@ -281,7 +309,7 @@ export function buildQaToolCoverageReport(params: { evaluated, totalTools: rows.length, requiredTools: rows.filter((row) => row.required).length, - reportOnlyTools: rows.filter((row) => !row.required || Boolean(row.tracking)).length, + reportOnlyTools: rows.filter((row) => !row.required).length, trackedTools: rows.filter((row) => Boolean(row.tracking)).length, nativeWorkspaceTools: rows.filter((row) => row.bucket === "codex-native-workspace").length, dynamicIntegrationTools: rows.filter((row) => row.bucket === "openclaw-dynamic-integration") @@ -291,14 +319,7 @@ export function buildQaToolCoverageReport(params: { ).length, optionalTools: rows.filter((row) => row.bucket === "optional-profile-or-plugin").length, passingTools: evaluated - ? rows.filter( - (row) => - row.required && - !row.tracking && - row.openclaw === "pass" && - row.codex === "pass" && - (isPassingToolCoverageDrift(row.drift, true) || !coverageFailureForRow(row)), - ).length + ? rows.filter((row) => row.required && !coverageFailureForRow(row)).length : 0, failingTools: failures.length, rows, diff --git a/qa/scenarios/runtime/tools/apply-patch.yaml b/qa/scenarios/runtime/tools/apply-patch.yaml index 1f31899db08b..a0402c0ac014 100644 --- a/qa/scenarios/runtime/tools/apply-patch.yaml +++ b/qa/scenarios/runtime/tools/apply-patch.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - agent-runtime.tool-apply-patch objective: Verify apply_patch behavior is tracked across OpenClaw and Codex while Codex owns patching natively. successCriteria: @@ -27,7 +27,7 @@ scenario: actualTool: apply_patch bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80320" codexDefaultImpact: P4 qaImpact: P2 diff --git a/qa/scenarios/runtime/tools/bash.yaml b/qa/scenarios/runtime/tools/bash.yaml index 4f1d1f062dfa..516ffbf77e64 100644 --- a/qa/scenarios/runtime/tools/bash.yaml +++ b/qa/scenarios/runtime/tools/bash.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - tools.exec-routing-bash objective: Verify shell command behavior is tracked across OpenClaw and Codex while Codex owns exec/process natively. successCriteria: @@ -28,7 +28,7 @@ scenario: actualTool: exec bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80319" codexDefaultImpact: P4 qaImpact: P1 diff --git a/qa/scenarios/runtime/tools/edit.yaml b/qa/scenarios/runtime/tools/edit.yaml index eb10c024dc50..3ff44ff654c9 100644 --- a/qa/scenarios/runtime/tools/edit.yaml +++ b/qa/scenarios/runtime/tools/edit.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - agent-runtime.tool-edit objective: Verify targeted edit behavior is tracked across OpenClaw and Codex while Codex owns edit natively. successCriteria: @@ -27,7 +27,7 @@ scenario: actualTool: edit bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80319" codexDefaultImpact: P4 qaImpact: P1 diff --git a/qa/scenarios/runtime/tools/exec.yaml b/qa/scenarios/runtime/tools/exec.yaml index 118c2cdbd224..24c9102b1a7c 100644 --- a/qa/scenarios/runtime/tools/exec.yaml +++ b/qa/scenarios/runtime/tools/exec.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - tools.exec-routing-exec objective: Verify command execution behavior is tracked across OpenClaw and Codex while Codex owns exec/process natively. successCriteria: @@ -27,7 +27,7 @@ scenario: actualTool: exec bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80319" codexDefaultImpact: P4 qaImpact: P1 diff --git a/qa/scenarios/runtime/tools/fs-list.yaml b/qa/scenarios/runtime/tools/fs-list.yaml index 1aa3e917bab5..1fc314390d99 100644 --- a/qa/scenarios/runtime/tools/fs-list.yaml +++ b/qa/scenarios/runtime/tools/fs-list.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - agent-runtime.tool-fs-list objective: Verify directory inspection behavior is tracked through read while Codex owns file inspection natively. successCriteria: @@ -27,7 +27,7 @@ scenario: actualTool: read bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80312" codexDefaultImpact: P4 qaImpact: P2 diff --git a/qa/scenarios/runtime/tools/fs-read.yaml b/qa/scenarios/runtime/tools/fs-read.yaml index c8e3794c9ffd..7d2f48bf5809 100644 --- a/qa/scenarios/runtime/tools/fs-read.yaml +++ b/qa/scenarios/runtime/tools/fs-read.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - agent-runtime.tool-fs-read objective: Verify file read behavior is tracked across OpenClaw and Codex while Codex owns read natively. successCriteria: @@ -27,7 +27,7 @@ scenario: actualTool: read bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80312" codexDefaultImpact: P4 qaImpact: P2 diff --git a/qa/scenarios/runtime/tools/fs-write.yaml b/qa/scenarios/runtime/tools/fs-write.yaml index f96e7cda8b1a..420ab06c1455 100644 --- a/qa/scenarios/runtime/tools/fs-write.yaml +++ b/qa/scenarios/runtime/tools/fs-write.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - agent-runtime.tool-fs-write objective: Verify file write behavior is tracked across OpenClaw and Codex while Codex owns write natively. successCriteria: @@ -27,7 +27,7 @@ scenario: actualTool: write bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80319" codexDefaultImpact: P4 qaImpact: P1 diff --git a/qa/scenarios/runtime/tools/grep.yaml b/qa/scenarios/runtime/tools/grep.yaml index 1a9f8861e458..ea714f3b41c7 100644 --- a/qa/scenarios/runtime/tools/grep.yaml +++ b/qa/scenarios/runtime/tools/grep.yaml @@ -5,7 +5,7 @@ scenario: surface: runtime-tools runtimePairLane: core coverage: - primary: + secondary: - agent-runtime.tool-grep objective: Verify grep-style search behavior is tracked through command execution while Codex owns exec/process natively. successCriteria: @@ -27,7 +27,7 @@ scenario: actualTool: exec bucket: codex-native-workspace expectedLayer: codex-native-workspace - required: true + required: false tracking: "#80319" codexDefaultImpact: P4 qaImpact: P1 diff --git a/qa/scenarios/runtime/tools/sessions-spawn.yaml b/qa/scenarios/runtime/tools/sessions-spawn.yaml index 42f36cd3f890..5599f994bdfb 100644 --- a/qa/scenarios/runtime/tools/sessions-spawn.yaml +++ b/qa/scenarios/runtime/tools/sessions-spawn.yaml @@ -7,12 +7,12 @@ scenario: coverage: primary: - agent-runtime.subagent-turns-sessions-spawn - objective: Verify sessions_spawn exposure and provider planning across OpenClaw and Codex. + objective: Verify sessions_spawn executes successfully across OpenClaw and Codex. successCriteria: - Effective tools expose sessions_spawn. - - The mock provider plans exactly one happy-path sessions_spawn call. - - The mock provider plans one denied-input failure-path sessions_spawn call. - - Runtime parity reports provider-plan evidence separately while transcript execution coverage remains tracked. + - The happy path retains a linked successful sessions_spawn call and result. + - The denied-input path retains a linked sessions_spawn failure result. + - Runtime parity hard-fails missing call/result evidence in either runtime. docsRefs: - qa/scenarios/index.yaml codeRefs: @@ -30,14 +30,10 @@ scenario: expectedLayer: openclaw-dynamic capabilityLayer: openclaw-dynamic-direct required: true - tracking: "#80319" codexDefaultImpact: P4 qaImpact: P1 - action: report-only until the core runtime-pair lane captures transcript-backed sessions_spawn execution - reason: The fixture proves exposure and provider planning, but its parity cell does not retain transcript-backed parent execution after subagent spawning. - knownHarnessGap: - issue: "#80319" - reason: The parity cell currently retains provider-plan evidence but not transcript-backed parent execution after subagent spawning. + action: hard gate in the core runtime-pair lane + reason: Canonical fixture session-key capture retains the parent call/result when spawning creates a newer child session, so this OpenClaw integration tool must hard-gate both runtimes. promptSnippet: "target=sessions_spawn" failurePromptSnippet: "failure target=sessions_spawn"