From 9b703d0cd6762da8423f27d2bd9ad8b88c19910c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 29 Jun 2026 16:44:37 -0700 Subject: [PATCH] test(qa): accept async image fixture coverage --- extensions/qa-lab/package.json | 2 +- .../qa-lab/src/runtime-tool-fixture.test.ts | 247 ++++++++++++++++++ extensions/qa-lab/src/runtime-tool-fixture.ts | 81 +++++- pnpm-lock.yaml | 52 ++-- pnpm-workspace.yaml | 2 +- 5 files changed, 341 insertions(+), 43 deletions(-) diff --git a/extensions/qa-lab/package.json b/extensions/qa-lab/package.json index db09e4f79ee2..eaf149f537ec 100644 --- a/extensions/qa-lab/package.json +++ b/extensions/qa-lab/package.json @@ -16,7 +16,7 @@ "@openclaw/plugin-sdk": "workspace:*", "@openclaw/slack": "workspace:*", "@openclaw/whatsapp": "workspace:*", - "@openclaw/crabline": "0.1.0", + "@openclaw/crabline": "0.1.6", "openclaw": "2026.5.28" }, "peerDependencies": { diff --git a/extensions/qa-lab/src/runtime-tool-fixture.test.ts b/extensions/qa-lab/src/runtime-tool-fixture.test.ts index 2aeb165e2eb1..ceb106d692a0 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.test.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.test.ts @@ -548,6 +548,253 @@ describe("runtime tool fixture", () => { expect(details).toContain("read mock provider failure planned args"); }); + it("accepts non-required mock fixtures when both paths are planned without direct output", async () => { + const env = await makeEnv({ + mock: { baseUrl: "http://127.0.0.1:9999" }, + }); + const fetchJson = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + allInputText: "target=image_generate", + plannedToolCallId: "call-image-happy", + plannedToolName: "image_generate", + plannedToolArgs: { prompt: "QA lighthouse", filename: "runtime-tool-fixture" }, + }, + { + allInputText: "failure target=image_generate", + plannedToolCallId: "call-image-failure", + plannedToolName: "image_generate", + plannedToolArgs: { __qaFailureMode: "denied-input" }, + }, + ]); + + const details = await runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + required: false, + action: "optional runtime parity gate with async image completion coverage", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ); + + expect(details).toContain("image_generate mock provider report-only"); + expect(details).toContain("image_generate mock provider happy planned args"); + expect(details).toContain("image_generate mock provider failure planned args"); + }); + + it("still rejects failed happy output for non-required mock fixtures", async () => { + const env = await makeEnv({ + mock: { baseUrl: "http://127.0.0.1:9999" }, + }); + const fetchJson = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + allInputText: "target=image_generate", + plannedToolCallId: "call-image-happy", + plannedToolName: "image_generate", + plannedToolArgs: { prompt: "QA lighthouse" }, + }, + { + allInputText: "target=image_generate", + toolOutputCallId: "call-image-happy", + toolOutput: "Failed: provider rejected image request", + }, + { + allInputText: "failure target=image_generate", + plannedToolCallId: "call-image-failure", + plannedToolName: "image_generate", + plannedToolArgs: { __qaFailureMode: "denied-input" }, + }, + ]); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + required: false, + action: "optional runtime parity gate with async image completion coverage", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("expected mock happy-path successful tool output for image_generate"); + }); + + it("still rejects successful failure output for non-required mock fixtures", async () => { + const env = await makeEnv({ + mock: { baseUrl: "http://127.0.0.1:9999" }, + }); + const fetchJson = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + allInputText: "target=image_generate", + plannedToolCallId: "call-image-happy", + plannedToolName: "image_generate", + plannedToolArgs: { prompt: "QA lighthouse" }, + }, + { + allInputText: "failure target=image_generate", + plannedToolCallId: "call-image-failure", + plannedToolName: "image_generate", + plannedToolArgs: { __qaFailureMode: "denied-input" }, + }, + { + allInputText: "failure target=image_generate", + toolOutputCallId: "call-image-failure", + toolOutput: "Task queued for async image delivery", + }, + ]); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + required: false, + action: "optional runtime parity gate with async image completion coverage", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("expected mock failure-path tool failure output for image_generate"); + }); + + it("rejects malformed report-only failure plans for non-required mock fixtures", async () => { + const env = await makeEnv({ + mock: { baseUrl: "http://127.0.0.1:9999" }, + }); + const fetchJson = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + allInputText: "target=image_generate", + plannedToolCallId: "call-image-happy", + plannedToolName: "image_generate", + plannedToolArgs: { prompt: "QA lighthouse" }, + }, + { + allInputText: "failure target=image_generate", + plannedToolCallId: "call-image-failure", + plannedToolName: "image_generate", + plannedToolArgs: { prompt: "not a denied-input failure" }, + }, + ]); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + required: false, + action: "optional runtime parity gate with async image completion coverage", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("expected mock failure-path denied-input args for image_generate"); + }); + + it("rejects malformed report-only happy plans for non-required mock fixtures", async () => { + const env = await makeEnv({ + mock: { baseUrl: "http://127.0.0.1:9999" }, + }); + const fetchJson = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + allInputText: "target=image_generate", + plannedToolCallId: "call-image-happy", + plannedToolName: "image_generate", + plannedToolArgs: {}, + }, + { + allInputText: "failure target=image_generate", + plannedToolCallId: "call-image-failure", + plannedToolName: "image_generate", + plannedToolArgs: { __qaFailureMode: "denied-input" }, + }, + ]); + + await expect( + runRuntimeToolFixture( + env, + { + toolName: "image_generate", + toolCoverage: { + bucket: "openclaw-dynamic-integration", + expectedLayer: "openclaw-dynamic", + required: false, + action: "optional runtime parity gate with async image completion coverage", + }, + promptSnippet: "target=image_generate", + failurePromptSnippet: "failure target=image_generate", + }, + { + createSession: vi.fn(async (_env, _label, key) => key!), + readEffectiveTools: vi.fn(async () => new Set(["image_generate"])), + runAgentPrompt: vi.fn(async () => ({})), + fetchJson, + ensureImageGenerationConfigured: vi.fn(), + }, + ), + ).rejects.toThrow("expected mock happy-path prompt args for image_generate"); + }); + it("rejects failure-shaped mock happy-path tool output", async () => { const env = await makeEnv({ mock: { baseUrl: "http://127.0.0.1:9999" }, diff --git a/extensions/qa-lab/src/runtime-tool-fixture.ts b/extensions/qa-lab/src/runtime-tool-fixture.ts index cb91639a6718..6bb11eff6431 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.ts @@ -2,7 +2,10 @@ import fs from "node:fs/promises"; import path from "node:path"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { readRuntimeToolCoverageMetadata } from "./runtime-tool-metadata.js"; +import { + type QaRuntimeToolCoverageMetadata, + readRuntimeToolCoverageMetadata, +} from "./runtime-tool-metadata.js"; import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js"; import { readRawQaSessionStore } from "./suite-runtime-agent-session.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; @@ -554,6 +557,37 @@ function formatCodexNativeWorkspaceDetails(params: { .join("\n"); } +function formatReportOnlyMockDetails(params: { + toolName: string; + happyRequest: QaRuntimeToolFixtureRequest; + failureRequest: QaRuntimeToolFixtureRequest; +}) { + return [ + `${params.toolName} mock provider report-only: direct tool output is not required by this fixture`, + `${params.toolName} mock provider happy planned args (diagnostic only): ${formatPlannedToolArgs(params.happyRequest.plannedToolArgs)}`, + `${params.toolName} mock provider failure planned args (diagnostic only): ${formatPlannedToolArgs(params.failureRequest.plannedToolArgs)}`, + ].join("\n"); +} + +function isAsyncReportOnlyMockCoverage(metadata: QaRuntimeToolCoverageMetadata) { + return !metadata.required && /\basync\b/iu.test(metadata.action ?? ""); +} + +function plannedRequestHasDeniedInputFailure(request: QaRuntimeToolFixtureRequest) { + return ( + isRecord(request.plannedToolArgs) && + request.plannedToolArgs["__qaFailureMode"] === "denied-input" + ); +} + +function plannedRequestHasPrompt(request: QaRuntimeToolFixtureRequest) { + return ( + isRecord(request.plannedToolArgs) && + typeof request.plannedToolArgs.prompt === "string" && + request.plannedToolArgs.prompt.trim().length > 0 + ); +} + function formatKnownHarnessGapDetails(toolName: string, config: QaRuntimeToolFixtureConfig) { const knownHarnessGap = isKnownHarnessGap(config.knownHarnessGap) ? config.knownHarnessGap : {}; const issue = readString(knownHarnessGap.issue); @@ -709,6 +743,39 @@ export async function runRuntimeToolFixture( excludedPromptSnippet: failurePromptSnippet, toolName, }); + const failurePlannedRequest = findPlannedRequest({ + requests, + requestCountBefore, + promptSnippet: failurePromptSnippet, + toolName, + }); + const failureRequest = findExecutedRequest({ + requests, + requestCountBefore, + promptSnippet: failurePromptSnippet, + toolName, + }); + if ( + isAsyncReportOnlyMockCoverage(metadata) && + happyPlannedRequest && + failurePlannedRequest && + !happyRequest + ) { + if (!plannedRequestHasPrompt(happyPlannedRequest)) { + throw new Error(`expected mock happy-path prompt args for ${toolName}`); + } + if (!plannedRequestHasDeniedInputFailure(failurePlannedRequest)) { + throw new Error(`expected mock failure-path denied-input args for ${toolName}`); + } + if (failureRequest && !requestHasFailureLikeToolOutput(failureRequest.outputRequest)) { + throw new Error(`expected mock failure-path tool failure output for ${toolName}`); + } + return formatReportOnlyMockDetails({ + toolName, + happyRequest: happyPlannedRequest, + failureRequest: failurePlannedRequest, + }); + } if (!happyRequest) { if (dynamicExposureIntentionallyExcluded) { return formatCodexNativeWorkspaceDetails({ @@ -733,18 +800,6 @@ export async function runRuntimeToolFixture( } throw new Error(`expected mock happy-path successful tool output for ${toolName}`); } - const failurePlannedRequest = findPlannedRequest({ - requests, - requestCountBefore, - promptSnippet: failurePromptSnippet, - toolName, - }); - const failureRequest = findExecutedRequest({ - requests, - requestCountBefore, - promptSnippet: failurePromptSnippet, - toolName, - }); if (!failureRequest) { if (dynamicExposureIntentionallyExcluded) { return formatCodexNativeWorkspaceDetails({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c45f525540f9..f3c46e10cd15 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1346,8 +1346,8 @@ importers: version: 4.4.3 devDependencies: '@openclaw/crabline': - specifier: 0.1.0 - version: 0.1.0 + specifier: 0.1.6 + version: 0.1.6 '@openclaw/discord': specifier: workspace:* version: link:../discord @@ -2368,18 +2368,10 @@ packages: resolution: {integrity: sha512-fT1qHVGAag4IEkrupZ6lRRbNCs1vS9P01KB/sG8zKgvUztbYtFBtQpjSITNwooDZ83tpsPzP0mRNs1/KVszCRA==} engines: {node: '>= 20.12.0'} - '@clack/core@1.4.1': - resolution: {integrity: sha512-FILJa1gGKEFTGZAJE9RpVhrjKz3c3h4ar60dSv6cGuDqufQ84YEIS3GAGvZiN+H6yaLbbvTFNejjCC4tXpZEuw==} - engines: {node: '>= 20.12.0'} - '@clack/prompts@1.4.0': resolution: {integrity: sha512-S0My7XPGIgpRWMDG8uRqalbgT+a6FmCUdOW+HaIOVVpUPHOb7RrpvjTjiODadKp06fsrVDJZlIzc6yCTp4AnxA==} engines: {node: '>= 20.12.0'} - '@clack/prompts@1.5.1': - resolution: {integrity: sha512-zccHj2z2oCCO4yrDiRSlFOxWerGqRiysP7a5jPK6uoI9URKAquwY42Dd/iUP8JWHxEzdRe4TlbvZCo8z1/mhrw==} - engines: {node: '>= 20.12.0'} - '@clawdbot/lobster@2026.5.22': resolution: {integrity: sha512-lrUnsLLmo4sVFDNd7oTQiEnt4whGhy1bDHe3s/sQoInbTLJSN55WKxlUCfberZbzwOq8GSNnBkP0ZhZ3mKSBwg==} engines: {node: '>=20'} @@ -3182,8 +3174,8 @@ packages: cpu: [x64] os: [win32] - '@openclaw/crabline@0.1.0': - resolution: {integrity: sha512-9M63tpAN3UNJGpCPB5e8bSu5aS4AGHfcH11fTKoNWr6ieetprJRA0iqToMu/ZDu/j1P2yNnOQwQ0XDTTLeU0lw==} + '@openclaw/crabline@0.1.6': + resolution: {integrity: sha512-nu/XD7eoly5DJOEG7krCfZY68cDcD/AC31mAMLoP36HzmetNVM17OtfnakpuWyBwdO+c4noF7b2LM4AAKvq9CQ==} engines: {node: '>=22'} hasBin: true @@ -4581,6 +4573,7 @@ packages: '@zed-industries/codex-acp@0.15.0': resolution: {integrity: sha512-eAv7sGBeiYrYkOulF729nrM51szS7WIhBtugRj5wWq6csRKZUhAZfoUZlF8xUWdHPtOIzd/eT6MNG6gMHu6z0w==} + deprecated: This package has been replaced by @agentclientprotocol/codex-acp. Please migrate to continue receiving updates. hasBin: true abort-controller@3.0.0: @@ -7530,6 +7523,10 @@ packages: resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} engines: {node: '>=20.18.1'} + undici@8.3.0: + resolution: {integrity: sha512-TkUDgb6tl7KOGZ+7e8E3d2FYgUQgF6z5YypqjWmixVQSQERFcVrVg0ySADm2LVLRh5ljAaHTCR5Fmz3Q34rB7Q==} + engines: {node: '>=22.19.0'} + undici@8.5.0: resolution: {integrity: sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==} engines: {node: '>=22.19.0'} @@ -8518,11 +8515,6 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 - '@clack/core@1.4.1': - dependencies: - fast-wrap-ansi: 0.2.2 - sisteransi: 1.0.5 - '@clack/prompts@1.4.0': dependencies: '@clack/core': 1.3.1 @@ -8530,13 +8522,6 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 - '@clack/prompts@1.5.1': - dependencies: - '@clack/core': 1.4.1 - fast-string-width: 3.0.2 - fast-wrap-ansi: 0.2.2 - sisteransi: 1.0.5 - '@clawdbot/lobster@2026.5.22': dependencies: ajv: 8.20.0 @@ -9269,18 +9254,27 @@ snapshots: '@openai/codex@0.139.0-win32-x64': optional: true - '@openclaw/crabline@0.1.0': + '@openclaw/crabline@0.1.6': dependencies: - commander: 14.0.3 + commander: 15.0.0 + curve25519-js: 0.0.4 picocolors: 1.1.1 + ws: 8.21.0 yaml: 2.9.0 zod: 4.4.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate '@openclaw/fs-safe@0.3.0': optionalDependencies: jszip: 3.10.1 tar: 7.5.16 + '@openclaw/proxyline@0.3.3(undici@8.3.0)': + dependencies: + undici: 8.3.0 + '@openclaw/proxyline@0.3.3(undici@8.5.0)': dependencies: undici: 8.5.0 @@ -12961,7 +12955,7 @@ snapshots: '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) '@mozilla/readability': 0.6.0 '@openclaw/fs-safe': 0.3.0 - '@openclaw/proxyline': 0.3.3(undici@8.5.0) + '@openclaw/proxyline': 0.3.3(undici@8.3.0) '@silvia-odwyer/photon-node': 0.3.4 chalk: 5.6.2 chokidar: 5.0.0 @@ -12999,7 +12993,7 @@ snapshots: tslog: 4.10.2 typebox: 1.1.39 typescript: 6.0.3 - undici: 8.5.0 + undici: 8.3.0 web-push: 3.6.7 web-tree-sitter: 0.26.9 ws: 8.21.0 @@ -14137,6 +14131,8 @@ snapshots: undici@7.28.0: {} + undici@8.3.0: {} + undici@8.5.0: {} unhomoglyph@1.0.6: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 532333962e00..b6022e7702e0 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,7 +7,7 @@ packages: minimumReleaseAge: 2880 minimumReleaseAgeExclude: - - "@openclaw/crabline@0.1.0" + - "@openclaw/crabline@0.1.6" - "@openclaw/fs-safe@0.3.0" - "@openclaw/proxyline@0.3.3" - "acpx"