ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

test(qa): accept async image fixture coverage

Browse Source 
(cherry picked from commit 9b703d0cd6)
This commit is contained in:
Vincent Koc
2026-06-29 16:44:37 -07:00
committed by Vincent Koc
parent 31f58cd9de
commit 51e0997c2b
5 changed files with 335 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/qa-lab/package.json
+1 -1
View File
@@ -16,7 +16,7 @@
"@openclaw/plugin-sdk": "workspace:*",
"@openclaw/slack": "workspace:*",
"@openclaw/whatsapp": "workspace:*",
"@openclaw/crabline": "0.1.5",
"@openclaw/crabline": "0.1.6",
"openclaw": "2026.5.28"
},
"peerDependencies": {
extensions/qa-lab/src/runtime-tool-fixture.test.ts
+247
View File
@@ -776,6 +776,253 @@ describe("runtime tool fixture", () => {
expect(details).toContain("read mock provider failure planned args");
});
it("accepts non-required mock fixtures when both paths are planned without direct output", async () => {
const env = await makeEnv({
mock: { baseUrl: "http://127.0.0.1:9999" },
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
plannedToolCallId: "call-image-happy",
plannedToolName: "image_generate",
plannedToolArgs: { prompt: "QA lighthouse", filename: "runtime-tool-fixture" },
},
{
allInputText: "failure target=image_generate",
plannedToolCallId: "call-image-failure",
plannedToolName: "image_generate",
plannedToolArgs: { __qaFailureMode: "denied-input" },
},
]);
const details = await runRuntimeToolFixture(
env,
{
toolName: "image_generate",
toolCoverage: {
bucket: "openclaw-dynamic-integration",
expectedLayer: "openclaw-dynamic",
required: false,
action: "optional runtime parity gate with async image completion coverage",
},
promptSnippet: "target=image_generate",
failurePromptSnippet: "failure target=image_generate",
},
{
createSession: vi.fn(async (_env, _label, key) => key!),
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
runAgentPrompt: vi.fn(async () => ({})),
fetchJson,
ensureImageGenerationConfigured: vi.fn(),
},
);
expect(details).toContain("image_generate mock provider report-only");
expect(details).toContain("image_generate mock provider happy planned args");
expect(details).toContain("image_generate mock provider failure planned args");
});
it("still rejects failed happy output for non-required mock fixtures", async () => {
const env = await makeEnv({
mock: { baseUrl: "http://127.0.0.1:9999" },
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
plannedToolCallId: "call-image-happy",
plannedToolName: "image_generate",
plannedToolArgs: { prompt: "QA lighthouse" },
},
{
allInputText: "target=image_generate",
toolOutputCallId: "call-image-happy",
toolOutput: "Failed: provider rejected image request",
},
{
allInputText: "failure target=image_generate",
plannedToolCallId: "call-image-failure",
plannedToolName: "image_generate",
plannedToolArgs: { __qaFailureMode: "denied-input" },
},
]);
await expect(
runRuntimeToolFixture(
env,
{
toolName: "image_generate",
toolCoverage: {
bucket: "openclaw-dynamic-integration",
expectedLayer: "openclaw-dynamic",
required: false,
action: "optional runtime parity gate with async image completion coverage",
},
promptSnippet: "target=image_generate",
failurePromptSnippet: "failure target=image_generate",
},
{
createSession: vi.fn(async (_env, _label, key) => key!),
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
runAgentPrompt: vi.fn(async () => ({})),
fetchJson,
ensureImageGenerationConfigured: vi.fn(),
},
),
).rejects.toThrow("expected mock happy-path successful tool output for image_generate");
});
it("still rejects successful failure output for non-required mock fixtures", async () => {
const env = await makeEnv({
mock: { baseUrl: "http://127.0.0.1:9999" },
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
plannedToolCallId: "call-image-happy",
plannedToolName: "image_generate",
plannedToolArgs: { prompt: "QA lighthouse" },
},
{
allInputText: "failure target=image_generate",
plannedToolCallId: "call-image-failure",
plannedToolName: "image_generate",
plannedToolArgs: { __qaFailureMode: "denied-input" },
},
{
allInputText: "failure target=image_generate",
toolOutputCallId: "call-image-failure",
toolOutput: "Task queued for async image delivery",
},
]);
await expect(
runRuntimeToolFixture(
env,
{
toolName: "image_generate",
toolCoverage: {
bucket: "openclaw-dynamic-integration",
expectedLayer: "openclaw-dynamic",
required: false,
action: "optional runtime parity gate with async image completion coverage",
},
promptSnippet: "target=image_generate",
failurePromptSnippet: "failure target=image_generate",
},
{
createSession: vi.fn(async (_env, _label, key) => key!),
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
runAgentPrompt: vi.fn(async () => ({})),
fetchJson,
ensureImageGenerationConfigured: vi.fn(),
},
),
).rejects.toThrow("expected mock failure-path tool failure output for image_generate");
});
it("rejects malformed report-only failure plans for non-required mock fixtures", async () => {
const env = await makeEnv({
mock: { baseUrl: "http://127.0.0.1:9999" },
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
plannedToolCallId: "call-image-happy",
plannedToolName: "image_generate",
plannedToolArgs: { prompt: "QA lighthouse" },
},
{
allInputText: "failure target=image_generate",
plannedToolCallId: "call-image-failure",
plannedToolName: "image_generate",
plannedToolArgs: { prompt: "not a denied-input failure" },
},
]);
await expect(
runRuntimeToolFixture(
env,
{
toolName: "image_generate",
toolCoverage: {
bucket: "openclaw-dynamic-integration",
expectedLayer: "openclaw-dynamic",
required: false,
action: "optional runtime parity gate with async image completion coverage",
},
promptSnippet: "target=image_generate",
failurePromptSnippet: "failure target=image_generate",
},
{
createSession: vi.fn(async (_env, _label, key) => key!),
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
runAgentPrompt: vi.fn(async () => ({})),
fetchJson,
ensureImageGenerationConfigured: vi.fn(),
},
),
).rejects.toThrow("expected mock failure-path denied-input args for image_generate");
});
it("rejects malformed report-only happy plans for non-required mock fixtures", async () => {
const env = await makeEnv({
mock: { baseUrl: "http://127.0.0.1:9999" },
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
plannedToolCallId: "call-image-happy",
plannedToolName: "image_generate",
plannedToolArgs: {},
},
{
allInputText: "failure target=image_generate",
plannedToolCallId: "call-image-failure",
plannedToolName: "image_generate",
plannedToolArgs: { __qaFailureMode: "denied-input" },
},
]);
await expect(
runRuntimeToolFixture(
env,
{
toolName: "image_generate",
toolCoverage: {
bucket: "openclaw-dynamic-integration",
expectedLayer: "openclaw-dynamic",
required: false,
action: "optional runtime parity gate with async image completion coverage",
},
promptSnippet: "target=image_generate",
failurePromptSnippet: "failure target=image_generate",
},
{
createSession: vi.fn(async (_env, _label, key) => key!),
readEffectiveTools: vi.fn(async () => new Set(["image_generate"])),
runAgentPrompt: vi.fn(async () => ({})),
fetchJson,
ensureImageGenerationConfigured: vi.fn(),
},
),
).rejects.toThrow("expected mock happy-path prompt args for image_generate");
});
it("rejects failure-shaped mock happy-path tool output", async () => {
const env = await makeEnv({
mock: { baseUrl: "http://127.0.0.1:9999" },
extensions/qa-lab/src/runtime-tool-fixture.ts
+68 -13
View File
@@ -2,7 +2,10 @@
import fs from "node:fs/promises";
import path from "node:path";
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
import { readRuntimeToolCoverageMetadata } from "./runtime-tool-metadata.js";
import {
type QaRuntimeToolCoverageMetadata,
readRuntimeToolCoverageMetadata,
} from "./runtime-tool-metadata.js";
import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js";
import { readRawQaSessionStore } from "./suite-runtime-agent-session.js";
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
@@ -555,6 +558,37 @@ function formatCodexNativeWorkspaceDetails(params: {
.join("\n");
}
function formatReportOnlyMockDetails(params: {
toolName: string;
happyRequest: QaRuntimeToolFixtureRequest;
failureRequest: QaRuntimeToolFixtureRequest;
}) {
return [
`${params.toolName} mock provider report-only: direct tool output is not required by this fixture`,
`${params.toolName} mock provider happy planned args (diagnostic only): ${formatPlannedToolArgs(params.happyRequest.plannedToolArgs)}`,
`${params.toolName} mock provider failure planned args (diagnostic only): ${formatPlannedToolArgs(params.failureRequest.plannedToolArgs)}`,
].join("\n");
}
function isAsyncReportOnlyMockCoverage(metadata: QaRuntimeToolCoverageMetadata) {
return !metadata.required && /\basync\b/iu.test(metadata.action ?? "");
}
function plannedRequestHasDeniedInputFailure(request: QaRuntimeToolFixtureRequest) {
return (
isRecord(request.plannedToolArgs) &&
request.plannedToolArgs["__qaFailureMode"] === "denied-input"
);
}
function plannedRequestHasPrompt(request: QaRuntimeToolFixtureRequest) {
return (
isRecord(request.plannedToolArgs) &&
typeof request.plannedToolArgs.prompt === "string" &&
request.plannedToolArgs.prompt.trim().length > 0
);
}
function formatKnownHarnessGapDetails(toolName: string, config: QaRuntimeToolFixtureConfig) {
const knownHarnessGap = isKnownHarnessGap(config.knownHarnessGap) ? config.knownHarnessGap : {};
const issue = readString(knownHarnessGap.issue);
@@ -722,6 +756,39 @@ export async function runRuntimeToolFixture(
excludedPromptSnippet: failurePromptSnippet,
toolName,
});
const failurePlannedRequest = findPlannedRequest({
requests,
requestCountBefore,
promptSnippet: failurePromptSnippet,
toolName,
});
const failureRequest = findExecutedRequest({
requests,
requestCountBefore,
promptSnippet: failurePromptSnippet,
toolName,
});
if (
isAsyncReportOnlyMockCoverage(metadata) &&
happyPlannedRequest &&
failurePlannedRequest &&
!happyRequest
) {
if (!plannedRequestHasPrompt(happyPlannedRequest)) {
throw new Error(`expected mock happy-path prompt args for ${toolName}`);
}
if (!plannedRequestHasDeniedInputFailure(failurePlannedRequest)) {
throw new Error(`expected mock failure-path denied-input args for ${toolName}`);
}
if (failureRequest && !requestHasFailureLikeToolOutput(failureRequest.outputRequest)) {
throw new Error(`expected mock failure-path tool failure output for ${toolName}`);
}
return formatReportOnlyMockDetails({
toolName,
happyRequest: happyPlannedRequest,
failureRequest: failurePlannedRequest,
});
}
// Async runtime tools prove the start call here; completion is covered by
// their task lifecycle scenarios.
const happyPlannedOnly = Boolean(happyPlannedRequest && !happyPathOutputRequired);
@@ -749,18 +816,6 @@ export async function runRuntimeToolFixture(
}
throw new Error(`expected mock happy-path successful tool output for ${toolName}`);
}
const failurePlannedRequest = findPlannedRequest({
requests,
requestCountBefore,
promptSnippet: failurePromptSnippet,
toolName,
});
const failureRequest = findExecutedRequest({
requests,
requestCountBefore,
promptSnippet: failurePromptSnippet,
toolName,
});
if (!failureRequest) {
if (dynamicExposureIntentionallyExcluded) {
return formatCodexNativeWorkspaceDetails({
pnpm-lock.yaml
Generated
+18 -27
View File
@@ -1346,8 +1346,8 @@ importers:
version: 4.4.3
devDependencies:
'@openclaw/crabline':
specifier: 0.1.5
version: 0.1.5
specifier: 0.1.6
version: 0.1.6
'@openclaw/discord':
specifier: workspace:*
version: link:../discord
@@ -2368,18 +2368,10 @@ packages:
resolution: {integrity: sha512-fT1qHVGAag4IEkrupZ6lRRbNCs1vS9P01KB/sG8zKgvUztbYtFBtQpjSITNwooDZ83tpsPzP0mRNs1/KVszCRA==}
engines: {node: '>= 20.12.0'}
'@clack/core@1.4.1':
resolution: {integrity: sha512-FILJa1gGKEFTGZAJE9RpVhrjKz3c3h4ar60dSv6cGuDqufQ84YEIS3GAGvZiN+H6yaLbbvTFNejjCC4tXpZEuw==}
engines: {node: '>= 20.12.0'}
'@clack/prompts@1.4.0':
resolution: {integrity: sha512-S0My7XPGIgpRWMDG8uRqalbgT+a6FmCUdOW+HaIOVVpUPHOb7RrpvjTjiODadKp06fsrVDJZlIzc6yCTp4AnxA==}
engines: {node: '>= 20.12.0'}
'@clack/prompts@1.5.1':
resolution: {integrity: sha512-zccHj2z2oCCO4yrDiRSlFOxWerGqRiysP7a5jPK6uoI9URKAquwY42Dd/iUP8JWHxEzdRe4TlbvZCo8z1/mhrw==}
engines: {node: '>= 20.12.0'}
'@clawdbot/lobster@2026.5.22':
resolution: {integrity: sha512-lrUnsLLmo4sVFDNd7oTQiEnt4whGhy1bDHe3s/sQoInbTLJSN55WKxlUCfberZbzwOq8GSNnBkP0ZhZ3mKSBwg==}
engines: {node: '>=20'}
@@ -3182,8 +3174,8 @@ packages:
cpu: [x64]
os: [win32]
'@openclaw/crabline@0.1.5':
resolution: {integrity: sha512-OK1YUdM6GO1qgTEvcHhMl/MQ77yw9MmcM84jB1uPNXutQnIjPbIhRin6MXRSMbuYqH5feu4CmTzAtH5VtHv8gA==}
'@openclaw/crabline@0.1.6':
resolution: {integrity: sha512-nu/XD7eoly5DJOEG7krCfZY68cDcD/AC31mAMLoP36HzmetNVM17OtfnakpuWyBwdO+c4noF7b2LM4AAKvq9CQ==}
engines: {node: '>=22'}
hasBin: true
@@ -4581,6 +4573,7 @@ packages:
'@zed-industries/codex-acp@0.15.0':
resolution: {integrity: sha512-eAv7sGBeiYrYkOulF729nrM51szS7WIhBtugRj5wWq6csRKZUhAZfoUZlF8xUWdHPtOIzd/eT6MNG6gMHu6z0w==}
deprecated: This package has been replaced by @agentclientprotocol/codex-acp. Please migrate to continue receiving updates.
hasBin: true
abort-controller@3.0.0:
@@ -7530,6 +7523,10 @@ packages:
resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==}
engines: {node: '>=20.18.1'}
undici@8.3.0:
resolution: {integrity: sha512-TkUDgb6tl7KOGZ+7e8E3d2FYgUQgF6z5YypqjWmixVQSQERFcVrVg0ySADm2LVLRh5ljAaHTCR5Fmz3Q34rB7Q==}
engines: {node: '>=22.19.0'}
undici@8.5.0:
resolution: {integrity: sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==}
engines: {node: '>=22.19.0'}
@@ -8518,11 +8515,6 @@ snapshots:
fast-wrap-ansi: 0.2.2
sisteransi: 1.0.5
'@clack/core@1.4.1':
dependencies:
fast-wrap-ansi: 0.2.2
sisteransi: 1.0.5
'@clack/prompts@1.4.0':
dependencies:
'@clack/core': 1.3.1
@@ -8530,13 +8522,6 @@ snapshots:
fast-wrap-ansi: 0.2.2
sisteransi: 1.0.5
'@clack/prompts@1.5.1':
dependencies:
'@clack/core': 1.4.1
fast-string-width: 3.0.2
fast-wrap-ansi: 0.2.2
sisteransi: 1.0.5
'@clawdbot/lobster@2026.5.22':
dependencies:
ajv: 8.20.0
@@ -9269,7 +9254,7 @@ snapshots:
'@openai/codex@0.139.0-win32-x64':
optional: true
'@openclaw/crabline@0.1.5':
'@openclaw/crabline@0.1.6':
dependencies:
commander: 15.0.0
curve25519-js: 0.0.4
@@ -9286,6 +9271,10 @@ snapshots:
jszip: 3.10.1
tar: 7.5.16
'@openclaw/proxyline@0.3.3(undici@8.3.0)':
dependencies:
undici: 8.3.0
'@openclaw/proxyline@0.3.3(undici@8.5.0)':
dependencies:
undici: 8.5.0
@@ -12966,7 +12955,7 @@ snapshots:
'@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3)
'@mozilla/readability': 0.6.0
'@openclaw/fs-safe': 0.3.0
'@openclaw/proxyline': 0.3.3(undici@8.5.0)
'@openclaw/proxyline': 0.3.3(undici@8.3.0)
'@silvia-odwyer/photon-node': 0.3.4
chalk: 5.6.2
chokidar: 5.0.0
@@ -13004,7 +12993,7 @@ snapshots:
tslog: 4.10.2
typebox: 1.1.39
typescript: 6.0.3
undici: 8.5.0
undici: 8.3.0
web-push: 3.6.7
web-tree-sitter: 0.26.9
ws: 8.21.0
@@ -14142,6 +14131,8 @@ snapshots:
undici@7.28.0: {}
undici@8.3.0: {}
undici@8.5.0: {}
unhomoglyph@1.0.6: {}
pnpm-workspace.yaml
+1 -1
View File
@@ -7,7 +7,7 @@ packages:
minimumReleaseAge: 2880
minimumReleaseAgeExclude:
- "@openclaw/crabline@0.1.5"
- "@openclaw/crabline@0.1.6"
- "@openclaw/fs-safe@0.3.0"
- "@openclaw/proxyline@0.3.3"
- "acpx"