diff --git a/.github/workflows/qa-profile-evidence.yml b/.github/workflows/qa-profile-evidence.yml index 5a15f87c59e5..93dab2fde2a7 100644 --- a/.github/workflows/qa-profile-evidence.yml +++ b/.github/workflows/qa-profile-evidence.yml @@ -243,6 +243,9 @@ jobs: NODE_OPTIONS: --max-old-space-size=8192 run: node scripts/build-all.mjs qaRuntime + - name: Ensure Playwright Chromium + run: node scripts/ensure-playwright-chromium.mjs + - name: Run QA profile id: run_profile env: diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index 8b7985337382..cace09e0d5d2 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -3314,7 +3314,7 @@ describe("qa mock openai server", () => { const toolPlanOutput = outputItem(await response.json()); expect(toolPlanOutput.type).toBe("function_call"); expect(toolPlanOutput.name).toBe("web_search"); - expect(String(toolPlanOutput.arguments)).toContain("denied-input"); + expect(String(toolPlanOutput.arguments)).toContain("OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT"); }); it("plans QA subagent handoff calls even when Codex dynamic tools are not in body.tools", async () => { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index c7c49ea4f433..c7a41e071c00 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -5,6 +5,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import { escapeRegExp } from "openclaw/plugin-sdk/text-utility-runtime"; import { readRequestBodyWithLimit } from "openclaw/plugin-sdk/webhook-ingress"; import { closeQaHttpServer } from "../../bus-server.js"; +import { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY } from "../../qa-web-search-provider.js"; import { writeJson } from "../shared/http-json.js"; type ResponsesInputItem = Record; @@ -860,6 +861,9 @@ function extractToolSearchTarget(text: string): string | null { } function buildQaToolSearchArgs(targetTool: string, failureMode: boolean): Record { + if (failureMode && targetTool === "web_search") { + return { query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY }; + } if (failureMode) { return { __qaFailureMode: "denied-input" }; } @@ -1535,49 +1539,57 @@ function buildToolCallEvents(prompt: string): StreamEvent[] { function buildReleaseAuditJson() { return `${JSON.stringify( { - verified: true, + verified: false, findings: [ { id: "REL-GATEWAY-417", source: "src/gateway/reconnect.ts", status: "retry jitter verified, resume token fallback still needs manual spot check", + verified: true, }, { id: "REL-CHANNEL-238", source: "src/channels/delivery.ts", status: "thread replies preserve ordering, root-channel fallback needs handoff note", + verified: true, }, { id: "REL-CRON-904", source: "src/scheduling/cron.ts", status: "single-run lock verified for restart wakeups", + verified: true, }, { id: "REL-MEMORY-552", source: "src/memory/recall.ts", status: "fallback summary survives empty memory search; ranking sample needs second reviewer", + verified: true, }, { id: "REL-PLUGIN-319", source: "src/plugins/runtime.ts", status: "bundled runtime manifest loads cleanly after restart", + verified: true, }, { id: "REL-INSTALL-846", source: "install/update.ts", status: "update smoke passed from previous stable tag", + verified: true, }, { id: "REL-DOCS-611", source: "docs/operator-notes.md", status: "docs mention reconnect, cron, memory, plugin, and installer checks; channel ordering and UI notes need maintainer handoff", + verified: true, }, { id: "REL-UI-BLOCKED", source: "ui/control-panel.ts", status: "blocked: source file was referenced by checklist but missing from the fixture", + verified: false, }, ], }, diff --git a/extensions/qa-lab/src/qa-web-search-provider.test.ts b/extensions/qa-lab/src/qa-web-search-provider.test.ts index 820ee72bee1f..115428059e31 100644 --- a/extensions/qa-lab/src/qa-web-search-provider.test.ts +++ b/extensions/qa-lab/src/qa-web-search-provider.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { createQaLabWebSearchProvider as createQaLabWebSearchContractProvider } from "../web-search-contract-api.js"; import { createQaLabWebSearchProvider, + QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY, QA_LAB_WEB_SEARCH_PROVIDER_ID, } from "./qa-web-search-provider.js"; @@ -55,4 +56,16 @@ describe("qa-lab web search provider", () => { await expect(tool.execute({ __qaFailureMode: "denied-input" })).rejects.toThrow(/query/i); }); + + it("keeps the QA failure sentinel as a deterministic tool failure", async () => { + const provider = createQaLabWebSearchProvider(); + const tool = provider.createTool({}); + if (!tool) { + throw new Error("expected QA Lab web search tool"); + } + + await expect(tool.execute({ query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY })).rejects.toThrow( + /denied input sentinel/i, + ); + }); }); diff --git a/extensions/qa-lab/src/qa-web-search-provider.ts b/extensions/qa-lab/src/qa-web-search-provider.ts index ba11c0b9b77d..e9f29115fa11 100644 --- a/extensions/qa-lab/src/qa-web-search-provider.ts +++ b/extensions/qa-lab/src/qa-web-search-provider.ts @@ -9,6 +9,7 @@ import { } from "openclaw/plugin-sdk/provider-web-search"; export const QA_LAB_WEB_SEARCH_PROVIDER_ID = "qa-lab-search"; +export const QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY = "OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT"; const QaLabWebSearchSchema = { type: "object", @@ -64,6 +65,9 @@ export function createQaLabWebSearchProvider(): WebSearchProviderPlugin { parameters: QaLabWebSearchSchema, execute: async (args) => { const query = readStringParam(args, "query", { required: true }); + if (query === QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY) { + throw new Error("QA Lab web_search denied input sentinel"); + } const count = readPositiveIntegerParam(args, "count", { max: MAX_SEARCH_COUNT, diff --git a/qa/scenarios/memory/commitments-heartbeat-target-none.yaml b/qa/scenarios/memory/commitments-heartbeat-target-none.yaml index a1ffb3513c3a..0f61c07899b1 100644 --- a/qa/scenarios/memory/commitments-heartbeat-target-none.yaml +++ b/qa/scenarios/memory/commitments-heartbeat-target-none.yaml @@ -55,7 +55,7 @@ flow: - call: reset - set: beforeHeartbeatTs value: - expr: "((await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 30000) }))?.ts ?? 0)" + expr: "((await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 15000) }))?.ts ?? 0)" - set: sessionKey value: expr: "`agent:qa:qa-channel:${config.conversationId}`" @@ -106,7 +106,7 @@ flow: args: - lambda: async: true - expr: "(async () => { const last = await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 30000) }); return last && last.ts > beforeHeartbeatTs ? last : undefined; })()" + expr: "(async () => { const last = await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 15000) }); return last && last.ts > beforeHeartbeatTs ? last : undefined; })()" - expr: liveTurnTimeoutMs(env, 45000) - 250 - call: sleep diff --git a/qa/scenarios/runtime/tools/web-search.yaml b/qa/scenarios/runtime/tools/web-search.yaml index 50f397571f28..15170fea0387 100644 --- a/qa/scenarios/runtime/tools/web-search.yaml +++ b/qa/scenarios/runtime/tools/web-search.yaml @@ -44,6 +44,7 @@ scenario: action: hard gate in the standard direct-loading tier reason: web_search is an OpenClaw integration tool and must stay visible and callable under OpenClaw and Codex direct runtime parity. promptSnippet: "target=web_search" + failurePrompt: "tool search qa failure target=web_search. Call web_search exactly once with query OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT and then summarize the failure." failurePromptSnippet: "failure target=web_search" flow: diff --git a/qa/scenarios/workspace/long-running-release-audit.yaml b/qa/scenarios/workspace/long-running-release-audit.yaml index d51edd90cd08..d902ab592dd7 100644 --- a/qa/scenarios/workspace/long-running-release-audit.yaml +++ b/qa/scenarios/workspace/long-running-release-audit.yaml @@ -14,6 +14,7 @@ scenario: - Agent starts from the seeded project README instead of guessing. - Agent inspects docs and source files across multiple directories. - Agent writes a JSON audit and a Markdown handoff with all required findings. + - Each JSON finding carries a boolean verified field; missing current source evidence is verified false. - Agent verifies the generated JSON before finalizing. docsRefs: - docs/help/testing.md @@ -131,7 +132,8 @@ scenario: Do a release readiness audit for the small project under `audit-fixture/`. Start from `audit-fixture/README.md`, find the current checklist, inspect the referenced docs/source, then create `audit-fixture/release-audit.json` and `audit-fixture/release-handoff.md`. - The JSON should include current finding ids, source files, statuses, and a boolean `verified`. + The JSON should include current finding ids, source files, statuses, and a boolean `verified` on each finding. + If a current checklist source is referenced but missing from the fixture, include that source as a finding with `verified: false`. The Markdown handoff should summarize what is ready and what needs follow-up. Check your generated JSON before finalizing. Final reply exactly: RELEASE-AUDIT-COMPLETE @@ -211,14 +213,20 @@ flow: expr: "!JSON.stringify(Array.isArray(report.findings) ? report.findings : report).includes('REL-STALE-000') && !handoffText.includes('REL-STALE-000')" message: expr: "`stale archive finding leaked into audit: report=${reportText}\\nhandoff=${handoffText}`" + - set: reportFindings + value: + expr: "Array.isArray(report) ? report : (Array.isArray(report.findings) ? report.findings : [])" + - set: uiFinding + value: + expr: "reportFindings.find((finding) => JSON.stringify(finding).includes('ui/control-panel.ts'))" - assert: - expr: "JSON.stringify(report).includes('ui/control-panel.ts') && /blocked|missing|not found|no current source file|no matching source file/i.test(`${reportText}\\n${handoffText}`)" + expr: "uiFinding && uiFinding.verified === false" message: - expr: "`missing UI evidence was not explicitly blocked: report=${reportText}\\nhandoff=${handoffText}`" + expr: "`missing UI evidence was not marked unverified: report=${reportText}\\nhandoff=${handoffText}`" - assert: - expr: "JSON.stringify(report).includes('verified')" + expr: "reportFindings.length > 0 && reportFindings.every((finding) => typeof finding?.verified === 'boolean')" message: - expr: "`report did not include a verification field: ${reportText}`" + expr: "`each report finding must include a boolean verified field: ${reportText}`" - call: waitForAgentHistoryReply saveAs: outbound args: diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 77702b6f8ee0..b9fec8366397 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -619,6 +619,10 @@ describe("ci workflow guards", () => { "taxonomy.profiles.find((entry) => entry.id === requested)", ); expect(validateProfileStep.run).toContain("profile=${profile.id}"); + const ensurePlaywrightStep = qaRunJob.steps.find( + (step) => step.name === "Ensure Playwright Chromium", + ); + expect(ensurePlaywrightStep.run).toBe("node scripts/ensure-playwright-chromium.mjs"); expect(generateJob.if).toBe("${{ inputs.qa_evidence_run_id == '' }}"); expect(generateJob.uses).toBe("./.github/workflows/qa-profile-evidence.yml"); expect(generateJob.with).toMatchObject({