test(qa): harden all-profile evidence scenarios (#96003)

.github/workflows/qa-profile-evidence.yml

@@ -243,6 +243,9 @@ jobs:
           NODE_OPTIONS: --max-old-space-size=8192
         run: node scripts/build-all.mjs qaRuntime
 
+      - name: Ensure Playwright Chromium
+        run: node scripts/ensure-playwright-chromium.mjs
+
       - name: Run QA profile
         id: run_profile
         env:

extensions/qa-lab/src/providers/mock-openai/server.test.ts

@@ -3314,7 +3314,7 @@ describe("qa mock openai server", () => {
     const toolPlanOutput = outputItem(await response.json());
     expect(toolPlanOutput.type).toBe("function_call");
     expect(toolPlanOutput.name).toBe("web_search");
-    expect(String(toolPlanOutput.arguments)).toContain("denied-input");
+    expect(String(toolPlanOutput.arguments)).toContain("OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT");
   });
 
   it("plans QA subagent handoff calls even when Codex dynamic tools are not in body.tools", async () => {

extensions/qa-lab/src/providers/mock-openai/server.ts

@@ -5,6 +5,7 @@ import { setTimeout as sleep } from "node:timers/promises";
 import { escapeRegExp } from "openclaw/plugin-sdk/text-utility-runtime";
 import { readRequestBodyWithLimit } from "openclaw/plugin-sdk/webhook-ingress";
 import { closeQaHttpServer } from "../../bus-server.js";
+import { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY } from "../../qa-web-search-provider.js";
 import { writeJson } from "../shared/http-json.js";
 
 type ResponsesInputItem = Record<string, unknown>;
@@ -860,6 +861,9 @@ function extractToolSearchTarget(text: string): string | null {
 }
 
 function buildQaToolSearchArgs(targetTool: string, failureMode: boolean): Record<string, unknown> {
+  if (failureMode && targetTool === "web_search") {
+    return { query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY };
+  }
   if (failureMode) {
     return { __qaFailureMode: "denied-input" };
   }
@@ -1535,49 +1539,57 @@ function buildToolCallEvents(prompt: string): StreamEvent[] {
 function buildReleaseAuditJson() {
   return `${JSON.stringify(
     {
-      verified: true,
+      verified: false,
       findings: [
         {
           id: "REL-GATEWAY-417",
           source: "src/gateway/reconnect.ts",
           status: "retry jitter verified, resume token fallback still needs manual spot check",
+          verified: true,
         },
         {
           id: "REL-CHANNEL-238",
           source: "src/channels/delivery.ts",
           status: "thread replies preserve ordering, root-channel fallback needs handoff note",
+          verified: true,
         },
         {
           id: "REL-CRON-904",
           source: "src/scheduling/cron.ts",
           status: "single-run lock verified for restart wakeups",
+          verified: true,
         },
         {
           id: "REL-MEMORY-552",
           source: "src/memory/recall.ts",
           status:
             "fallback summary survives empty memory search; ranking sample needs second reviewer",
+          verified: true,
         },
         {
           id: "REL-PLUGIN-319",
           source: "src/plugins/runtime.ts",
           status: "bundled runtime manifest loads cleanly after restart",
+          verified: true,
         },
         {
           id: "REL-INSTALL-846",
           source: "install/update.ts",
           status: "update smoke passed from previous stable tag",
+          verified: true,
         },
         {
           id: "REL-DOCS-611",
           source: "docs/operator-notes.md",
           status:
             "docs mention reconnect, cron, memory, plugin, and installer checks; channel ordering and UI notes need maintainer handoff",
+          verified: true,
         },
         {
           id: "REL-UI-BLOCKED",
           source: "ui/control-panel.ts",
           status: "blocked: source file was referenced by checklist but missing from the fixture",
+          verified: false,
         },
       ],
     },

extensions/qa-lab/src/qa-web-search-provider.test.ts

@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
 import { createQaLabWebSearchProvider as createQaLabWebSearchContractProvider } from "../web-search-contract-api.js";
 import {
   createQaLabWebSearchProvider,
+  QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY,
   QA_LAB_WEB_SEARCH_PROVIDER_ID,
 } from "./qa-web-search-provider.js";
 
@@ -55,4 +56,16 @@ describe("qa-lab web search provider", () => {
 
     await expect(tool.execute({ __qaFailureMode: "denied-input" })).rejects.toThrow(/query/i);
   });
+
+  it("keeps the QA failure sentinel as a deterministic tool failure", async () => {
+    const provider = createQaLabWebSearchProvider();
+    const tool = provider.createTool({});
+    if (!tool) {
+      throw new Error("expected QA Lab web search tool");
+    }
+
+    await expect(tool.execute({ query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY })).rejects.toThrow(
+      /denied input sentinel/i,
+    );
+  });
 });

extensions/qa-lab/src/qa-web-search-provider.ts

@@ -9,6 +9,7 @@ import {
 } from "openclaw/plugin-sdk/provider-web-search";
 
 export const QA_LAB_WEB_SEARCH_PROVIDER_ID = "qa-lab-search";
+export const QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY = "OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT";
 
 const QaLabWebSearchSchema = {
   type: "object",
@@ -64,6 +65,9 @@ export function createQaLabWebSearchProvider(): WebSearchProviderPlugin {
       parameters: QaLabWebSearchSchema,
       execute: async (args) => {
         const query = readStringParam(args, "query", { required: true });
+        if (query === QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY) {
+          throw new Error("QA Lab web_search denied input sentinel");
+        }
         const count =
           readPositiveIntegerParam(args, "count", {
             max: MAX_SEARCH_COUNT,

qa/scenarios/memory/commitments-heartbeat-target-none.yaml

@@ -55,7 +55,7 @@ flow:
         - call: reset
         - set: beforeHeartbeatTs
           value:
-            expr: "((await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 30000) }))?.ts ?? 0)"
+            expr: "((await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 15000) }))?.ts ?? 0)"
         - set: sessionKey
           value:
             expr: "`agent:qa:qa-channel:${config.conversationId}`"
@@ -106,7 +106,7 @@ flow:
           args:
             - lambda:
                 async: true
-                expr: "(async () => { const last = await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 30000) }); return last && last.ts > beforeHeartbeatTs ? last : undefined; })()"
+                expr: "(async () => { const last = await env.gateway.call('last-heartbeat', {}, { timeoutMs: liveTurnTimeoutMs(env, 15000) }); return last && last.ts > beforeHeartbeatTs ? last : undefined; })()"
             - expr: liveTurnTimeoutMs(env, 45000)
             - 250
         - call: sleep

qa/scenarios/runtime/tools/web-search.yaml

@@ -44,6 +44,7 @@ scenario:
         action: hard gate in the standard direct-loading tier
         reason: web_search is an OpenClaw integration tool and must stay visible and callable under OpenClaw and Codex direct runtime parity.
       promptSnippet: "target=web_search"
+      failurePrompt: "tool search qa failure target=web_search. Call web_search exactly once with query OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT and then summarize the failure."
       failurePromptSnippet: "failure target=web_search"
 
 flow:

qa/scenarios/workspace/long-running-release-audit.yaml

@@ -14,6 +14,7 @@ scenario:
     - Agent starts from the seeded project README instead of guessing.
     - Agent inspects docs and source files across multiple directories.
     - Agent writes a JSON audit and a Markdown handoff with all required findings.
+    - Each JSON finding carries a boolean verified field; missing current source evidence is verified false.
     - Agent verifies the generated JSON before finalizing.
   docsRefs:
     - docs/help/testing.md
@@ -131,7 +132,8 @@ scenario:
         Do a release readiness audit for the small project under `audit-fixture/`.
         Start from `audit-fixture/README.md`, find the current checklist, inspect the referenced docs/source, then create `audit-fixture/release-audit.json` and `audit-fixture/release-handoff.md`.
 
-        The JSON should include current finding ids, source files, statuses, and a boolean `verified`.
+        The JSON should include current finding ids, source files, statuses, and a boolean `verified` on each finding.
+        If a current checklist source is referenced but missing from the fixture, include that source as a finding with `verified: false`.
         The Markdown handoff should summarize what is ready and what needs follow-up.
         Check your generated JSON before finalizing.
         Final reply exactly: RELEASE-AUDIT-COMPLETE
@@ -211,14 +213,20 @@ flow:
             expr: "!JSON.stringify(Array.isArray(report.findings) ? report.findings : report).includes('REL-STALE-000') && !handoffText.includes('REL-STALE-000')"
             message:
               expr: "`stale archive finding leaked into audit: report=${reportText}\\nhandoff=${handoffText}`"
+        - set: reportFindings
+          value:
+            expr: "Array.isArray(report) ? report : (Array.isArray(report.findings) ? report.findings : [])"
+        - set: uiFinding
+          value:
+            expr: "reportFindings.find((finding) => JSON.stringify(finding).includes('ui/control-panel.ts'))"
         - assert:
-            expr: "JSON.stringify(report).includes('ui/control-panel.ts') && /blocked|missing|not found|no current source file|no matching source file/i.test(`${reportText}\\n${handoffText}`)"
+            expr: "uiFinding && uiFinding.verified === false"
             message:
-              expr: "`missing UI evidence was not explicitly blocked: report=${reportText}\\nhandoff=${handoffText}`"
+              expr: "`missing UI evidence was not marked unverified: report=${reportText}\\nhandoff=${handoffText}`"
         - assert:
-            expr: "JSON.stringify(report).includes('verified')"
+            expr: "reportFindings.length > 0 && reportFindings.every((finding) => typeof finding?.verified === 'boolean')"
             message:
-              expr: "`report did not include a verification field: ${reportText}`"
+              expr: "`each report finding must include a boolean verified field: ${reportText}`"
         - call: waitForAgentHistoryReply
           saveAs: outbound
           args:

test/scripts/ci-workflow-guards.test.ts

@@ -619,6 +619,10 @@ describe("ci workflow guards", () => {
       "taxonomy.profiles.find((entry) => entry.id === requested)",
     );
     expect(validateProfileStep.run).toContain("profile=${profile.id}");
+    const ensurePlaywrightStep = qaRunJob.steps.find(
+      (step) => step.name === "Ensure Playwright Chromium",
+    );
+    expect(ensurePlaywrightStep.run).toBe("node scripts/ensure-playwright-chromium.mjs");
     expect(generateJob.if).toBe("${{ inputs.qa_evidence_run_id == '' }}");
     expect(generateJob.uses).toBe("./.github/workflows/qa-profile-evidence.yml");
     expect(generateJob.with).toMatchObject({