test(qa): speed up smoke profile (#96340)

extensions/qa-lab/src/cli.runtime.test.ts

@@ -558,6 +558,8 @@ describe("qa cli runtime", () => {
         "qa-channel-reconnect-dedupe",
         "reaction-edit-delete",
         "thread-follow-up",
+        "claude-cli-provider-capabilities",
+        "claude-cli-provider-capabilities-subscription",
         "image-generation-roundtrip",
         "image-understanding-attachment",
         "native-image-generation",

extensions/qa-lab/src/providers/mock-openai/server.test.ts

@@ -1846,6 +1846,52 @@ describe("qa mock openai server", () => {
     expect(memorySearch.status).toBe(200);
     expect(await memorySearch.text()).toContain('"name":"memory_search"');
 
+    const memoryGetFromPathOnlySearchResult = await fetch(`${server.baseUrl}/v1/responses`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+      },
+      body: JSON.stringify({
+        stream: true,
+        input: [
+          {
+            role: "user",
+            content: [
+              {
+                type: "input_text",
+                text: "Memory tools check: what is the hidden project codename stored only in memory? Use memory tools first.",
+              },
+            ],
+          },
+          {
+            type: "function_call_output",
+            output: JSON.stringify({
+              results: [
+                {
+                  path: "MEMORY.md",
+                  snippet: "Hidden QA fact: the project codename is ORBIT-9.",
+                },
+              ],
+            }),
+          },
+          {
+            role: "user",
+            content: [
+              {
+                type: "input_text",
+                text: "Protocol note: acknowledged. Continue with the QA scenario plan.",
+              },
+            ],
+          },
+        ],
+      }),
+    });
+    expect(memoryGetFromPathOnlySearchResult.status).toBe(200);
+    const memoryGetText = await memoryGetFromPathOnlySearchResult.text();
+    expect(memoryGetText).toContain('"name":"memory_get"');
+    expect(memoryGetText).toContain('\\"path\\":\\"MEMORY.md\\"');
+    expect(memoryGetText).toContain('\\"from\\":1');
+
     const image = await fetch(`${server.baseUrl}/v1/images/generations`, {
       method: "POST",
       headers: {

extensions/qa-lab/src/providers/mock-openai/server.ts

@@ -2612,8 +2612,8 @@ async function buildResponsesPayload(
       });
     }
   }
-  if (/memory tools check/i.test(prompt)) {
-    if (!toolOutput) {
+  if (/memory tools check/i.test(allInputText)) {
+    if (!scenarioToolOutput) {
       return buildToolCallEventsWithArgs("memory_search", {
         query: "project codename ORBIT-9",
         maxResults: 3,
@@ -2623,10 +2623,7 @@ async function buildResponsesPayload(
       ? (toolJson.results as Array<Record<string, unknown>>)
       : [];
     const first = results[0];
-    if (
-      typeof first?.path === "string" &&
-      (typeof first.startLine === "number" || typeof first.endLine === "number")
-    ) {
+    if (typeof first?.path === "string") {
       const from =
         typeof first.startLine === "number"
           ? Math.max(1, first.startLine)

extensions/qa-lab/src/suite-launch.runtime.test.ts

@@ -469,6 +469,94 @@ describe("qa suite runtime launcher", () => {
     expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
   });
 
+  it("starts native suite proof before isolated flow work fills the weighted queue", async () => {
+    const repoRoot = await makeTempRepo("qa-suite-native-before-isolated-");
+    let releaseShared!: () => void;
+    let markSharedStarted!: () => void;
+    const sharedStarted = new Promise<void>((resolve) => {
+      markSharedStarted = resolve;
+    });
+    const sharedBlocked = new Promise<void>((resolve) => {
+      releaseShared = resolve;
+    });
+    let releaseTestFile!: () => void;
+    let markTestFileStarted!: () => void;
+    const testFileStarted = new Promise<void>((resolve) => {
+      markTestFileStarted = resolve;
+    });
+    const testFileBlocked = new Promise<void>((resolve) => {
+      releaseTestFile = resolve;
+    });
+    runQaFlowSuite.mockImplementationOnce(
+      async (params: { outputDir?: string; scenarioIds?: string[] } | undefined) => {
+        markSharedStarted();
+        await sharedBlocked;
+        const outputDir = params?.outputDir ?? "/tmp/qa-flow";
+        const evidencePath = path.join(outputDir, "qa-evidence.json");
+        await writeEvidence(evidencePath);
+        const scenarioIds = params?.scenarioIds ?? ["channel-chat-baseline"];
+        return {
+          outputDir,
+          evidencePath,
+          reportPath: path.join(outputDir, "qa-suite-report.md"),
+          summaryPath: path.join(outputDir, "qa-suite-summary.json"),
+          report: "# QA Suite Report\n",
+          scenarios: scenarioIds.map((scenarioId) => ({
+            name: scenarioId,
+            status: "pass",
+            steps: [],
+          })),
+          watchUrl: "http://127.0.0.1:43124",
+        };
+      },
+    );
+    runQaTestFileScenarios.mockImplementationOnce(
+      async (params: {
+        outputDir: string;
+        scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>;
+      }) => {
+        markTestFileStarted();
+        await testFileBlocked;
+        const evidencePath = path.join(params.outputDir, "qa-evidence.json");
+        await writeEvidence(evidencePath);
+        return {
+          outputDir: params.outputDir,
+          executionKind: params.scenarios[0]?.execution.kind ?? "playwright",
+          evidencePath,
+          results: params.scenarios.map((scenarioItem) => ({
+            durationMs: 1,
+            logPath: path.join(params.outputDir, `${scenarioItem.id}.log`),
+            scenario: scenarioItem,
+            status: "pass",
+          })),
+        };
+      },
+    );
+
+    const runPromise = runQaSuite({
+      repoRoot,
+      outputDir: ".artifacts/qa-e2e/native-before-isolated",
+      concurrency: 2,
+      scenarioIds: [
+        "channel-chat-baseline",
+        "group-visible-reply-tool",
+        "control-ui-chat-flow-playwright",
+      ],
+    });
+    await sharedStarted;
+    await testFileStarted;
+    await Promise.resolve();
+
+    expect(runQaFlowSuite).toHaveBeenCalledTimes(1);
+    expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
+
+    releaseTestFile();
+    releaseShared();
+    await runPromise;
+
+    expect(runQaFlowSuite).toHaveBeenCalledTimes(2);
+  });
+
   it("waits for already-started partitions before rejecting a unified suite", async () => {
     const repoRoot = await makeTempRepo("qa-suite-reject-settle-");
     let releaseTestFile!: () => void;

extensions/qa-lab/src/suite-launch.runtime.ts

@@ -448,7 +448,9 @@ async function runUnifiedQaSuite(params: {
   );
   const evidenceSummaries: QaEvidenceSummaryJson[] = [];
   const scenarioResultsById = new Map<string, QaSuiteScenarioResult>();
-  const partitionTasks: QaUnifiedPartitionTask[] = [];
+  const sharedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
+  const isolatedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
+  const testFilePartitionTasks: QaUnifiedPartitionTask[] = [];
   if (params.plan.flowScenarios.length > 0) {
     const sharedFlowScenarios = params.plan.flowScenarios.filter(
       (scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario),
@@ -488,7 +490,7 @@ async function runUnifiedQaSuite(params: {
     for (const partition of flowPartitions) {
       const isolatedPartition =
         partition.kind === "isolated" || partition.kind.startsWith("isolated-");
-      partitionTasks.push({
+      const task = {
         weight: partition.concurrency,
         run: async () => {
           const result = await runFlowSuite({
@@ -525,11 +527,16 @@ async function runUnifiedQaSuite(params: {
             scenarioResults,
           };
         },
-      });
+      } satisfies QaUnifiedPartitionTask;
+      if (isolatedPartition) {
+        isolatedFlowPartitionTasks.push(task);
+      } else {
+        sharedFlowPartitionTasks.push(task);
+      }
     }
   }
   if (params.plan.testFileScenariosByKind.size > 0) {
-    partitionTasks.push({
+    testFilePartitionTasks.push({
       weight: 1,
       run: async () => {
         const testFileEvidenceSummaries: QaEvidenceSummaryJson[] = [];
@@ -561,6 +568,11 @@ async function runUnifiedQaSuite(params: {
       },
     });
   }
+  const partitionTasks = [
+    ...sharedFlowPartitionTasks,
+    ...testFilePartitionTasks,
+    ...isolatedFlowPartitionTasks,
+  ];
   const partitionResults = await runWeightedUnifiedPartitionTasks(partitionTasks, concurrency);
   for (const partitionResult of partitionResults) {
     for (const scenarioResult of partitionResult.scenarioResults) {

qa/scenarios/memory/memory-tools-channel-context.yaml

@@ -8,10 +8,9 @@ scenario:
       - memory.tools
     secondary:
       - channels.group-messages
-  objective: Verify the agent uses memory_search and memory_get in a shared channel when the answer lives only in memory files, not the live transcript.
+  objective: Verify the agent uses memory tools in a shared channel when the answer lives only in memory files, not the live transcript.
   successCriteria:
     - Agent uses memory_search before answering.
-    - Agent narrows with memory_get before answering.
     - Final reply returns the memory-only fact correctly in-channel.
   docsRefs:
     - docs/concepts/memory.md
@@ -21,7 +20,7 @@ scenario:
     - extensions/qa-lab/src/suite.ts
   execution:
     kind: flow
-    summary: Verify the agent uses memory_search and memory_get in a shared channel when the answer lives only in memory files, not the live transcript.
+    summary: Verify the agent uses memory tools in a shared channel when the answer lives only in memory files, not the live transcript.
     config:
       channelId: qa-memory-room
       channelTitle: QA Memory Room
@@ -33,7 +32,7 @@ scenario:
 
 flow:
   steps:
-    - name: uses memory_search plus memory_get before answering in-channel
+    - name: uses memory_search before answering in-channel
       actions:
         - call: reset
         - call: fs.writeFile
@@ -80,7 +79,4 @@ flow:
         - assert:
             expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'memory_search')"
             message: expected memory_search in mock request plan
-        - assert:
-            expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).some((request) => request.plannedToolName === 'memory_get')"
-            message: expected memory_get in mock request plan
       detailsExpr: outbound.text

qa/scenarios/models/claude-cli-provider-capabilities-subscription.yaml

@@ -31,6 +31,7 @@ scenario:
     summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode subscription --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities-subscription`.
     config:
       authMode: subscription
+      requiredProviderMode: live-frontier
       requiredProvider: claude-cli
       chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK"
       chatExpected: CLAUDE-CLI-CHAT-OK

qa/scenarios/models/claude-cli-provider-capabilities.yaml

@@ -31,6 +31,7 @@ scenario:
     summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode api-key --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities`.
     config:
       authMode: api-key
+      requiredProviderMode: live-frontier
       requiredProvider: claude-cli
       chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK"
       chatExpected: CLAUDE-CLI-CHAT-OK

qa/scenarios/plugins/mcp-plugin-tools-call.yaml

@@ -18,47 +18,8 @@ scenario:
     - docs/gateway/protocol.md
   codeRefs:
     - src/mcp/plugin-tools-serve.ts
-    - extensions/qa-lab/src/suite.ts
+    - src/mcp/plugin-tools-mcp-client.test.ts
   execution:
-    kind: flow
+    kind: vitest
+    path: src/mcp/plugin-tools-mcp-client.test.ts
     summary: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully.
-    config:
-      memoryFact: "MCP fact: the codename is ORBIT-9."
-      query: "ORBIT-9 codename"
-      expectedNeedle: "ORBIT-9"
-
-flow:
-  steps:
-    - name: serves and calls memory_search over MCP
-      actions:
-        - call: fs.writeFile
-          args:
-            - expr: "path.join(env.gateway.workspaceDir, 'MEMORY.md')"
-            - expr: "`${config.memoryFact}\\n`"
-            - utf8
-        - call: forceMemoryIndex
-          args:
-            - env:
-                ref: env
-              query:
-                expr: config.query
-              expectedNeedle:
-                expr: config.expectedNeedle
-        - call: callPluginToolsMcp
-          saveAs: result
-          args:
-            - env:
-                ref: env
-              toolName: memory_search
-              args:
-                query:
-                  expr: config.query
-                maxResults: 3
-        - set: text
-          value:
-            expr: "JSON.stringify(result.content ?? [])"
-        - assert:
-            expr: "text.includes(config.expectedNeedle)"
-            message:
-              expr: "`MCP memory_search missed expected fact: ${text}`"
-      detailsExpr: text

src/mcp/plugin-tools-mcp-client.test.ts

@@ -0,0 +1,60 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { describe, expect, it, vi } from "vitest";
+import type { AnyAgentTool } from "../agents/tools/common.js";
+import type { OpenClawConfig } from "../config/types.openclaw.js";
+import { createPluginToolsMcpServer } from "./plugin-tools-serve.js";
+
+describe("plugin tools MCP client bridge", () => {
+  it("lists and calls a plugin tool through a real MCP client", async () => {
+    const execute = vi.fn().mockResolvedValue({
+      content: [{ type: "text", text: "MCP fact: the codename is ORBIT-9." }],
+    });
+    const tool = {
+      name: "memory_search",
+      description: "Search memory",
+      parameters: {
+        type: "object",
+        properties: {
+          query: { type: "string" },
+          maxResults: { type: "number" },
+        },
+        required: ["query"],
+      },
+      execute,
+    } as unknown as AnyAgentTool;
+
+    const server = createPluginToolsMcpServer({
+      config: { plugins: { enabled: true } } as OpenClawConfig,
+      tools: [tool],
+    });
+    const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+    const client = new Client(
+      { name: "plugin-tools-test-client", version: "0.0.0" },
+      { capabilities: {} },
+    );
+
+    await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]);
+
+    try {
+      const listed = await client.listTools();
+      expect(listed.tools.map((listedTool) => listedTool.name)).toContain("memory_search");
+
+      const result = await client.callTool({
+        name: "memory_search",
+        arguments: { query: "ORBIT-9 codename", maxResults: 3 },
+      });
+
+      expect(execute).toHaveBeenCalledWith(
+        expect.stringMatching(/^mcp-\d+$/),
+        { query: "ORBIT-9 codename", maxResults: 3 },
+        expect.any(AbortSignal),
+        undefined,
+      );
+      expect(JSON.stringify(result.content)).toContain("ORBIT-9");
+    } finally {
+      await client.close();
+      await server.close();
+    }
+  });
+});