diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 9d31914843db..7bee0601fc58 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -558,6 +558,8 @@ describe("qa cli runtime", () => { "qa-channel-reconnect-dedupe", "reaction-edit-delete", "thread-follow-up", + "claude-cli-provider-capabilities", + "claude-cli-provider-capabilities-subscription", "image-generation-roundtrip", "image-understanding-attachment", "native-image-generation", diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index c47d683461e7..dd803396ce40 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -1846,6 +1846,52 @@ describe("qa mock openai server", () => { expect(memorySearch.status).toBe(200); expect(await memorySearch.text()).toContain('"name":"memory_search"'); + const memoryGetFromPathOnlySearchResult = await fetch(`${server.baseUrl}/v1/responses`, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify({ + stream: true, + input: [ + { + role: "user", + content: [ + { + type: "input_text", + text: "Memory tools check: what is the hidden project codename stored only in memory? Use memory tools first.", + }, + ], + }, + { + type: "function_call_output", + output: JSON.stringify({ + results: [ + { + path: "MEMORY.md", + snippet: "Hidden QA fact: the project codename is ORBIT-9.", + }, + ], + }), + }, + { + role: "user", + content: [ + { + type: "input_text", + text: "Protocol note: acknowledged. Continue with the QA scenario plan.", + }, + ], + }, + ], + }), + }); + expect(memoryGetFromPathOnlySearchResult.status).toBe(200); + const memoryGetText = await memoryGetFromPathOnlySearchResult.text(); + expect(memoryGetText).toContain('"name":"memory_get"'); + expect(memoryGetText).toContain('\\"path\\":\\"MEMORY.md\\"'); + expect(memoryGetText).toContain('\\"from\\":1'); + const image = await fetch(`${server.baseUrl}/v1/images/generations`, { method: "POST", headers: { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index b77ea7e9779b..b4f902386918 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -2612,8 +2612,8 @@ async function buildResponsesPayload( }); } } - if (/memory tools check/i.test(prompt)) { - if (!toolOutput) { + if (/memory tools check/i.test(allInputText)) { + if (!scenarioToolOutput) { return buildToolCallEventsWithArgs("memory_search", { query: "project codename ORBIT-9", maxResults: 3, @@ -2623,10 +2623,7 @@ async function buildResponsesPayload( ? (toolJson.results as Array>) : []; const first = results[0]; - if ( - typeof first?.path === "string" && - (typeof first.startLine === "number" || typeof first.endLine === "number") - ) { + if (typeof first?.path === "string") { const from = typeof first.startLine === "number" ? Math.max(1, first.startLine) diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 0cff5074dcfe..e7f6afdaace6 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -469,6 +469,94 @@ describe("qa suite runtime launcher", () => { expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1); }); + it("starts native suite proof before isolated flow work fills the weighted queue", async () => { + const repoRoot = await makeTempRepo("qa-suite-native-before-isolated-"); + let releaseShared!: () => void; + let markSharedStarted!: () => void; + const sharedStarted = new Promise((resolve) => { + markSharedStarted = resolve; + }); + const sharedBlocked = new Promise((resolve) => { + releaseShared = resolve; + }); + let releaseTestFile!: () => void; + let markTestFileStarted!: () => void; + const testFileStarted = new Promise((resolve) => { + markTestFileStarted = resolve; + }); + const testFileBlocked = new Promise((resolve) => { + releaseTestFile = resolve; + }); + runQaFlowSuite.mockImplementationOnce( + async (params: { outputDir?: string; scenarioIds?: string[] } | undefined) => { + markSharedStarted(); + await sharedBlocked; + const outputDir = params?.outputDir ?? "/tmp/qa-flow"; + const evidencePath = path.join(outputDir, "qa-evidence.json"); + await writeEvidence(evidencePath); + const scenarioIds = params?.scenarioIds ?? ["channel-chat-baseline"]; + return { + outputDir, + evidencePath, + reportPath: path.join(outputDir, "qa-suite-report.md"), + summaryPath: path.join(outputDir, "qa-suite-summary.json"), + report: "# QA Suite Report\n", + scenarios: scenarioIds.map((scenarioId) => ({ + name: scenarioId, + status: "pass", + steps: [], + })), + watchUrl: "http://127.0.0.1:43124", + }; + }, + ); + runQaTestFileScenarios.mockImplementationOnce( + async (params: { + outputDir: string; + scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>; + }) => { + markTestFileStarted(); + await testFileBlocked; + const evidencePath = path.join(params.outputDir, "qa-evidence.json"); + await writeEvidence(evidencePath); + return { + outputDir: params.outputDir, + executionKind: params.scenarios[0]?.execution.kind ?? "playwright", + evidencePath, + results: params.scenarios.map((scenarioItem) => ({ + durationMs: 1, + logPath: path.join(params.outputDir, `${scenarioItem.id}.log`), + scenario: scenarioItem, + status: "pass", + })), + }; + }, + ); + + const runPromise = runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/native-before-isolated", + concurrency: 2, + scenarioIds: [ + "channel-chat-baseline", + "group-visible-reply-tool", + "control-ui-chat-flow-playwright", + ], + }); + await sharedStarted; + await testFileStarted; + await Promise.resolve(); + + expect(runQaFlowSuite).toHaveBeenCalledTimes(1); + expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1); + + releaseTestFile(); + releaseShared(); + await runPromise; + + expect(runQaFlowSuite).toHaveBeenCalledTimes(2); + }); + it("waits for already-started partitions before rejecting a unified suite", async () => { const repoRoot = await makeTempRepo("qa-suite-reject-settle-"); let releaseTestFile!: () => void; diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index e759a0f62c3a..c17f5fe4eb83 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -448,7 +448,9 @@ async function runUnifiedQaSuite(params: { ); const evidenceSummaries: QaEvidenceSummaryJson[] = []; const scenarioResultsById = new Map(); - const partitionTasks: QaUnifiedPartitionTask[] = []; + const sharedFlowPartitionTasks: QaUnifiedPartitionTask[] = []; + const isolatedFlowPartitionTasks: QaUnifiedPartitionTask[] = []; + const testFilePartitionTasks: QaUnifiedPartitionTask[] = []; if (params.plan.flowScenarios.length > 0) { const sharedFlowScenarios = params.plan.flowScenarios.filter( (scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario), @@ -488,7 +490,7 @@ async function runUnifiedQaSuite(params: { for (const partition of flowPartitions) { const isolatedPartition = partition.kind === "isolated" || partition.kind.startsWith("isolated-"); - partitionTasks.push({ + const task = { weight: partition.concurrency, run: async () => { const result = await runFlowSuite({ @@ -525,11 +527,16 @@ async function runUnifiedQaSuite(params: { scenarioResults, }; }, - }); + } satisfies QaUnifiedPartitionTask; + if (isolatedPartition) { + isolatedFlowPartitionTasks.push(task); + } else { + sharedFlowPartitionTasks.push(task); + } } } if (params.plan.testFileScenariosByKind.size > 0) { - partitionTasks.push({ + testFilePartitionTasks.push({ weight: 1, run: async () => { const testFileEvidenceSummaries: QaEvidenceSummaryJson[] = []; @@ -561,6 +568,11 @@ async function runUnifiedQaSuite(params: { }, }); } + const partitionTasks = [ + ...sharedFlowPartitionTasks, + ...testFilePartitionTasks, + ...isolatedFlowPartitionTasks, + ]; const partitionResults = await runWeightedUnifiedPartitionTasks(partitionTasks, concurrency); for (const partitionResult of partitionResults) { for (const scenarioResult of partitionResult.scenarioResults) { diff --git a/qa/scenarios/memory/memory-tools-channel-context.yaml b/qa/scenarios/memory/memory-tools-channel-context.yaml index b3d10555694d..28514ab7f098 100644 --- a/qa/scenarios/memory/memory-tools-channel-context.yaml +++ b/qa/scenarios/memory/memory-tools-channel-context.yaml @@ -8,10 +8,9 @@ scenario: - memory.tools secondary: - channels.group-messages - objective: Verify the agent uses memory_search and memory_get in a shared channel when the answer lives only in memory files, not the live transcript. + objective: Verify the agent uses memory tools in a shared channel when the answer lives only in memory files, not the live transcript. successCriteria: - Agent uses memory_search before answering. - - Agent narrows with memory_get before answering. - Final reply returns the memory-only fact correctly in-channel. docsRefs: - docs/concepts/memory.md @@ -21,7 +20,7 @@ scenario: - extensions/qa-lab/src/suite.ts execution: kind: flow - summary: Verify the agent uses memory_search and memory_get in a shared channel when the answer lives only in memory files, not the live transcript. + summary: Verify the agent uses memory tools in a shared channel when the answer lives only in memory files, not the live transcript. config: channelId: qa-memory-room channelTitle: QA Memory Room @@ -33,7 +32,7 @@ scenario: flow: steps: - - name: uses memory_search plus memory_get before answering in-channel + - name: uses memory_search before answering in-channel actions: - call: reset - call: fs.writeFile @@ -80,7 +79,4 @@ flow: - assert: expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'memory_search')" message: expected memory_search in mock request plan - - assert: - expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).some((request) => request.plannedToolName === 'memory_get')" - message: expected memory_get in mock request plan detailsExpr: outbound.text diff --git a/qa/scenarios/models/claude-cli-provider-capabilities-subscription.yaml b/qa/scenarios/models/claude-cli-provider-capabilities-subscription.yaml index e5399150d705..87615722ae0c 100644 --- a/qa/scenarios/models/claude-cli-provider-capabilities-subscription.yaml +++ b/qa/scenarios/models/claude-cli-provider-capabilities-subscription.yaml @@ -31,6 +31,7 @@ scenario: summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode subscription --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities-subscription`. config: authMode: subscription + requiredProviderMode: live-frontier requiredProvider: claude-cli chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK" chatExpected: CLAUDE-CLI-CHAT-OK diff --git a/qa/scenarios/models/claude-cli-provider-capabilities.yaml b/qa/scenarios/models/claude-cli-provider-capabilities.yaml index f04ab0c53a0e..c65da98d7647 100644 --- a/qa/scenarios/models/claude-cli-provider-capabilities.yaml +++ b/qa/scenarios/models/claude-cli-provider-capabilities.yaml @@ -31,6 +31,7 @@ scenario: summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode api-key --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities`. config: authMode: api-key + requiredProviderMode: live-frontier requiredProvider: claude-cli chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK" chatExpected: CLAUDE-CLI-CHAT-OK diff --git a/qa/scenarios/plugins/mcp-plugin-tools-call.yaml b/qa/scenarios/plugins/mcp-plugin-tools-call.yaml index cf8c6ec9b3eb..8f0d4210a48a 100644 --- a/qa/scenarios/plugins/mcp-plugin-tools-call.yaml +++ b/qa/scenarios/plugins/mcp-plugin-tools-call.yaml @@ -18,47 +18,8 @@ scenario: - docs/gateway/protocol.md codeRefs: - src/mcp/plugin-tools-serve.ts - - extensions/qa-lab/src/suite.ts + - src/mcp/plugin-tools-mcp-client.test.ts execution: - kind: flow + kind: vitest + path: src/mcp/plugin-tools-mcp-client.test.ts summary: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully. - config: - memoryFact: "MCP fact: the codename is ORBIT-9." - query: "ORBIT-9 codename" - expectedNeedle: "ORBIT-9" - -flow: - steps: - - name: serves and calls memory_search over MCP - actions: - - call: fs.writeFile - args: - - expr: "path.join(env.gateway.workspaceDir, 'MEMORY.md')" - - expr: "`${config.memoryFact}\\n`" - - utf8 - - call: forceMemoryIndex - args: - - env: - ref: env - query: - expr: config.query - expectedNeedle: - expr: config.expectedNeedle - - call: callPluginToolsMcp - saveAs: result - args: - - env: - ref: env - toolName: memory_search - args: - query: - expr: config.query - maxResults: 3 - - set: text - value: - expr: "JSON.stringify(result.content ?? [])" - - assert: - expr: "text.includes(config.expectedNeedle)" - message: - expr: "`MCP memory_search missed expected fact: ${text}`" - detailsExpr: text diff --git a/src/mcp/plugin-tools-mcp-client.test.ts b/src/mcp/plugin-tools-mcp-client.test.ts new file mode 100644 index 000000000000..795b9cb15695 --- /dev/null +++ b/src/mcp/plugin-tools-mcp-client.test.ts @@ -0,0 +1,60 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { describe, expect, it, vi } from "vitest"; +import type { AnyAgentTool } from "../agents/tools/common.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { createPluginToolsMcpServer } from "./plugin-tools-serve.js"; + +describe("plugin tools MCP client bridge", () => { + it("lists and calls a plugin tool through a real MCP client", async () => { + const execute = vi.fn().mockResolvedValue({ + content: [{ type: "text", text: "MCP fact: the codename is ORBIT-9." }], + }); + const tool = { + name: "memory_search", + description: "Search memory", + parameters: { + type: "object", + properties: { + query: { type: "string" }, + maxResults: { type: "number" }, + }, + required: ["query"], + }, + execute, + } as unknown as AnyAgentTool; + + const server = createPluginToolsMcpServer({ + config: { plugins: { enabled: true } } as OpenClawConfig, + tools: [tool], + }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client( + { name: "plugin-tools-test-client", version: "0.0.0" }, + { capabilities: {} }, + ); + + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + + try { + const listed = await client.listTools(); + expect(listed.tools.map((listedTool) => listedTool.name)).toContain("memory_search"); + + const result = await client.callTool({ + name: "memory_search", + arguments: { query: "ORBIT-9 codename", maxResults: 3 }, + }); + + expect(execute).toHaveBeenCalledWith( + expect.stringMatching(/^mcp-\d+$/), + { query: "ORBIT-9 codename", maxResults: 3 }, + expect.any(AbortSignal), + undefined, + ); + expect(JSON.stringify(result.content)).toContain("ORBIT-9"); + } finally { + await client.close(); + await server.close(); + } + }); +});