From 9d68f877ac3ea28c1abffaf7a043c8c916bea217 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 3 Jul 2026 20:26:16 -0700 Subject: [PATCH] test(qa): run gateway and MCP scenarios over real transports (#99735) --- extensions/qa-lab/api.ts | 1 + .../qa-lab/src/bundled-plugin-staging.ts | 13 +- extensions/qa-lab/src/gateway-child.test.ts | 24 +- extensions/qa-lab/src/gateway-child.ts | 85 ++- .../plugins/mcp-plugin-tools-call.yaml | 16 +- qa/scenarios/runtime/gateway-smoke.yaml | 15 +- .../mcp-gateway-connect-startup-retry.yaml | 20 +- .../runtime/gateway-mcp-real-transports.ts | 701 ++++++++++++++++++ 8 files changed, 850 insertions(+), 25 deletions(-) create mode 100644 test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts diff --git a/extensions/qa-lab/api.ts b/extensions/qa-lab/api.ts index 79ec77e349e1..54b35d68a35f 100644 --- a/extensions/qa-lab/api.ts +++ b/extensions/qa-lab/api.ts @@ -101,6 +101,7 @@ export { testing as __testing, buildQaRuntimeEnv, type QaCliBackendAuthMode, + type QaGatewayChildListeningContext, type QaGatewayChildCommand, type QaGatewayChildStateMutationContext, resolveQaControlUiRoot, diff --git a/extensions/qa-lab/src/bundled-plugin-staging.ts b/extensions/qa-lab/src/bundled-plugin-staging.ts index 3007054f5997..92daebeb141e 100644 --- a/extensions/qa-lab/src/bundled-plugin-staging.ts +++ b/extensions/qa-lab/src/bundled-plugin-staging.ts @@ -194,12 +194,13 @@ function collectQaBundledPluginIds(params: { repoRoot: string; allowedPluginIds: readonly string[]; }) { - const pluginIds = new Set( - params.allowedPluginIds.map((pluginId) => { - assertSafeQaBundledPluginId(pluginId); - return pluginId; - }), - ); + const pluginIds = new Set(); + for (const pluginId of params.allowedPluginIds) { + assertSafeQaBundledPluginId(pluginId); + if (resolveQaBundledPluginSourceDir({ repoRoot: params.repoRoot, pluginId })) { + pluginIds.add(pluginId); + } + } for (const pluginId of QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS) { if ( resolveQaBundledPluginSourceDir({ diff --git a/extensions/qa-lab/src/gateway-child.test.ts b/extensions/qa-lab/src/gateway-child.test.ts index 5cda43fe4db0..6ce2e7c1a46e 100644 --- a/extensions/qa-lab/src/gateway-child.test.ts +++ b/extensions/qa-lab/src/gateway-child.test.ts @@ -11,6 +11,7 @@ import { resolveQaControlUiRoot, startQaGatewayChild, } from "./gateway-child.js"; +import { createTempDirHarness } from "./temp-dir.test-helper.js"; const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn()); const resolveQaNodeExecPathMock = vi.hoisted(() => vi.fn(async () => process.execPath)); @@ -22,7 +23,8 @@ vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ fetchWithSsrFGuard: fetchWithSsrFGuardMock, })); -vi.mock("openclaw/plugin-sdk/temp-path", () => ({ +vi.mock("openclaw/plugin-sdk/temp-path", async (importOriginal) => ({ + ...(await importOriginal()), resolvePreferredOpenClawTmpDir: () => qaTempPathState.preferredTmpDir, })); @@ -31,6 +33,7 @@ vi.mock("./node-exec.js", () => ({ })); const cleanups: Array<() => Promise> = []; +const tempDirs = createTempDirHarness(); afterEach(async () => { fetchWithSsrFGuardMock.mockReset(); @@ -39,6 +42,7 @@ afterEach(async () => { while (cleanups.length > 0) { await cleanups.pop()?.(); } + await tempDirs.cleanup(); }); function createParams(baseEnv?: NodeJS.ProcessEnv) { @@ -1623,6 +1627,24 @@ describe("qa bundled plugin dir", () => { ).rejects.toThrow("invalid QA bundled plugin id: ../escape"); }); + it("leaves external allowed plugins to configured load paths", async () => { + const repoRoot = await tempDirs.makeTempDir("qa-bundled-external-id-"); + await writeFile( + path.join(repoRoot, "package.json"), + JSON.stringify({ name: "openclaw", type: "module" }, null, 2), + "utf8", + ); + const tempRoot = await tempDirs.makeTempDir("qa-bundled-external-target-"); + + const { bundledPluginsDir } = await testing.createQaBundledPluginsDir({ + repoRoot, + tempRoot, + allowedPluginIds: ["external-fixture"], + }); + + await expect(readdir(bundledPluginsDir)).resolves.not.toContain("external-fixture"); + }); + it("stages source-only bundled plugins into a repo-like runtime root with node_modules", async () => { const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-bundled-source-stage-")); cleanups.push(async () => { diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index fa294f5a7580..6e7267090676 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -77,10 +77,19 @@ export type QaGatewayChildCommand = { usePackagedPlugins?: boolean; }; +export type QaGatewayChildListeningContext = { + attempt: number; + baseUrl: string; + wsUrl: string; + token: string; + configPath: string; + runtimeEnv: NodeJS.ProcessEnv; +}; + async function getFreePort() { return await new Promise((resolve, reject) => { const server = net.createServer(); - server.once("error", reject); + server.once("error", (error) => reject(error)); server.listen(0, "127.0.0.1", () => { const address = server.address(); if (!address || typeof address === "string") { @@ -319,6 +328,23 @@ async function fetchLocalGatewayHealth(params: { } } +async function fetchLocalGatewayListening(baseUrl: string): Promise { + const { release } = await fetchWithSsrFGuard({ + url: `${baseUrl}/healthz`, + init: { + method: "HEAD", + headers: { + connection: "close", + }, + signal: AbortSignal.timeout(2_000), + }, + policy: { allowPrivateNetwork: true }, + auditContext: "qa-lab-gateway-child-listening", + }); + await release(); + return true; +} + async function waitForQaGatewayRestartBoundary(params: { logs: () => string; offset: number; @@ -577,6 +603,47 @@ async function waitForGatewayReady(params: { ); } +async function waitForGatewayListening(params: { + baseUrl: string; + logs: () => string; + child: { + exitCode: number | null; + signalCode: NodeJS.Signals | null; + }; + getSpawnError?: () => unknown; + timeoutMs?: number; +}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) { + const spawnError = params.getSpawnError?.(); + if (spawnError) { + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `gateway failed to spawn: ${formatErrorMessage(spawnError)}\n${params.logs()}`, + { cause: spawnError }, + ); + } + if (params.child.exitCode !== null || params.child.signalCode !== null) { + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `gateway exited before listening (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`, + ); + } + try { + if (await fetchLocalGatewayListening(params.baseUrl)) { + return; + } + } catch { + // retry until the HTTP listener accepts requests + } + await sleep(100); + } + throw new QaSuiteInfraError( + "gateway_startup_unhealthy", + `gateway failed to listen before timeout:\n${params.logs()}`, + ); +} + function isRetryableRpcStartupError(error: unknown) { const details = formatErrorMessage(error); return ( @@ -616,6 +683,7 @@ export async function startQaGatewayChild(params: { enabledPluginIds?: string[]; forwardHostHome?: boolean; mockAuthAgentIds?: readonly string[]; + onListening?: (context: QaGatewayChildListeningContext) => Promise | void; mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig; runtimeEnvPatch?: NodeJS.ProcessEnv; }) { @@ -836,6 +904,21 @@ export async function startQaGatewayChild(params: { const getAttemptSpawnError = monitorQaGatewayChildSpawnError(attemptChild, output); try { + await waitForGatewayListening({ + baseUrl, + logs, + child: attemptChild, + getSpawnError: getAttemptSpawnError, + timeoutMs: 120_000, + }); + await params.onListening?.({ + attempt, + baseUrl, + wsUrl, + token: gatewayToken, + configPath, + runtimeEnv: env, + }); await waitForGatewayReady({ baseUrl, logs, diff --git a/qa/scenarios/plugins/mcp-plugin-tools-call.yaml b/qa/scenarios/plugins/mcp-plugin-tools-call.yaml index f19fdf1a4e2d..77a39ba386fc 100644 --- a/qa/scenarios/plugins/mcp-plugin-tools-call.yaml +++ b/qa/scenarios/plugins/mcp-plugin-tools-call.yaml @@ -4,7 +4,7 @@ scenario: id: mcp-plugin-tools-call surface: mcp coverage: - secondary: + primary: - plugins.mcp-tools - tools.invocation objective: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully. @@ -16,9 +16,15 @@ scenario: - docs/cli/mcp.md - docs/gateway/protocol.md codeRefs: + - test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts - src/mcp/plugin-tools-serve.ts - - src/mcp/plugin-tools-mcp-client.test.ts + - src/mcp/plugin-tools-handlers.ts execution: - kind: vitest - path: src/mcp/plugin-tools-mcp-client.test.ts - summary: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully. + kind: script + path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts + summary: Registers a fixture plugin, starts the real plugin-tools stdio server, and calls its tool with a real MCP client. + args: + - --scenario + - mcp-plugin-tools-call + - --artifact-base + - ${outputDir} diff --git a/qa/scenarios/runtime/gateway-smoke.yaml b/qa/scenarios/runtime/gateway-smoke.yaml index 91c814d3c60e..73544fd3f247 100644 --- a/qa/scenarios/runtime/gateway-smoke.yaml +++ b/qa/scenarios/runtime/gateway-smoke.yaml @@ -4,7 +4,7 @@ scenario: id: gateway-smoke surface: runtime coverage: - secondary: + primary: - gateway.websocket-transport - gateway.health-apis - gateway.hello-ok-snapshot @@ -18,8 +18,15 @@ scenario: - docs/gateway/index.md - docs/concepts/qa-e2e-automation.md codeRefs: + - test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts + - extensions/qa-lab/src/gateway-child.ts - test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts execution: - kind: vitest - path: test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts - summary: Vitest coverage for gateway health and WebSocket smoke checks. + kind: script + path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts + summary: Starts a real Gateway child and exercises the Gateway smoke client against its WebSocket and health RPC surfaces. + args: + - --scenario + - gateway-smoke + - --artifact-base + - ${outputDir} diff --git a/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml b/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml index 2705f2f68f8e..36f8eb763b91 100644 --- a/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml +++ b/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml @@ -4,7 +4,7 @@ scenario: id: mcp-gateway-connect-startup-retry surface: runtime coverage: - secondary: + primary: - gateway.connect-request - gateway.protocol-version-negotiation - gateway.startup-retry @@ -18,11 +18,15 @@ scenario: - docs/cli/mcp.md - docs/concepts/qa-e2e-automation.md codeRefs: - - src/gateway/client.test.ts - - test/e2e/qa-lab/runtime/mcp-channels.fixture.ts - - test/e2e/qa-lab/runtime/mcp-client-temp-state.fixture.ts - - test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts + - test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts + - extensions/qa-lab/src/gateway-child.ts + - src/mcp/channel-bridge.ts execution: - kind: vitest - path: src/gateway/client.test.ts - summary: Vitest coverage for Gateway connect request, protocol negotiation, and startup retry behavior used by MCP bridge clients. + kind: script + path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts + summary: Starts the real MCP client before a delayed real Gateway becomes ready and captures retry, connect-frame, and negotiated-protocol evidence. + args: + - --scenario + - mcp-gateway-connect-startup-retry + - --artifact-base + - ${outputDir} diff --git a/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts b/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts new file mode 100644 index 000000000000..48e010a2c232 --- /dev/null +++ b/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts @@ -0,0 +1,701 @@ +import { existsSync } from "node:fs"; +import fs from "node:fs/promises"; +// QA Lab producer proves Gateway and MCP scenarios across real process and protocol boundaries. +import { createServer, type Server } from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { WebSocket, WebSocketServer, type RawData } from "ws"; +import { + QA_EVIDENCE_FILENAME, + startQaGatewayChild, + type QaEvidenceSummaryJson, + type QaGatewayChildListeningContext, +} from "../../../../extensions/qa-lab/api.js"; +import { + PROTOCOL_VERSION, + MIN_CLIENT_PROTOCOL_VERSION, +} from "../../../../packages/gateway-protocol/src/version.js"; +import { runGatewaySmoke } from "../../../../scripts/dev/gateway-smoke.js"; +import type { OpenClawConfig } from "../../../../src/config/types.openclaw.js"; +import { formatErrorMessage } from "../../../../src/infra/errors.js"; +import { createMcpClientTempState } from "./mcp-client-temp-state.fixture.ts"; +import { createQaScriptEvidenceWriter, type QaScriptEvidenceStatus } from "./script-evidence.ts"; + +const FIXTURE_PLUGIN_ID = "qa-real-transports-fixture"; +const FIXTURE_TOOL_NAME = "memory_search"; +const FIXTURE_FACT = "MCP fact: the codename is ORBIT-9."; +const STARTUP_GATE_TIMEOUT_MS = 30_000; +const MCP_CONNECT_TIMEOUT_MS = 30_000; +const SOURCE_PATH = "test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts"; + +type ScenarioId = "gateway-smoke" | "mcp-gateway-connect-startup-retry" | "mcp-plugin-tools-call"; + +type ProducerOptions = { + artifactBase: string; + repoRoot: string; + scenarioId: ScenarioId; +}; + +type ProofResult = { + details?: string; + durationMs: number; + status: QaScriptEvidenceStatus; +}; + +type GatewayFrameCapture = { + connectFrames: Array<{ minProtocol: number; maxProtocol: number }>; + helloProtocols: number[]; + startupUnavailableResponses: number; +}; + +type GatewayProxy = { + capture: GatewayFrameCapture; + stop: () => Promise; + url: string; +}; + +type McpClientHandle = { + client: Client; + cleanup: () => void; + stderr: () => string; + transport: StdioClientTransport; +}; + +const SCENARIOS = { + "gateway-smoke": { + title: "Gateway smoke evidence", + sourcePath: "qa/scenarios/runtime/gateway-smoke.yaml", + primaryCoverageIds: [ + "gateway.websocket-transport", + "gateway.health-apis", + "gateway.hello-ok-snapshot", + ], + docsRefs: ["docs/gateway/index.md", "docs/concepts/qa-e2e-automation.md"], + codeRefs: [ + SOURCE_PATH, + "extensions/qa-lab/src/gateway-child.ts", + "scripts/dev/gateway-smoke.ts", + ], + }, + "mcp-gateway-connect-startup-retry": { + title: "MCP Gateway connect startup retry", + sourcePath: "qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml", + primaryCoverageIds: [ + "gateway.connect-request", + "gateway.protocol-version-negotiation", + "gateway.startup-retry", + ], + docsRefs: ["docs/gateway/protocol.md", "docs/cli/mcp.md"], + codeRefs: [SOURCE_PATH, "extensions/qa-lab/src/gateway-child.ts", "src/mcp/channel-bridge.ts"], + }, + "mcp-plugin-tools-call": { + title: "MCP plugin-tools call", + sourcePath: "qa/scenarios/plugins/mcp-plugin-tools-call.yaml", + primaryCoverageIds: ["plugins.mcp-tools", "tools.invocation"], + docsRefs: ["docs/cli/mcp.md", "docs/gateway/protocol.md"], + codeRefs: [SOURCE_PATH, "src/mcp/plugin-tools-serve.ts", "src/mcp/plugin-tools-handlers.ts"], + }, +} as const; + +function parseOptions(argv: readonly string[]): ProducerOptions { + const readValue = (name: string) => { + const index = argv.indexOf(name); + return index >= 0 ? argv[index + 1] : undefined; + }; + const scenarioId = readValue("--scenario"); + if (!scenarioId || !(scenarioId in SCENARIOS)) { + throw new Error(`--scenario must be one of: ${Object.keys(SCENARIOS).join(", ")}`); + } + const artifactBase = readValue("--artifact-base"); + if (!artifactBase) { + throw new Error("--artifact-base is required"); + } + return { + artifactBase: path.resolve(artifactBase), + repoRoot: path.resolve(readValue("--repo-root") ?? process.cwd()), + scenarioId: scenarioId as ScenarioId, + }; +} + +async function createFixturePlugin() { + // openclaw-temp-dir: allow standalone producer cleans this root in each scenario finally block + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-gateway-mcp-fixture-")); + const pluginDir = path.join(root, FIXTURE_PLUGIN_ID); + const startupGatePath = path.join(root, "startup-connect-observed"); + await fs.mkdir(pluginDir, { recursive: true }); + await fs.writeFile( + path.join(pluginDir, "openclaw.plugin.json"), + `${JSON.stringify( + { + id: FIXTURE_PLUGIN_ID, + activation: { onStartup: true }, + configSchema: { type: "object", additionalProperties: false, properties: {} }, + contracts: { tools: [FIXTURE_TOOL_NAME] }, + }, + null, + 2, + )}\n`, + "utf8", + ); + await fs.writeFile( + path.join(pluginDir, "index.js"), + `const fs = require("node:fs"); + +module.exports = { + id: ${JSON.stringify(FIXTURE_PLUGIN_ID)}, + register(api) { + api.registerTool({ + name: ${JSON.stringify(FIXTURE_TOOL_NAME)}, + description: "Search fixture memory", + parameters: { + type: "object", + properties: { query: { type: "string" }, maxResults: { type: "number" } }, + required: ["query"], + }, + async execute(_toolCallId, params) { + return { content: [{ type: "text", text: ${JSON.stringify(FIXTURE_FACT)} + " query=" + String(params.query) }] }; + }, + }); + api.registerService({ + id: "qa-startup-delay", + async start() { + const deadline = Date.now() + ${STARTUP_GATE_TIMEOUT_MS}; + while (!fs.existsSync(${JSON.stringify(startupGatePath)})) { + if (Date.now() >= deadline) { + throw new Error("timed out waiting for the QA MCP startup connect frame"); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + }, + stop() {}, + }); + }, +};\n`, + "utf8", + ); + return { + pluginDir, + startupGatePath, + cleanup: () => fs.rm(root, { force: true, recursive: true }), + }; +} + +function withFixturePlugin(config: OpenClawConfig, pluginDir: string): OpenClawConfig { + const existingPaths = config.plugins?.load?.paths ?? []; + const existingAllow = config.plugins?.allow ?? []; + return { + ...config, + plugins: { + ...config.plugins, + enabled: true, + allow: [...new Set([...existingAllow, FIXTURE_PLUGIN_ID])], + load: { + ...config.plugins?.load, + paths: [...new Set([...existingPaths, pluginDir])], + }, + entries: { + ...config.plugins?.entries, + [FIXTURE_PLUGIN_ID]: { enabled: true }, + }, + }, + }; +} + +function emptyTransport() { + return { + requiredPluginIds: [] as string[], + createGatewayConfig: () => ({}), + }; +} + +function parseJsonFrame(data: RawData): Record | null { + try { + const text = Array.isArray(data) + ? Buffer.concat(data).toString("utf8") + : Buffer.from(data).toString("utf8"); + const value = JSON.parse(text); + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; + } catch { + return null; + } +} + +async function startGatewayProxy( + upstreamUrl: string, + onConnectFrame?: () => void, +): Promise { + const capture: GatewayFrameCapture = { + connectFrames: [], + helloProtocols: [], + startupUnavailableResponses: 0, + }; + const connectRequestIds = new Set(); + const server: Server = createServer(); + const wss = new WebSocketServer({ server }); + wss.on("connection", (downstream) => { + const upstream = new WebSocket(upstreamUrl); + const pending: RawData[] = []; + downstream.on("message", (data) => { + const frame = parseJsonFrame(data); + if (frame?.method === "connect" && typeof frame.id === "string") { + const params = frame.params as Record | undefined; + if (typeof params?.minProtocol === "number" && typeof params.maxProtocol === "number") { + capture.connectFrames.push({ + minProtocol: params.minProtocol, + maxProtocol: params.maxProtocol, + }); + connectRequestIds.add(frame.id); + onConnectFrame?.(); + } + } + if (upstream.readyState === WebSocket.OPEN) { + upstream.send(data); + } else { + pending.push(data); + } + }); + upstream.on("open", () => { + for (const data of pending.splice(0)) { + upstream.send(data); + } + }); + upstream.on("message", (data) => { + const frame = parseJsonFrame(data); + if (typeof frame?.id === "string" && connectRequestIds.has(frame.id)) { + const error = frame.error as Record | undefined; + const details = error?.details as Record | undefined; + if (error?.retryable === true && details?.reason === "startup-sidecars") { + capture.startupUnavailableResponses += 1; + } + const payload = frame.payload as Record | undefined; + if (payload?.type === "hello-ok" && typeof payload.protocol === "number") { + capture.helloProtocols.push(payload.protocol); + } + } + if (downstream.readyState === WebSocket.OPEN) { + downstream.send(data); + } + }); + const closeDownstream = () => { + if (downstream.readyState === WebSocket.OPEN) { + downstream.close(1013, "gateway unavailable"); + } + }; + upstream.on("error", closeDownstream); + upstream.on("close", (code, reason) => { + if (downstream.readyState === WebSocket.OPEN) { + downstream.close(code, reason.toString()); + } + }); + downstream.on("close", () => upstream.close()); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("gateway frame proxy did not bind a TCP port"); + } + return { + capture, + url: `ws://127.0.0.1:${address.port}`, + async stop() { + for (const client of wss.clients) { + client.terminate(); + } + await new Promise((resolve) => { + wss.close(() => resolve()); + }); + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); + }, + }; +} + +async function connectChannelMcpClient(params: { + gatewayUrl: string; + gatewayToken: string; + repoRoot: string; +}): Promise { + const tempState = createMcpClientTempState({ gatewayToken: params.gatewayToken }); + const stderrChunks: Buffer[] = []; + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + path.join(params.repoRoot, "dist/index.js"), + "mcp", + "serve", + "--url", + params.gatewayUrl, + "--token-file", + tempState.tokenFile, + "--claude-channel-mode", + "off", + "--verbose", + ], + cwd: params.repoRoot, + env: { + ...process.env, + OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1", + OPENCLAW_LOG_LEVEL: "debug", + OPENCLAW_STATE_DIR: tempState.stateDir, + }, + stderr: "pipe", + }); + transport.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk))); + const client = new Client({ name: "qa-gateway-mcp-client", version: "1.0.0" }); + let connectTimeout: NodeJS.Timeout | undefined; + try { + await Promise.race([ + client.connect(transport), + new Promise((_, reject) => { + connectTimeout = setTimeout( + () => reject(new Error("MCP channel client connect timed out")), + MCP_CONNECT_TIMEOUT_MS, + ); + }), + ]); + return { + client, + cleanup: tempState.cleanup, + transport, + stderr: () => Buffer.concat(stderrChunks).toString("utf8"), + }; + } catch (error) { + await Promise.allSettled([client.close(), transport.close()]); + tempState.cleanup(); + throw error; + } finally { + if (connectTimeout) { + clearTimeout(connectTimeout); + } + } +} + +async function closeMcpClient(handle: McpClientHandle | undefined) { + if (!handle) { + return; + } + await Promise.allSettled([handle.client.close(), handle.transport.close()]); + handle.cleanup(); +} + +async function approvePendingMcpPairing(gateway: Awaited>) { + const pairing = (await gateway.call("device.pair.list", {})) as { + pending?: Array<{ requestId?: string; role?: string }>; + }; + const pending = pairing.pending?.find((entry) => entry.role === "operator"); + if (!pending?.requestId) { + return false; + } + try { + await gateway.call("device.pair.approve", { requestId: pending.requestId }); + return true; + } catch (error) { + if (formatErrorMessage(error).includes("unknown requestId")) { + return false; + } + throw error; + } +} + +async function runGatewaySmokeProof(options: ProducerOptions): Promise { + const gateway = await startQaGatewayChild({ + repoRoot: options.repoRoot, + transport: emptyTransport(), + transportBaseUrl: "http://127.0.0.1", + controlUiEnabled: false, + }); + const tempRoot = gateway.tempRoot; + const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1"; + let details = ""; + try { + const stdout: string[] = []; + const stderr: string[] = []; + const exitCode = await runGatewaySmoke( + { token: gateway.token, urlRaw: gateway.wsUrl }, + { + stdout: (message) => stdout.push(message), + stderr: (message) => stderr.push(message), + }, + ); + if (exitCode !== 0) { + throw new Error(`gateway smoke exited ${exitCode}: ${stderr.join("\n")}`); + } + const health = (await gateway.call("health", {})) as { ok?: boolean }; + if (health.ok !== true) { + throw new Error(`gateway health RPC returned ${JSON.stringify(health)}`); + } + details = `real Gateway pid=${gateway.pid ?? "unknown"}; ${stdout.join("; ")}; health.ok=true`; + } finally { + await gateway.stop(); + } + if (!keepTemp && existsSync(tempRoot)) { + throw new Error(`Gateway temp root was not cleaned up: ${tempRoot}`); + } + return details; +} + +async function runMcpGatewayStartupRetryProof(options: ProducerOptions): Promise { + const fixture = await createFixturePlugin(); + let proxy: GatewayProxy | undefined; + let mcp: McpClientHandle | undefined; + let gateway: Awaited> | undefined; + let beforeSpawnAt = 0; + const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1"; + let details = ""; + let proofError: Error | undefined; + try { + const onListening = async (context: QaGatewayChildListeningContext) => { + await closeMcpClient(mcp); + await proxy?.stop(); + proxy = await startGatewayProxy(context.wsUrl, () => { + void fs.writeFile(fixture.startupGatePath, "observed\n", "utf8"); + }); + beforeSpawnAt = Date.now(); + mcp = await connectChannelMcpClient({ + gatewayUrl: proxy.url, + gatewayToken: context.token, + repoRoot: options.repoRoot, + }); + }; + gateway = await startQaGatewayChild({ + repoRoot: options.repoRoot, + transport: emptyTransport(), + transportBaseUrl: "http://127.0.0.1", + controlUiEnabled: false, + onListening, + mutateConfig: (config) => withFixturePlugin(config, fixture.pluginDir), + }); + if (!proxy || !mcp) { + throw new Error("MCP client was not started by the Gateway before-spawn hook"); + } + const gatewayReadyAt = Date.now(); + if (beforeSpawnAt >= gatewayReadyAt) { + throw new Error("MCP client did not start before Gateway readiness"); + } + if (await approvePendingMcpPairing(gateway)) { + await closeMcpClient(mcp); + mcp = await connectChannelMcpClient({ + gatewayUrl: proxy.url, + gatewayToken: gateway.token, + repoRoot: options.repoRoot, + }); + } + const tools = await mcp.client.listTools(); + if (!tools.tools.some((tool) => tool.name === "conversations_list")) { + throw new Error("real MCP channel server did not expose conversations_list"); + } + const conversations = await mcp.client.callTool({ + name: "conversations_list", + arguments: { limit: 1 }, + }); + if (conversations.isError) { + throw new Error(`conversations_list failed: ${JSON.stringify(conversations.content)}`); + } + const capture = proxy.capture; + if (capture.startupUnavailableResponses < 1) { + throw new Error( + `expected a retryable startup-unavailable response; captured=${JSON.stringify(capture)}`, + ); + } + if ( + !capture.connectFrames.some( + (frame) => + frame.minProtocol === MIN_CLIENT_PROTOCOL_VERSION && + frame.maxProtocol === PROTOCOL_VERSION, + ) + ) { + throw new Error( + `MCP Gateway connect frame used unexpected protocol range: ${JSON.stringify(capture)}`, + ); + } + if (!capture.helloProtocols.includes(PROTOCOL_VERSION)) { + throw new Error(`MCP Gateway negotiation did not select protocol ${PROTOCOL_VERSION}`); + } + details = [ + `MCP started ${gatewayReadyAt - beforeSpawnAt}ms before Gateway readiness`, + `startup retries=${capture.startupUnavailableResponses}`, + `connect frames=${capture.connectFrames.length}`, + `negotiated protocol=${PROTOCOL_VERSION}`, + ].join("; "); + } catch (error) { + const diagnostics = [ + mcp?.stderr(), + proxy ? `captured Gateway frames: ${JSON.stringify(proxy.capture)}` : undefined, + gateway?.logs(), + ] + .filter((value): value is string => Boolean(value)) + .join("\n"); + proofError = new Error(`${formatErrorMessage(error)}${diagnostics ? `\n${diagnostics}` : ""}`, { + cause: error, + }); + } finally { + await closeMcpClient(mcp); + await proxy?.stop().catch(() => undefined); + const tempRoot = gateway?.tempRoot; + await gateway?.stop().catch(() => undefined); + await fixture.cleanup(); + if (!keepTemp && tempRoot && existsSync(tempRoot) && !proofError) { + proofError = new Error(`Gateway temp root was not cleaned up: ${tempRoot}`); + } + } + if (proofError) { + throw proofError; + } + return details; +} + +async function writePluginToolsConfig(root: string, pluginDir: string) { + const configPath = path.join(root, "openclaw.json"); + const config = withFixturePlugin({} as OpenClawConfig, pluginDir); + await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 }); + return configPath; +} + +async function runMcpPluginToolsProof(options: ProducerOptions): Promise { + const fixture = await createFixturePlugin(); + // openclaw-temp-dir: allow standalone producer cleans and verifies this root in its finally block + const runtimeRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-plugin-tools-mcp-")); + const stateDir = path.join(runtimeRoot, "state"); + const homeDir = path.join(runtimeRoot, "home"); + await Promise.all([ + fs.mkdir(stateDir, { recursive: true }), + fs.mkdir(homeDir, { recursive: true }), + ]); + const configPath = await writePluginToolsConfig(runtimeRoot, fixture.pluginDir); + const stderrChunks: Buffer[] = []; + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + "--import", + "tsx", + "--eval", + `import(${JSON.stringify(pathToFileURL(path.join(options.repoRoot, "src/mcp/plugin-tools-serve.ts")).href)}).then((module) => module.servePluginToolsMcp())`, + ], + cwd: options.repoRoot, + env: { + ...process.env, + HOME: homeDir, + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_STATE_DIR: stateDir, + }, + stderr: "pipe", + }); + transport.stderr?.on("data", (chunk) => stderrChunks.push(Buffer.from(chunk))); + const client = new Client({ name: "qa-plugin-tools-client", version: "1.0.0" }); + let details = ""; + let proofError: Error | undefined; + try { + await client.connect(transport); + const listed = await client.listTools(); + if (!listed.tools.some((tool) => tool.name === FIXTURE_TOOL_NAME)) { + throw new Error( + `fixture plugin tool was not listed: ${listed.tools.map((tool) => tool.name).join(", ")}`, + ); + } + const result = await client.callTool({ + name: FIXTURE_TOOL_NAME, + arguments: { query: "ORBIT-9 codename", maxResults: 3 }, + }); + if (result.isError || !JSON.stringify(result.content).includes(FIXTURE_FACT)) { + throw new Error(`fixture plugin tool returned unexpected payload: ${JSON.stringify(result)}`); + } + details = `real plugin-tools pid=${transport.pid ?? "unknown"}; listed and called ${FIXTURE_TOOL_NAME}; received ORBIT-9`; + } catch (error) { + const stderr = Buffer.concat(stderrChunks).toString("utf8"); + proofError = new Error( + `${formatErrorMessage(error)}${stderr ? `\nplugin-tools stderr:\n${stderr}` : ""}`, + { + cause: error, + }, + ); + } finally { + await Promise.allSettled([client.close(), transport.close()]); + await Promise.all([fixture.cleanup(), fs.rm(runtimeRoot, { force: true, recursive: true })]); + if (existsSync(runtimeRoot) && !proofError) { + proofError = new Error(`plugin-tools runtime root was not cleaned up: ${runtimeRoot}`); + } + } + if (proofError) { + throw proofError; + } + return details; +} + +async function produceProof(options: ProducerOptions): Promise { + const startedAt = Date.now(); + try { + const details = + options.scenarioId === "gateway-smoke" + ? await runGatewaySmokeProof(options) + : options.scenarioId === "mcp-gateway-connect-startup-retry" + ? await runMcpGatewayStartupRetryProof(options) + : await runMcpPluginToolsProof(options); + return { details, durationMs: Math.max(1, Date.now() - startedAt), status: "pass" }; + } catch (error) { + return { + details: formatErrorMessage(error), + durationMs: Math.max(1, Date.now() - startedAt), + status: "fail", + }; + } +} + +export async function runGatewayMcpRealTransportProducer( + options: ProducerOptions, +): Promise { + const scenario = SCENARIOS[options.scenarioId]; + const writer = createQaScriptEvidenceWriter({ + artifactBase: options.artifactBase, + logFileName: `${options.scenarioId}.log`, + primaryModel: "mock-openai/gpt-5.5", + providerMode: "mock-openai", + repoRoot: options.repoRoot, + target: { + id: options.scenarioId, + title: scenario.title, + sourcePath: scenario.sourcePath, + primaryCoverageIds: scenario.primaryCoverageIds, + docsRefs: scenario.docsRefs, + codeRefs: scenario.codeRefs, + }, + }); + const result = await produceProof(options); + writer.appendLog(`${result.status}: ${result.details ?? "no details"}\n`); + return await writer.write(result); +} + +async function main(argv: readonly string[]) { + const options = parseOptions(argv); + const evidence = await runGatewayMcpRealTransportProducer(options); + const status = evidence.entries[0]?.result.status; + console.log(`Gateway/MCP real transport evidence: ${QA_EVIDENCE_FILENAME}`); + console.log(`Gateway/MCP real transport status: ${status}`); + return status === "pass" ? 0 : 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)) + .then((exitCode) => { + process.exit(exitCode); + }) + .catch((error: unknown) => { + console.error(formatErrorMessage(error)); + process.exitCode = 1; + }); +}