From 7daf990688e7989bf9bc75835c254d74c8b9a3cc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 4 Aug 2026 05:38:23 +0800 Subject: [PATCH] test(qa): cover Prometheus install and runtime export (#118867) --- .../src/install-runtime.e2e.test.ts | 374 ++++++++++++++++++ ...diagnostics-prometheus-plugin-install.yaml | 27 ++ .../runtime/docker-prometheus-smoke.yaml | 16 +- 3 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 extensions/diagnostics-prometheus/src/install-runtime.e2e.test.ts create mode 100644 qa/scenarios/observability/diagnostics-prometheus-plugin-install.yaml diff --git a/extensions/diagnostics-prometheus/src/install-runtime.e2e.test.ts b/extensions/diagnostics-prometheus/src/install-runtime.e2e.test.ts new file mode 100644 index 000000000000..c11d0c60de56 --- /dev/null +++ b/extensions/diagnostics-prometheus/src/install-runtime.e2e.test.ts @@ -0,0 +1,374 @@ +// Proves the external Prometheus plugin's managed install and trusted runtime boundary. +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import fs from "node:fs/promises"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; + +const execFileAsync = promisify(execFile); +const packageName = "@openclaw/diagnostics-prometheus"; +const pluginId = "diagnostics-prometheus"; +const repoRoot = path.resolve(import.meta.dirname, "../../.."); +const pluginRoot = path.resolve(import.meta.dirname, ".."); +const tempDirs: string[] = []; +const children: ChildProcess[] = []; + +async function stopChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + const exited = once(child, "exit").then(() => true); + child.kill("SIGTERM"); + if (!(await Promise.race([exited, delay(5_000, false)]))) { + child.kill("SIGKILL"); + await Promise.race([exited, delay(5_000)]); + } +} + +afterEach(async () => { + await Promise.all(children.splice(0).map(stopChild)); + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +async function makeTempDir(): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-prometheus-install-")); + tempDirs.push(dir); + return dir; +} + +async function reservePort(): Promise { + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("failed to reserve a loopback port"); + } + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + return address.port; +} + +function isolatedEnv(params: { + configPath: string; + home: string; + registry?: string; + stateDir: string; +}): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: params.home, + USERPROFILE: params.home, + OPENCLAW_HOME: params.home, + OPENCLAW_STATE_DIR: params.stateDir, + OPENCLAW_CONFIG_PATH: params.configPath, + OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1", + NODE_ENV: "production", + NO_COLOR: "1", + }; + for (const key of [ + "OPENCLAW_BUNDLED_PLUGINS_DIR", + "OPENCLAW_PLUGIN_CATALOG_PATHS", + "OPENCLAW_PLUGINS_PATHS", + "OPENCLAW_TEST_FAST", + "OPENCLAW_TEST_HOME", + "OPENCLAW_TEST_MINIMAL_GATEWAY", + "OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR", + "VITEST", + "VITEST_POOL_ID", + "VITEST_WORKER_ID", + ]) { + delete env[key]; + } + if (params.registry) { + env.NPM_CONFIG_REGISTRY = params.registry; + env.npm_config_registry = params.registry; + } + return env; +} + +async function runCli(args: string[], env: NodeJS.ProcessEnv, build = false): Promise { + const entry = build ? "scripts/run-node.mjs" : "openclaw.mjs"; + const result = await execFileAsync(process.execPath, [entry, ...args], { + cwd: repoRoot, + env, + maxBuffer: 4 * 1024 * 1024, + timeout: 180_000, + }); + return result.stdout; +} + +async function packPlugin(outputDir: string): Promise<{ + files: string[]; + tarballPath: string; +}> { + const stagingDir = path.join(outputDir, "package-source"); + await fs.cp(pluginRoot, stagingDir, { + recursive: true, + filter: (source) => { + const relative = path.relative(pluginRoot, source); + const topLevel = relative.split(path.sep)[0]; + return topLevel !== "dist" && topLevel !== "node_modules"; + }, + }); + await execFileAsync(process.execPath, ["scripts/lib/plugin-npm-runtime-build.mjs", stagingDir], { + cwd: repoRoot, + maxBuffer: 2 * 1024 * 1024, + timeout: 60_000, + }); + const result = await execFileAsync( + process.execPath, + [ + "scripts/lib/plugin-npm-package-manifest.mjs", + "--run", + stagingDir, + "--", + "npm", + "pack", + "--json", + "--ignore-scripts", + "--pack-destination", + outputDir, + ], + { + cwd: repoRoot, + env: { + ...process.env, + OPENCLAW_PLUGIN_NPM_BUNDLE_DEPENDENCIES: "1", + }, + maxBuffer: 2 * 1024 * 1024, + timeout: 60_000, + }, + ); + const entries = JSON.parse(result.stdout) as Array<{ + filename?: string; + files?: Array<{ path?: string }>; + }>; + const entry = entries[0]; + const filename = entry?.filename; + if (!filename) { + throw new Error("npm pack did not report the diagnostics-prometheus tarball"); + } + return { + files: (entry.files ?? []).flatMap((file) => + typeof file.path === "string" ? [file.path] : [], + ), + tarballPath: path.join(outputDir, filename), + }; +} + +async function readPluginVersion(): Promise { + const manifest = JSON.parse(await fs.readFile(path.join(pluginRoot, "package.json"), "utf8")) as { + version?: unknown; + }; + if (typeof manifest.version !== "string" || !manifest.version.trim()) { + throw new Error("diagnostics-prometheus package version is missing"); + } + return manifest.version.trim(); +} + +async function waitForFile( + filePath: string, + child: ChildProcess, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error("local npm registry exited before publishing its port"); + } + try { + if ((await fs.stat(filePath)).size > 0) { + return; + } + } catch { + // The registry writes the port file only after the listener is ready. + } + await delay(50); + } + throw new Error("timed out waiting for local npm registry"); +} + +async function startRegistry(params: { + root: string; + tarballPath: string; + version: string; +}): Promise { + const portFile = path.join(params.root, "registry-port"); + const logPath = path.join(params.root, "registry.log"); + const logHandle = await fs.open(logPath, "a"); + const child = spawn( + process.execPath, + [ + "scripts/e2e/lib/plugins/npm-registry-server.mjs", + portFile, + packageName, + params.version, + params.tarballPath, + ], + { + cwd: repoRoot, + env: isolatedEnv({ + configPath: path.join(params.root, "unused.json"), + home: params.root, + stateDir: path.join(params.root, "unused-state"), + }), + stdio: ["ignore", logHandle.fd, logHandle.fd], + }, + ); + children.push(child); + await logHandle.close(); + await waitForFile(portFile, child, 10_000); + const port = (await fs.readFile(portFile, "utf8")).trim(); + return `http://127.0.0.1:${port}`; +} + +async function waitForGateway(params: { + child: ChildProcess; + logPath: string; + port: number; +}): Promise { + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + if (params.child.exitCode !== null || params.child.signalCode !== null) { + const logs = await fs.readFile(params.logPath, "utf8").catch(() => ""); + throw new Error(`Gateway exited before readiness:\n${logs.slice(-8_000)}`); + } + try { + const response = await fetch(`http://127.0.0.1:${params.port}/readyz`, { + signal: AbortSignal.timeout(1_000), + }); + if (response.ok) { + return; + } + } catch { + // Readiness is authoritative only after the HTTP endpoint responds. + } + await delay(200); + } + const logs = await fs.readFile(params.logPath, "utf8").catch(() => ""); + throw new Error(`Gateway did not become ready:\n${logs.slice(-8_000)}`); +} + +describe("diagnostics-prometheus managed install runtime", () => { + it("installs the exact official package and exports metrics at Gateway startup", async () => { + const root = await makeTempDir(); + const home = path.join(root, "home"); + const stateDir = path.join(root, "state"); + const configPath = path.join(stateDir, "openclaw.json"); + const gatewayLog = path.join(root, "gateway.log"); + const gatewayToken = "prometheus-managed-install-test-token"; + const gatewayPort = await reservePort(); + await fs.mkdir(home, { recursive: true }); + await fs.mkdir(stateDir, { recursive: true }); + await fs.writeFile( + configPath, + `${JSON.stringify( + { + diagnostics: { enabled: true }, + gateway: { + mode: "local", + bind: "loopback", + port: gatewayPort, + auth: { mode: "token", token: gatewayToken }, + }, + }, + null, + 2, + )}\n`, + "utf8", + ); + + const pluginVersion = await readPluginVersion(); + const packedPlugin = await packPlugin(root); + expect(packedPlugin.files.some((file) => /^dist\/index\.(?:js|mjs|cjs)$/u.test(file))).toBe( + true, + ); + const registry = await startRegistry({ + root, + tarballPath: packedPlugin.tarballPath, + version: pluginVersion, + }); + const env = isolatedEnv({ + configPath, + home, + registry, + stateDir, + }); + + await runCli(["plugins", "install", `npm:${packageName}@${pluginVersion}`], env, true); + const inspect = JSON.parse( + await runCli(["plugins", "inspect", pluginId, "--runtime", "--json"], env), + ) as { + install?: { + artifactKind?: unknown; + installPath?: unknown; + resolvedName?: unknown; + resolvedVersion?: unknown; + source?: unknown; + sourcePath?: unknown; + }; + plugin?: { + enabled?: unknown; + id?: unknown; + origin?: unknown; + status?: unknown; + trustedOfficialInstall?: unknown; + }; + }; + expect(inspect.install).toMatchObject({ + source: "npm", + resolvedName: packageName, + resolvedVersion: pluginVersion, + }); + expect(typeof inspect.install?.installPath).toBe("string"); + expect(inspect.install?.artifactKind).toBeUndefined(); + expect(inspect.install?.sourcePath).toBeUndefined(); + expect(inspect.plugin).toMatchObject({ + id: pluginId, + enabled: true, + status: "loaded", + trustedOfficialInstall: true, + }); + expect(["config", "global"]).toContain(inspect.plugin?.origin); + + const gatewayLogHandle = await fs.open(gatewayLog, "a"); + const gateway = spawn( + process.execPath, + ["openclaw.mjs", "gateway", "run", "--bind", "loopback", "--port", String(gatewayPort)], + { + cwd: repoRoot, + env, + stdio: ["ignore", gatewayLogHandle.fd, gatewayLogHandle.fd], + }, + ); + children.push(gateway); + await gatewayLogHandle.close(); + await waitForGateway({ child: gateway, logPath: gatewayLog, port: gatewayPort }); + + const url = `http://127.0.0.1:${gatewayPort}/api/diagnostics/prometheus`; + const unauthenticated = await fetch(url); + expect([401, 403]).toContain(unauthenticated.status); + const authenticated = await fetch(url, { + headers: { authorization: `Bearer ${gatewayToken}` }, + }); + const body = await authenticated.text(); + expect(authenticated.status).toBe(200); + expect(authenticated.headers.get("content-type")).toContain("text/plain"); + expect(body).toContain( + 'openclaw_telemetry_exporter_total{exporter="diagnostics-prometheus",reason="configured",signal="metrics",status="started"} 1', + ); + const gatewayLogs = await fs.readFile(gatewayLog, "utf8"); + expect(gatewayLogs).not.toContain( + "diagnostics-prometheus: internal diagnostics capability unavailable", + ); + }, 300_000); +}); diff --git a/qa/scenarios/observability/diagnostics-prometheus-plugin-install.yaml b/qa/scenarios/observability/diagnostics-prometheus-plugin-install.yaml new file mode 100644 index 000000000000..77bda5bdf685 --- /dev/null +++ b/qa/scenarios/observability/diagnostics-prometheus-plugin-install.yaml @@ -0,0 +1,27 @@ +title: Diagnostics Prometheus managed install and runtime + +scenario: + id: diagnostics-prometheus-plugin-install + surface: telemetry + coverage: + primary: + - observability.diagnostics-prometheus-plugin-install + objective: Verify the exact checkout plugin installs through the official npm path, loads as a trusted managed plugin, and exports runtime metrics after Gateway startup. + successCriteria: + - The exact checkout plugin package installs through the canonical npm plugin command. + - The managed npm install record identifies the official package and installed version. + - Runtime inspection reports the enabled plugin loaded from the trusted official install. + - An isolated Gateway exposes the protected Prometheus endpoint from the installed plugin. + - The authenticated scrape contains the diagnostics-prometheus exporter-start sample. + docsRefs: + - docs/plugins/reference/diagnostics-prometheus.md + - docs/gateway/prometheus.md + codeRefs: + - extensions/diagnostics-prometheus/src/install-runtime.e2e.test.ts + - src/plugins/official-external-install-trust.ts + - src/plugins/manifest-registry.ts + - src/plugins/services.ts + execution: + kind: vitest + path: extensions/diagnostics-prometheus/src/install-runtime.e2e.test.ts + summary: Install the exact checkout plugin from a local npm registry, start an isolated Gateway, and scrape its trusted runtime exporter. diff --git a/qa/scenarios/runtime/docker-prometheus-smoke.yaml b/qa/scenarios/runtime/docker-prometheus-smoke.yaml index 2141fb75c7b8..54e696c082dd 100644 --- a/qa/scenarios/runtime/docker-prometheus-smoke.yaml +++ b/qa/scenarios/runtime/docker-prometheus-smoke.yaml @@ -7,8 +7,10 @@ scenario: primary: - observability.prometheus-api-auth - observability.prometheus - secondary: - observability.prometheus-qa-lab + - observability.prometheus-runtime-validation + - observability.telemetry-prometheus + secondary: - containers.e2e-plan-runtime-validation objective: Verify a QA-lab gateway run emits protected, bounded Prometheus diagnostics metrics through the diagnostics-prometheus plugin. successCriteria: @@ -128,6 +130,18 @@ flow: - assert: expr: "prometheusText.includes('# TYPE openclaw_harness_run_total counter')" message: "missing harness run counter" + - assert: + expr: "/^openclaw_run_completed_total\\{[^}\\n]*outcome=\"completed\"[^}\\n]*\\}\\s+[1-9]\\d*(?:\\.\\d+)?$/m.test(prometheusText)" + message: "missing positive completed-run sample" + - assert: + expr: "/^openclaw_model_call_total\\{[^}\\n]*outcome=\"completed\"[^}\\n]*\\}\\s+[1-9]\\d*(?:\\.\\d+)?$/m.test(prometheusText)" + message: "missing positive completed model-call sample" + - assert: + expr: "/^openclaw_harness_run_total\\{[^}\\n]*outcome=\"completed\"[^}\\n]*\\}\\s+[1-9]\\d*(?:\\.\\d+)?$/m.test(prometheusText)" + message: "missing positive completed harness-run sample" + - assert: + expr: 'prometheusText.includes(''openclaw_telemetry_exporter_total{exporter="diagnostics-prometheus",reason="configured",signal="metrics",status="started"} 1'')' + message: "missing diagnostics-prometheus exporter-start sample" - assert: expr: "!prometheusText.includes(config.secretNeedle)" message: "prometheus output leaked prompt sentinel"