diff --git a/extensions/qa-lab/src/managed-gateway-service-lifecycle.e2e.test.ts b/extensions/qa-lab/src/managed-gateway-service-lifecycle.e2e.test.ts new file mode 100644 index 000000000000..fd92fdaef349 --- /dev/null +++ b/extensions/qa-lab/src/managed-gateway-service-lifecycle.e2e.test.ts @@ -0,0 +1,73 @@ +// QA Lab owns the real child/provider fixture used by the managed service proof. +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { startQaGatewayChild } from "./gateway-child.js"; +import { startQaProviderServer } from "./providers/server-runtime.js"; + +const repoRoot = fileURLToPath(new URL("../../../", import.meta.url)); + +async function fetchJson(url: string, label: string): Promise> { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`${label} returned HTTP ${response.status}`); + } + const body: unknown = await response.json(); + if (!body || typeof body !== "object" || Array.isArray(body)) { + throw new Error(`${label} did not return a JSON object`); + } + return body as Record; +} + +describe("managed gateway service lifecycle product proof", () => { + it( + "runs a foreground gateway with live, ready, and RPC health", + { timeout: 180_000 }, + async () => { + const provider = await startQaProviderServer("mock-openai"); + if (!provider) { + throw new Error("mock OpenAI provider did not start"); + } + let gateway: Awaited> | undefined; + let proofFailure: Error | undefined; + try { + gateway = await startQaGatewayChild({ + repoRoot, + useRepoCli: true, + providerBaseUrl: `${provider.baseUrl}/v1`, + providerMode: "mock-openai", + transportBaseUrl: "http://127.0.0.1", + controlUiEnabled: false, + }); + expect(gateway.pid).toBeGreaterThan(0); + await expect( + fetchJson(`${gateway.baseUrl}/healthz`, "gateway /healthz"), + ).resolves.toMatchObject({ status: "live" }); + await expect( + fetchJson(`${gateway.baseUrl}/readyz`, "gateway /readyz"), + ).resolves.toMatchObject({ ready: true }); + const rpcHealth = await gateway.call("health", {}); + expect(rpcHealth).not.toBeNull(); + expect(Array.isArray(rpcHealth)).toBe(false); + expect(typeof rpcHealth).toBe("object"); + } catch (error) { + proofFailure = error instanceof Error ? error : new Error(String(error)); + } + const cleanup = await Promise.allSettled([gateway?.stop(), provider.stop()]); + const cleanupFailures = cleanup.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (proofFailure && cleanupFailures.length > 0) { + throw new AggregateError( + [proofFailure, ...cleanupFailures], + "foreground gateway proof and cleanup failed", + ); + } + if (proofFailure) { + throw proofFailure; + } + if (cleanupFailures.length > 0) { + throw new AggregateError(cleanupFailures, "foreground gateway cleanup failed"); + } + }, + ); +}); diff --git a/qa/scenarios/cli/managed-gateway-service-lifecycle.yaml b/qa/scenarios/cli/managed-gateway-service-lifecycle.yaml new file mode 100644 index 000000000000..2f35db35c95e --- /dev/null +++ b/qa/scenarios/cli/managed-gateway-service-lifecycle.yaml @@ -0,0 +1,43 @@ +title: Managed gateway service lifecycle + +scenario: + id: managed-gateway-service-lifecycle + surface: cli + category: cli.gateway-service-management + coverage: + primary: + - cli.foreground-gateway-runs + - cli.service-install-and-control + - cli.drift-and-reinstall-recovery + - cli.service-health-checks + objective: Exercise foreground startup and the managed gateway install, control, repair, and health boundaries across the supported service adapters. + successCriteria: + - The real foreground gateway CLI starts in an isolated state dir, reports live and ready HTTP probes plus RPC health, and terminates cleanly. + - Foreground option validation reports invalid or unsafe startup requests before launch. + - Managed install and control exercise the CLI owner boundary plus launchd, systemd, and Windows Scheduled Task adapter contracts. + - Drifted service definitions and environments trigger explicit reinstall or repair behavior instead of being accepted as current. + - Restart and status flows distinguish healthy, stale, failed, and timed-out post-operation outcomes. + - On macOS and Windows hosts, the producer also runs the native launchd or Scheduled Task integration lane. + docsRefs: + - docs/cli/gateway.md + - docs/install/updating.md + - docs/gateway/troubleshooting.md + - docs/reference/test.md + codeRefs: + - test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.ts + - extensions/qa-lab/src/managed-gateway-service-lifecycle.e2e.test.ts + - src/cli/gateway-cli/run.option-collisions.test.ts + - src/cli/daemon-cli/install.integration.test.ts + - src/cli/daemon-cli/lifecycle.test.ts + - src/cli/daemon-cli/restart-health.test.ts + - src/daemon/launchd.integration.e2e.test.ts + - src/daemon/systemd.test.ts + - src/daemon/schtasks.integration.e2e.test.ts + execution: + kind: script + path: test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.ts + summary: Runs four separately reported CLI lifecycle phases, all supported service-adapter contracts, and the current host's native service integration lane. + timeoutMs: 1800000 + args: + - --artifact-base + - ${outputDir} diff --git a/qa/scenarios/config/system-agent-ring-zero-setup.yaml b/qa/scenarios/config/system-agent-ring-zero-setup.yaml index b86f69ce9449..bb5feb562a73 100644 --- a/qa/scenarios/config/system-agent-ring-zero-setup.yaml +++ b/qa/scenarios/config/system-agent-ring-zero-setup.yaml @@ -4,6 +4,9 @@ scenario: id: system-agent-ring-zero-setup surface: config coverage: + primary: + - cli.service-auth-wiring-create + - cli.service-auth-wiring-system-agent-setup secondary: - channels.setup-onboarding-flows-system-agent-setup - discord.token-and-application-id-configuration @@ -13,7 +16,8 @@ scenario: - OpenClaw fails closed in an empty state dir and directs the user to inference onboarding. - The packaged activation module tests a fake Claude inference backend before persisting its model, with remaining setup deferred to OpenClaw. - A fuzzy packaged CLI request reaches the verified planner and resolves to a typed setup operation only after activation. - - Supporting one-shot commands write the workspace/model and create a non-main agent. + - The approved openclaw.setup operation independently writes the default workspace and model. + - The approved agents.create operation independently creates a non-main agent with its own workspace. - Discord is enabled and configured through an env SecretRef without persisting the raw token. - Config validation passes and audit entries exist for every applied write. docsRefs: @@ -27,7 +31,7 @@ scenario: execution: kind: script path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts - summary: Redirects to the packaged inference-gate and typed-operation Docker lane; coverage remains secondary until a real interactive agent/tool/approval flow is exercised. + summary: Runs the packaged inference-gate and typed-operation Docker lane, with separate assertions for openclaw.setup and agents.create; interactive channel coverage remains secondary. args: - --lane - system-agent-first-run diff --git a/test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.test.ts b/test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.test.ts new file mode 100644 index 000000000000..a0d55b0c8237 --- /dev/null +++ b/test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.test.ts @@ -0,0 +1,57 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + QA_EVIDENCE_FILENAME, + validateQaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/api.js"; +import { testing } from "./managed-gateway-service-lifecycle-product-proof.js"; + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawnSync: vi.fn(), +})); + +const tempRoots: string[] = []; + +afterEach(async () => { + vi.mocked(spawnSync).mockReset(); + await Promise.all( + tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })), + ); +}); + +describe("managed gateway service lifecycle evidence producer", () => { + it("records failed evidence and exits nonzero when a child phase is signaled", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-service-lifecycle-")); + const artifactBase = path.join(root, "artifacts"); + tempRoots.push(root); + vi.mocked(spawnSync).mockReturnValue({ + output: [null, null, null], + pid: 123, + signal: "SIGTERM", + status: null, + stderr: Buffer.alloc(0), + stdout: Buffer.alloc(0), + }); + + await expect(testing.main(["--artifact-base", artifactBase])).resolves.toBe(1); + + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(path.join(artifactBase, QA_EVIDENCE_FILENAME), "utf8")), + ); + expect(evidence.entries[0]).toMatchObject({ + result: { + failure: { + reason: expect.stringContaining("terminated by signal SIGTERM"), + }, + status: "fail", + }, + }); + await expect( + fs.readFile(path.join(artifactBase, "managed-gateway-service-lifecycle.log"), "utf8"), + ).resolves.toContain("fail: foreground-cli-runtime terminated by signal SIGTERM"); + }); +}); diff --git a/test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.ts b/test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.ts new file mode 100644 index 000000000000..4696f5db93b0 --- /dev/null +++ b/test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.ts @@ -0,0 +1,267 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + QA_EVIDENCE_FILENAME, + type QaEvidenceSummaryJson, + validateQaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/api.js"; +import { createQaScriptEvidenceWriter } from "./script-evidence.js"; + +const SCENARIO_ID = "managed-gateway-service-lifecycle"; +const SOURCE_PATH = "test/e2e/qa-lab/runtime/managed-gateway-service-lifecycle-product-proof.ts"; + +type ProofPhase = { + coverageIds: string[]; + name: string; + passDetails?: string; + testPaths: string[]; + vitestArgs?: string[]; +}; + +type ProducerOptions = { + artifactBase: string; + repoRoot: string; +}; + +type ProofResult = { + coverageIds: string[]; + name: string; + status: "passed"; +}; + +const phases: ProofPhase[] = [ + { + name: "foreground-cli-runtime", + coverageIds: ["cli.foreground-gateway-runs"], + passDetails: "/healthz=live, /readyz=ready, health RPC=object, child/provider cleanup=passed", + testPaths: ["extensions/qa-lab/src/managed-gateway-service-lifecycle.e2e.test.ts"], + vitestArgs: ["run", "--config", "test/vitest/vitest.e2e.config.ts"], + }, + { + name: "foreground-cli-option-validation", + coverageIds: ["cli.foreground-gateway-runs"], + testPaths: ["src/cli/gateway-cli/run.option-collisions.test.ts"], + }, + { + name: "install-and-control", + coverageIds: ["cli.service-install-and-control"], + testPaths: [ + "src/cli/daemon-cli/install.integration.test.ts", + "src/cli/daemon-cli/register-service-commands.test.ts", + "src/daemon/launchd-system.test.ts", + "src/daemon/systemd.test.ts", + "src/daemon/systemd-system.test.ts", + "src/daemon/systemd-unit.test.ts", + "src/daemon/schtasks.install.test.ts", + "src/daemon/schtasks.stop.test.ts", + ], + }, + { + name: "drift-and-reinstall-recovery", + coverageIds: ["cli.drift-and-reinstall-recovery"], + testPaths: ["src/cli/daemon-cli/install.test.ts", "src/cli/daemon-cli/lifecycle.test.ts"], + }, + { + name: "post-operation-health", + coverageIds: ["cli.service-health-checks"], + testPaths: [ + "src/cli/daemon-cli/restart-health.test.ts", + "src/cli/daemon-cli/restart-health-probe.test.ts", + "src/cli/daemon-cli/restart-health-wait.test.ts", + "src/cli/daemon-cli/status.test.ts", + ], + }, +]; + +if (process.platform === "darwin") { + phases.push({ + name: "native-launchd", + coverageIds: ["cli.service-install-and-control", "cli.service-health-checks"], + testPaths: ["src/daemon/launchd.integration.e2e.test.ts"], + vitestArgs: ["run", "--config", "test/vitest/vitest.e2e.config.ts"], + }); +} else if (process.platform === "win32") { + phases.push({ + name: "native-scheduled-task", + coverageIds: ["cli.service-install-and-control", "cli.service-health-checks"], + testPaths: ["src/daemon/schtasks.integration.e2e.test.ts"], + vitestArgs: ["run", "--config", "test/vitest/vitest.e2e.config.ts"], + }); +} + +function formatErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function parseOptions(args: string[]): ProducerOptions { + if (args.length !== 2 || args[0] !== "--artifact-base" || !args[1]) { + throw new Error("usage: --artifact-base "); + } + return { + artifactBase: path.resolve(args[1]), + repoRoot: process.cwd(), + }; +} + +function runVitestPhase(phase: ProofPhase, log: (message: string) => void): ProofResult { + log(`[managed-gateway-service] START ${phase.name} (${phase.coverageIds.join(", ")})`); + const result = spawnSync( + process.execPath, + [ + "scripts/run-vitest.mjs", + ...(phase.vitestArgs ?? []), + ...phase.testPaths, + "--reporter=verbose", + ], + { + cwd: process.cwd(), + env: process.env, + stdio: "inherit", + }, + ); + if (result.error) { + throw result.error; + } + if (result.signal) { + throw new Error( + `${phase.name} terminated by signal ${result.signal} for ${phase.coverageIds.join(", ")}`, + ); + } + if (result.status !== 0) { + throw new Error( + `${phase.name} failed with exit ${result.status ?? "unknown"} for ${phase.coverageIds.join(", ")}`, + ); + } + log( + `[managed-gateway-service] PASS ${phase.name}${phase.passDetails ? ` (${phase.passDetails})` : ""}`, + ); + return { + coverageIds: phase.coverageIds, + name: phase.name, + status: "passed", + }; +} + +async function assertFreshEvidence(params: { + artifactBase: string; + expectedStatus: "pass" | "fail"; + startedAt: number; +}): Promise { + const evidencePath = path.join(params.artifactBase, QA_EVIDENCE_FILENAME); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(evidencePath, "utf8")), + ); + if (Date.parse(evidence.generatedAt) < params.startedAt) { + throw new Error(`${QA_EVIDENCE_FILENAME} was not written by the current producer run`); + } + const entry = evidence.entries.find((candidate) => candidate.test.id === SCENARIO_ID); + if (!entry || entry.result.status !== params.expectedStatus) { + throw new Error( + `${QA_EVIDENCE_FILENAME} does not record ${SCENARIO_ID} as ${params.expectedStatus}`, + ); + } +} + +function createEvidenceWriter(options: ProducerOptions) { + return createQaScriptEvidenceWriter({ + artifactBase: options.artifactBase, + logFileName: "managed-gateway-service-lifecycle.log", + primaryModel: "mock-openai/gpt-5.6-luna", + providerMode: "mock-openai", + repoRoot: options.repoRoot, + target: { + codeRefs: [ + SOURCE_PATH, + "extensions/qa-lab/src/managed-gateway-service-lifecycle.e2e.test.ts", + "src/cli/gateway-cli/run.option-collisions.test.ts", + "src/cli/daemon-cli/install.integration.test.ts", + "src/cli/daemon-cli/lifecycle.test.ts", + "src/cli/daemon-cli/restart-health.test.ts", + "src/daemon/launchd.integration.e2e.test.ts", + "src/daemon/systemd.test.ts", + "src/daemon/schtasks.integration.e2e.test.ts", + ], + docsRefs: [ + "docs/cli/gateway.md", + "docs/install/updating.md", + "docs/gateway/troubleshooting.md", + "docs/reference/test.md", + ], + id: SCENARIO_ID, + sourcePath: SOURCE_PATH, + title: "Managed gateway service lifecycle", + }, + }); +} + +async function runProducer(options: ProducerOptions): Promise { + const startedAt = Date.now(); + const writer = createEvidenceWriter(options); + const log = (message: string) => { + console.log(message); + writer.appendLog(`${message}\n`); + }; + + try { + const results: ProofResult[] = []; + for (const phase of phases) { + results.push(runVitestPhase(phase, log)); + } + log( + JSON.stringify({ + kind: "managed-gateway-service-lifecycle-proof", + platform: process.platform, + phases: results, + }), + ); + const evidence = await writer.write({ + durationMs: Math.max(1, Date.now() - startedAt), + status: "pass", + }); + await assertFreshEvidence({ + artifactBase: options.artifactBase, + expectedStatus: "pass", + startedAt, + }); + return evidence; + } catch (error) { + const details = formatErrorMessage(error); + writer.appendLog(`fail: ${details}\n`); + const evidence = await writer.write({ + details, + durationMs: Math.max(1, Date.now() - startedAt), + status: "fail", + }); + await assertFreshEvidence({ + artifactBase: options.artifactBase, + expectedStatus: "fail", + startedAt, + }); + return evidence; + } +} + +async function main(args: string[]): Promise { + const evidence = await runProducer(parseOptions(args)); + const status = evidence.entries[0]?.result.status; + console.log(`Managed gateway service lifecycle evidence: ${QA_EVIDENCE_FILENAME}`); + console.log(`Managed gateway service lifecycle status: ${status}`); + return status === "pass" ? 0 : 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)) + .then((exitCode) => { + process.exitCode = exitCode; + }) + .catch((error: unknown) => { + console.error(formatErrorMessage(error)); + process.exitCode = 1; + }); +} + +export const testing = { + main, +};