From 12beeb7b29476d0ac96590e51bbc14fff63fbe60 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 4 Aug 2026 11:13:05 +0800 Subject: [PATCH] test(qa): cover packaged browser profiles (#119043) --- .../browser-plugin-profiles-packaged.yaml | 37 ++++ scripts/e2e/browser-plugin-profiles-docker.sh | 180 ++++++++++++++++++ .../browser-plugin-profiles-packaged.test.ts | 59 ++++++ .../browser-plugin-profiles-packaged.ts | 132 +++++++++++++ 4 files changed, 408 insertions(+) create mode 100644 qa/scenarios/runtime/browser-plugin-profiles-packaged.yaml create mode 100644 scripts/e2e/browser-plugin-profiles-docker.sh create mode 100644 test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.test.ts create mode 100644 test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts diff --git a/qa/scenarios/runtime/browser-plugin-profiles-packaged.yaml b/qa/scenarios/runtime/browser-plugin-profiles-packaged.yaml new file mode 100644 index 000000000000..0fd603bff7e5 --- /dev/null +++ b/qa/scenarios/runtime/browser-plugin-profiles-packaged.yaml @@ -0,0 +1,37 @@ +title: Packaged browser plugin profiles + +scenario: + id: browser-plugin-profiles-packaged + surface: runtime-tool + category: tools.browser-automation + coverage: + primary: + - tools.browser-plugin-service + - tools.profiles + objective: Verify the packaged Gateway lazily activates the bundled browser plugin, routes authenticated browser CLI requests, and manages a real headless Chromium profile through cleanup. + successCriteria: + - The functional Docker image runs the packaged dist application and installs the Chromium revision pinned by its packaged playwright-core dependency. + - Gateway startup does not eagerly start browser control; the first authenticated browser CLI request starts it exactly once through browser.request. + - The CLI creates, lists, and selects a managed profile, then reports its stopped status. + - Starting the selected profile headless launches Chromium and exposes a working CDP /json/version endpoint. + - Stopping the profile closes CDP and reaps the Chromium child before the profile is deleted. + - The profile is absent after deletion, and the Gateway and container stop cleanly. + docsRefs: + - docs/tools/browser.md + - docs/tools/browser-control.md + - docs/help/testing.md + codeRefs: + - extensions/browser/plugin-registration.ts + - extensions/browser/src/gateway/browser-request.ts + - extensions/browser/src/control-service.ts + - extensions/browser/src/browser/profiles-service.ts + - scripts/e2e/browser-plugin-profiles-docker.sh + - test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts + execution: + kind: script + path: test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts + summary: Runs a packaged Docker Gateway and real Chromium through the authenticated browser profile lifecycle. + timeoutMs: 1800000 + args: + - --artifact-base + - ${outputDir} diff --git a/scripts/e2e/browser-plugin-profiles-docker.sh b/scripts/e2e/browser-plugin-profiles-docker.sh new file mode 100644 index 000000000000..548bfe957475 --- /dev/null +++ b/scripts/e2e/browser-plugin-profiles-docker.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh" + +BASE_IMAGE="$(docker_e2e_resolve_image "openclaw-browser-plugin-profiles-base-e2e" OPENCLAW_BROWSER_PLUGIN_PROFILES_BASE_IMAGE)" +IMAGE_NAME="${OPENCLAW_BROWSER_PLUGIN_PROFILES_IMAGE:-openclaw-browser-plugin-profiles-e2e:${OPENCLAW_DOCKER_ALL_LANE_NAME:-local}}" +CONTAINER_NAME="openclaw-browser-plugin-profiles-e2e-$$" +BUILD_DIR="" +PORT="18789" +TOKEN="browser-plugin-profiles-token" +PROFILE="qa-browser" +DOCKER_COMMAND_TIMEOUT="${OPENCLAW_BROWSER_PLUGIN_PROFILES_DOCKER_TIMEOUT:-1200s}" +PLAYWRIGHT_CORE_VERSION="$( + node -p 'require(process.argv[1]).dependencies["playwright-core"]' \ + "$ROOT_DIR/extensions/browser/package.json" +)" + +cleanup() { + if [ -n "$CONTAINER_NAME" ] && docker_e2e_docker_cmd inspect "$CONTAINER_NAME" >/dev/null 2>&1; then + docker_e2e_docker_cmd rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi + if [ -n "$BUILD_DIR" ]; then + rm -rf "$BUILD_DIR" + fi +} +trap cleanup EXIT INT TERM + +if [ "${OPENCLAW_BROWSER_PLUGIN_PROFILES_SKIP_BUILD:-0}" = "1" ]; then + docker_e2e_docker_cmd image inspect "$IMAGE_NAME" >/dev/null +else + docker_e2e_build_or_reuse "$BASE_IMAGE" browser-plugin-profiles-base + BUILD_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-browser-plugin-profiles-build.XXXXXX")" + cat >"$BUILD_DIR/Dockerfile" </dev/null +node \"\$entry\" config set browser.noSandbox true >/dev/null +openclaw_e2e_exec_gateway \"\$entry\" $PORT loopback /tmp/browser-plugin-profiles-gateway.log" >/dev/null + +if ! docker_e2e_wait_container_bash "$CONTAINER_NAME" 240 0.5 \ + "source scripts/lib/openclaw-e2e-instance.sh; openclaw_e2e_probe_tcp 127.0.0.1 $PORT"; then + echo "Packaged browser Gateway failed to become ready" >&2 + docker_e2e_tail_container_file_if_running \ + "$CONTAINER_NAME" "/tmp/browser-plugin-profiles-gateway.log" 160 + exit 1 +fi + +if docker_e2e_docker_cmd exec "$CONTAINER_NAME" \ + grep -q "Browser control service ready" /tmp/browser-plugin-profiles-gateway.log; then + echo "Browser control service started eagerly" >&2 + exit 1 +fi + +if [ -z "$BUILD_DIR" ]; then + BUILD_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-browser-plugin-profiles-run.XXXXXX")" +fi +LIFECYCLE_SCRIPT="$BUILD_DIR/profile-lifecycle.sh" +cat >"$LIFECYCLE_SCRIPT" <<'CONTAINER' +set -euo pipefail +PORT="$1" +TOKEN="$2" +PROFILE="$3" +source /tmp/openclaw-test-state-env +source scripts/lib/openclaw-e2e-instance.sh +entry="$(openclaw_e2e_resolve_entrypoint)" +base=(--url "ws://127.0.0.1:$PORT" --token "$TOKEN" --json) +browser() { node "$entry" browser "${base[@]}" "$@"; } +profile() { node "$entry" browser "${base[@]}" --browser-profile "$PROFILE" "$@"; } + +browser profiles >/tmp/profiles-initial.json +node -e 'const d=JSON.parse(require("node:fs").readFileSync(process.argv[1])); if (!Array.isArray(d.profiles) || d.profiles.some((p) => p.name === process.argv[2])) throw new Error("profile unexpectedly present")' \ + /tmp/profiles-initial.json "$PROFILE" + +browser create-profile --name "$PROFILE" --color "#1A73E8" >/tmp/profile-created.json +browser profiles >/tmp/profiles-created.json +node -e 'const d=JSON.parse(require("node:fs").readFileSync(process.argv[1])); if (!d.profiles.some((p) => p.name === process.argv[2])) throw new Error("created profile missing")' \ + /tmp/profiles-created.json "$PROFILE" + +profile status >/tmp/profile-stopped.json +node -e 'const d=JSON.parse(require("node:fs").readFileSync(process.argv[1])); if (d.profile !== process.argv[2] || d.running) throw new Error("selected profile was not stopped")' \ + /tmp/profile-stopped.json "$PROFILE" + +profile start --headless >/tmp/profile-started.json +profile status >/tmp/profile-running.json +BROWSER_PID="$(node -p 'JSON.parse(require("node:fs").readFileSync(process.argv[1])).pid' /tmp/profile-running.json)" +CDP_PORT="$(node -p 'JSON.parse(require("node:fs").readFileSync(process.argv[1])).cdpPort' /tmp/profile-running.json)" +node -e 'const d=JSON.parse(require("node:fs").readFileSync(process.argv[1])); if (!d.running || !d.headless || !d.cdpReady || !Number.isInteger(d.pid) || !Number.isInteger(d.cdpPort)) throw new Error("headless browser status incomplete")' \ + /tmp/profile-running.json +kill -0 "$BROWSER_PID" +node --input-type=module -e 'const response=await fetch(`http://127.0.0.1:${process.argv[1]}/json/version`); const body=await response.json(); if (!response.ok || typeof body.webSocketDebuggerUrl !== "string") throw new Error("CDP /json/version unavailable")' \ + "$CDP_PORT" + +profile stop >/tmp/profile-stopped-final.json +for _ in $(seq 1 80); do + if ! kill -0 "$BROWSER_PID" 2>/dev/null && + ! openclaw_e2e_probe_tcp 127.0.0.1 "$CDP_PORT" 2>/dev/null; then + break + fi + sleep 0.25 +done +! kill -0 "$BROWSER_PID" 2>/dev/null +! openclaw_e2e_probe_tcp 127.0.0.1 "$CDP_PORT" 2>/dev/null +profile status >/tmp/profile-closed.json +node -e 'const d=JSON.parse(require("node:fs").readFileSync(process.argv[1])); if (d.running || d.cdpReady) throw new Error("browser remained open after stop")' \ + /tmp/profile-closed.json + +browser delete-profile --name "$PROFILE" >/tmp/profile-deleted.json +browser profiles >/tmp/profiles-final.json +node -e 'const d=JSON.parse(require("node:fs").readFileSync(process.argv[1])); if (d.profiles.some((p) => p.name === process.argv[2])) throw new Error("deleted profile remained listed")' \ + /tmp/profiles-final.json "$PROFILE" +printf 'profile=%s browserPid=%s cdpPort=%s playwrightCore=%s\n' \ + "$PROFILE" "$BROWSER_PID" "$CDP_PORT" "$(node -p 'require("/app/node_modules/playwright-core/package.json").version')" +CONTAINER +docker_e2e_docker_cmd cp \ + "$LIFECYCLE_SCRIPT" "$CONTAINER_NAME:/tmp/browser-plugin-profiles-lifecycle.sh" + +if ! docker_e2e_docker_cmd exec "$CONTAINER_NAME" \ + bash /tmp/browser-plugin-profiles-lifecycle.sh "$PORT" "$TOKEN" "$PROFILE" +then + echo "Packaged browser plugin profile lifecycle failed" >&2 + docker_e2e_tail_container_file_if_running \ + "$CONTAINER_NAME" "/tmp/browser-plugin-profiles-gateway.log" 200 + exit 1 +fi + +READY_COUNT="$( + docker_e2e_docker_cmd exec "$CONTAINER_NAME" \ + grep -c "Browser control service ready" /tmp/browser-plugin-profiles-gateway.log || true +)" +if [ "$READY_COUNT" != "1" ]; then + echo "Expected exactly one lazy browser control service start, got $READY_COUNT" >&2 + exit 1 +fi + +docker_e2e_docker_cmd stop --time 20 "$CONTAINER_NAME" >/dev/null +if [ "$(docker_e2e_docker_cmd inspect --format '{{.State.Running}}' "$CONTAINER_NAME")" != "false" ]; then + echo "Gateway container remained running after graceful stop" >&2 + exit 1 +fi +docker_e2e_docker_cmd rm "$CONTAINER_NAME" >/dev/null +CONTAINER_NAME="" + +echo "BROWSER_PLUGIN_PROFILES_PACKAGED_OK" diff --git a/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.test.ts b/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.test.ts new file mode 100644 index 000000000000..f8766d1c8616 --- /dev/null +++ b/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { + classifyBrowserPluginProfilesOutcome, + parseBrowserPluginProfilesOptions, +} from "./browser-plugin-profiles-packaged.js"; + +describe("packaged browser plugin profiles evidence producer", () => { + it("parses the scenario artifact directory", () => { + expect( + parseBrowserPluginProfilesOptions(["--artifact-base", ".artifacts/browser"]).artifactBase, + ).toMatch(/\.artifacts\/browser$/u); + expect(() => parseBrowserPluginProfilesOptions([])).toThrow( + "usage: --artifact-base ", + ); + }); + + it("requires a clean Docker exit and the real success marker", () => { + expect( + classifyBrowserPluginProfilesOutcome({ + code: 0, + markerSeen: true, + signal: null, + }), + ).toEqual({ status: "pass" }); + expect( + classifyBrowserPluginProfilesOutcome({ + code: 0, + markerSeen: false, + signal: null, + }), + ).toEqual({ + details: "Docker producer omitted BROWSER_PLUGIN_PROFILES_PACKAGED_OK", + status: "fail", + }); + }); + + it("records process failures", () => { + expect( + classifyBrowserPluginProfilesOutcome({ + code: null, + markerSeen: false, + signal: "SIGTERM", + }), + ).toEqual({ + details: "Docker producer terminated by signal SIGTERM", + status: "fail", + }); + expect( + classifyBrowserPluginProfilesOutcome({ + code: 3, + markerSeen: false, + signal: null, + }), + ).toEqual({ + details: "Docker producer exited with code 3", + status: "fail", + }); + }); +}); diff --git a/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts b/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts new file mode 100644 index 000000000000..96fe713dadf5 --- /dev/null +++ b/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts @@ -0,0 +1,132 @@ +import { spawn } from "node:child_process"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + QA_EVIDENCE_FILENAME, + type QaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/api.js"; +import { createQaScriptEvidenceWriter } from "./script-evidence.js"; + +const SCENARIO_ID = "browser-plugin-profiles-packaged"; +const SOURCE_PATH = "test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts"; +const SCRIPT_PATH = "scripts/e2e/browser-plugin-profiles-docker.sh"; +const SUCCESS_MARKER = "BROWSER_PLUGIN_PROFILES_PACKAGED_OK"; +const PRIMARY_COVERAGE_IDS = ["tools.browser-plugin-service", "tools.profiles"] as const; + +type ProducerOptions = { artifactBase: string; repoRoot: string }; +type DockerOutcome = { + code: number | null; + error?: Error; + markerSeen: boolean; + signal: NodeJS.Signals | null; +}; + +function formatError(error: unknown) { + return error instanceof Error ? error.message : String(error); +} + +export function parseBrowserPluginProfilesOptions(args: string[]): ProducerOptions { + if (args.length !== 2 || args[0] !== "--artifact-base" || !args[1]) { + throw new Error("usage: --artifact-base "); + } + return { artifactBase: path.resolve(args[1]), repoRoot: process.cwd() }; +} + +export function classifyBrowserPluginProfilesOutcome(outcome: DockerOutcome): { + details?: string; + status: "pass" | "fail"; +} { + if (outcome.error) { + return { details: formatError(outcome.error), status: "fail" }; + } + if (outcome.signal) { + return { details: `Docker producer terminated by signal ${outcome.signal}`, status: "fail" }; + } + if (outcome.code !== 0) { + return { details: `Docker producer exited with code ${String(outcome.code)}`, status: "fail" }; + } + if (!outcome.markerSeen) { + return { details: `Docker producer omitted ${SUCCESS_MARKER}`, status: "fail" }; + } + return { status: "pass" }; +} + +async function runDockerProducer(appendLog: (chunk: unknown) => void): Promise { + return await new Promise((resolve) => { + const child = spawn("bash", [SCRIPT_PATH], { + cwd: process.cwd(), + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let error: Error | undefined; + let markerSeen = false; + let carry = ""; + const append = (chunk: Buffer) => { + const text = String(chunk); + process.stdout.write(chunk); + appendLog(chunk); + const scan = `${carry}${text}`; + markerSeen ||= scan.includes(SUCCESS_MARKER); + carry = scan.slice(-SUCCESS_MARKER.length); + }; + child.stdout.on("data", append); + child.stderr.on("data", (chunk: Buffer) => { + process.stderr.write(chunk); + appendLog(chunk); + const scan = `${carry}${String(chunk)}`; + markerSeen ||= scan.includes(SUCCESS_MARKER); + carry = scan.slice(-SUCCESS_MARKER.length); + }); + child.on("error", (value) => { + error = value; + }); + child.on("close", (code, signal) => resolve({ code, error, markerSeen, signal })); + }); +} + +async function runProducer(options: ProducerOptions): Promise { + const target = { + codeRefs: [SOURCE_PATH, SCRIPT_PATH, "extensions/browser/src/gateway/browser-request.ts"], + docsRefs: ["docs/tools/browser.md", "docs/help/testing.md"], + id: SCENARIO_ID, + primaryCoverageIds: PRIMARY_COVERAGE_IDS, + sourcePath: SOURCE_PATH, + title: "Packaged browser plugin profiles", + }; + const writer = createQaScriptEvidenceWriter({ + artifactBase: options.artifactBase, + logFileName: "browser-plugin-profiles-packaged.log", + packageSource: { kind: "package-installed-docker" }, + primaryModel: "docker/packaged-browser", + providerMode: "mock-openai", + repoRoot: options.repoRoot, + target, + }); + const startedAt = Date.now(); + const outcome = await runDockerProducer((chunk) => writer.appendLog(chunk)); + const result = classifyBrowserPluginProfilesOutcome(outcome); + return await writer.write({ + details: result.details, + durationMs: Math.max(1, Date.now() - startedAt), + status: result.status, + }); +} + +async function main(args: string[]) { + const evidence = await runProducer(parseBrowserPluginProfilesOptions(args)); + const status = evidence.entries[0]?.result.status; + console.log(`Packaged browser plugin profiles evidence: ${QA_EVIDENCE_FILENAME}`); + console.log(`Packaged browser plugin profiles status: ${status}`); + return status === "pass" ? 0 : 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)) + .then((code) => { + process.exitCode = code; + }) + .catch((error: unknown) => { + console.error(formatError(error)); + process.exitCode = 1; + }); +}