diff --git a/package.json b/package.json index 414a25179486..29bd79ef7888 100644 --- a/package.json +++ b/package.json @@ -1851,6 +1851,7 @@ "test:docker:update-restart-auth": "env OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE=auto-auth OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT=${OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT:-1500s} bash scripts/e2e/upgrade-survivor-docker.sh", "test:docker:upgrade-survivor": "bash scripts/e2e/upgrade-survivor-docker.sh", "test:env-mutations:report": "node --import tsx scripts/test-env-mutation-report.ts", + "test:skip-inventory:report": "node --import tsx scripts/test-skip-inventory.ts", "test:e2e": "pnpm test:e2e:gateway && pnpm test:ui:e2e", "test:e2e:gateway": "node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts", "test:e2e:openshell": "node scripts/run-with-env.mjs OPENCLAW_E2E_OPENSHELL=1 -- node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts extensions/openshell/src/backend.e2e.test.ts", diff --git a/scripts/test-skip-inventory.ts b/scripts/test-skip-inventory.ts new file mode 100644 index 000000000000..43ba7bd24788 --- /dev/null +++ b/scripts/test-skip-inventory.ts @@ -0,0 +1,522 @@ +#!/usr/bin/env node +// Test Skip Inventory reports skipped, conditional, todo, and focused tests. + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; +import { collectFilesSync, isCodeFile, toPosixPath } from "./check-file-utils.js"; + +type SkipInventoryKind = "alias" | "call"; +type SkipInventoryReason = + | "conditional-skip" + | "explicit-skip" + | "focused-only" + | "live-gate" + | "optional-dependency" + | "platform-gate" + | "todo"; + +type SkipInventoryTarget = "describe" | "it" | "test" | "unknown"; + +export type TestSkipInventoryFinding = { + excerpt: string; + file: string; + kind: SkipInventoryKind; + line: number; + method: "only" | "runIf" | "skip" | "skipIf" | "todo"; + reason: SkipInventoryReason; + target: SkipInventoryTarget; +}; + +export type TestSkipInventoryReport = { + findings: TestSkipInventoryFinding[]; + scannedFileCount: number; + schemaVersion: 1; + summary: { + findingCount: number; + reasonCounts: Record; + scannedFileCount: number; + touchedFileCount: number; + }; +}; + +const DEFAULT_SCAN_ROOTS = ["src", "test", "extensions", "packages", "ui", "scripts"]; +const DEFAULT_SKIPPED_DIR_NAMES = new Set([ + ".artifacts", + ".generated", + "coverage", + "dist", + "fixtures", + "node_modules", + "vendor", +]); +const EMPTY_REASON_COUNTS: Record = { + "conditional-skip": 0, + "explicit-skip": 0, + "focused-only": 0, + "live-gate": 0, + "optional-dependency": 0, + "platform-gate": 0, + todo: 0, +}; +const SKIP_METHODS = new Set(["only", "runIf", "skip", "skipIf", "todo"]); +const TEST_TARGETS = new Set(["describe", "it", "test"]); +const TRANSPARENT_CHAIN_METHODS = new Set(["concurrent", "each"]); + +function isTestRelatedFile(relativePath: string): boolean { + return ( + /(?:^|[/.])(?:test|spec)\.[cm]?[jt]sx?$/u.test(relativePath) || + /\.(?:e2e|live)\.test\.[cm]?[jt]sx?$/u.test(relativePath) || + /\.(?:test-helpers|test-utils|test-harness|test-support)\.[cm]?[jt]sx?$/u.test(relativePath) || + /-(?:test-helpers|test-utils|test-harness|test-support)\.[cm]?[jt]sx?$/u.test(relativePath) || + /(?:^|\/)(?:test|tests|test-helpers|test-utils|test-harness|test-support)\//u.test( + relativePath, + ) || + relativePath.startsWith("scripts/e2e/") || + /^scripts\/.*-(?:client|e2e|harness|probe|smoke)\.[cm]?[jt]s$/u.test(relativePath) + ); +} + +function listGitFiles(repoRoot: string): string[] | null { + try { + const stdout = execFileSync("git", ["-C", repoRoot, "ls-files", "--", ...DEFAULT_SCAN_ROOTS], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + return stdout.split(/\r?\n/u).filter(Boolean); + } catch { + return null; + } +} + +function listCandidateFiles(repoRoot: string): string[] { + const gitFiles = listGitFiles(repoRoot); + const relativeFiles = + gitFiles ?? + DEFAULT_SCAN_ROOTS.flatMap((root) => { + const absoluteRoot = path.join(repoRoot, root); + if (!fs.existsSync(absoluteRoot)) { + return []; + } + return collectFilesSync(absoluteRoot, { + includeFile: isCodeFile, + skipDirNames: DEFAULT_SKIPPED_DIR_NAMES, + }).map((filePath) => toPosixPath(path.relative(repoRoot, filePath))); + }); + return relativeFiles + .filter((file) => isCodeFile(file) && isTestRelatedFile(file)) + .toSorted((left, right) => left.localeCompare(right)); +} + +function expressionText(sourceFile: ts.SourceFile, node: ts.Node): string { + return node.getText(sourceFile); +} + +function targetFromExpression(expression: ts.Expression): SkipInventoryTarget { + if (ts.isIdentifier(expression) && TEST_TARGETS.has(expression.text)) { + return expression.text as SkipInventoryTarget; + } + return "unknown"; +} + +function propertyAccessSkipMethod( + expression: ts.Expression, +): { method: TestSkipInventoryFinding["method"]; target: SkipInventoryTarget } | null { + if (!ts.isPropertyAccessExpression(expression) || !SKIP_METHODS.has(expression.name.text)) { + return null; + } + const target = targetFromExpression(expression.expression); + if (target === "unknown") { + return null; + } + return { + method: expression.name.text as TestSkipInventoryFinding["method"], + target, + }; +} + +function skipMethodFromExpression( + expression: ts.Expression, +): { method: TestSkipInventoryFinding["method"]; target: SkipInventoryTarget } | null { + if (ts.isParenthesizedExpression(expression)) { + return skipMethodFromExpression(expression.expression); + } + + const directMethod = propertyAccessSkipMethod(expression); + if (directMethod) { + return directMethod; + } + + if (ts.isConditionalExpression(expression)) { + return ( + skipMethodFromExpression(expression.whenTrue) ?? + skipMethodFromExpression(expression.whenFalse) + ); + } + if (ts.isCallExpression(expression)) { + return skipMethodFromExpression(expression.expression); + } + if ( + ts.isPropertyAccessExpression(expression) && + TRANSPARENT_CHAIN_METHODS.has(expression.name.text) + ) { + return skipMethodFromExpression(expression.expression); + } + + return null; +} + +function methodReason(params: { + file: string; + method: TestSkipInventoryFinding["method"]; + sourceFile: ts.SourceFile; + textNode: ts.Node; +}): SkipInventoryReason { + if (params.method === "todo") { + return "todo"; + } + if (params.method === "only") { + return "focused-only"; + } + + const sourceText = expressionText(params.sourceFile, params.textNode).toLowerCase(); + const text = `${params.file}\n${sourceText}`.toLowerCase(); + if ( + sourceText.includes("process.platform") || + sourceText.includes("win32") || + sourceText.includes("darwin") || + sourceText.includes("linux") + ) { + return "platform-gate"; + } + if ( + text.includes(".live.test.") || + sourceText.includes("openclaw_live") || + sourceText.includes("live_test") || + sourceText.includes("api_key") || + sourceText.includes("token") || + /\blive\b/u.test(sourceText) + ) { + return "live-gate"; + } + if ( + text.includes("chromium") || + text.includes("browser") || + text.includes("docker") || + text.includes("ffmpeg") || + text.includes("gotoolchain") || + text.includes("go toolchain") + ) { + return "optional-dependency"; + } + if (params.method === "skipIf" || params.method === "runIf") { + return "conditional-skip"; + } + if (params.method === "skip" && containsConditionalExpression(params.textNode)) { + return "conditional-skip"; + } + return "explicit-skip"; +} + +function containsConditionalExpression(node: ts.Node): boolean { + if (ts.isConditionalExpression(node)) { + return true; + } + return node.getChildren().some((child) => containsConditionalExpression(child)); +} + +function createFinding(params: { + file: string; + kind: SkipInventoryKind; + lines: string[]; + method: TestSkipInventoryFinding["method"]; + node: ts.Node; + reasonNode: ts.Node; + sourceFile: ts.SourceFile; + target: SkipInventoryTarget; +}): TestSkipInventoryFinding { + const { line } = params.sourceFile.getLineAndCharacterOfPosition(params.node.getStart()); + return { + excerpt: params.lines[line]?.trim() ?? "", + file: params.file, + kind: params.kind, + line: line + 1, + method: params.method, + reason: methodReason({ + file: params.file, + method: params.method, + sourceFile: params.sourceFile, + textNode: params.reasonNode, + }), + target: params.target, + }; +} + +function skipAliasInitializer( + initializer: ts.Expression | undefined, +): { method: TestSkipInventoryFinding["method"]; target: SkipInventoryTarget } | null { + if (!initializer) { + return null; + } + return skipMethodFromExpression(initializer); +} + +function scanFile(params: { file: string; repoRoot: string }): TestSkipInventoryFinding[] { + const absolutePath = path.join(params.repoRoot, params.file); + const source = fs.readFileSync(absolutePath, "utf8"); + const sourceFile = ts.createSourceFile(params.file, source, ts.ScriptTarget.Latest, true); + const lines = source.split(/\r?\n/u); + const findings: TestSkipInventoryFinding[] = []; + + function addFinding(details: { + kind: SkipInventoryKind; + method: TestSkipInventoryFinding["method"]; + node: ts.Node; + reasonNode: ts.Node; + target: SkipInventoryTarget; + }): void { + findings.push( + createFinding({ + file: params.file, + kind: details.kind, + lines, + method: details.method, + node: details.node, + reasonNode: details.reasonNode, + sourceFile, + target: details.target, + }), + ); + } + + function visit(node: ts.Node): void { + if (ts.isVariableDeclaration(node)) { + const alias = skipAliasInitializer(node.initializer); + if (alias) { + addFinding({ + kind: "alias", + method: alias.method, + node, + reasonNode: node.initializer ?? node, + target: alias.target, + }); + } + } else if (ts.isCallExpression(node)) { + const method = isNestedTestModifierCallee(node) + ? null + : skipMethodFromExpression(node.expression); + if (method) { + addFinding({ + kind: "call", + method: method.method, + node, + reasonNode: node, + target: method.target, + }); + } + } + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return findings; +} + +function isNestedTestModifierCallee(node: ts.CallExpression): boolean { + const parent = node.parent; + return ( + (ts.isCallExpression(parent) && parent.expression === node) || + (ts.isPropertyAccessExpression(parent) && parent.expression === node) + ); +} + +export function collectTestSkipInventoryReport( + params: { repoRoot?: string } = {}, +): TestSkipInventoryReport { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const files = listCandidateFiles(repoRoot); + const findings = files.flatMap((file) => scanFile({ file, repoRoot })); + const reasonCounts = { ...EMPTY_REASON_COUNTS }; + for (const finding of findings) { + reasonCounts[finding.reason] += 1; + } + return { + findings, + scannedFileCount: files.length, + schemaVersion: 1, + summary: { + findingCount: findings.length, + reasonCounts, + scannedFileCount: files.length, + touchedFileCount: new Set(findings.map((finding) => finding.file)).size, + }, + }; +} + +function groupFindingsByFile( + findings: TestSkipInventoryFinding[], +): Map { + const grouped = new Map(); + for (const finding of findings) { + const fileFindings = grouped.get(finding.file); + if (fileFindings) { + fileFindings.push(finding); + } else { + grouped.set(finding.file, [finding]); + } + } + return grouped; +} + +function renderReasonCounts(reasonCounts: Record): string { + return Object.entries(reasonCounts) + .filter(([, count]) => count > 0) + .map(([reason, count]) => `${reason}: ${count}`) + .join(", "); +} + +function renderFindingGroups(findings: TestSkipInventoryFinding[], limit: number): string[] { + const lines: string[] = []; + let shown = 0; + for (const [file, fileFindings] of groupFindingsByFile(findings)) { + if (shown >= limit) { + break; + } + lines.push(`- ${file} (${fileFindings.length})`); + for (const finding of fileFindings) { + if (shown >= limit) { + break; + } + lines.push( + ` L${finding.line} ${finding.target}.${finding.method} ${finding.reason}: ${finding.excerpt}`, + ); + shown += 1; + } + } + if (findings.length > shown) { + lines.push( + `... ${findings.length - shown} more finding(s) not shown; pass --limit 0 to show all.`, + ); + } + return lines; +} + +export function renderTestSkipInventoryReport( + report: TestSkipInventoryReport, + options: { limit?: number } = {}, +): string { + const limit = options.limit === 0 ? Number.POSITIVE_INFINITY : (options.limit ?? 120); + const reasonCounts = renderReasonCounts(report.summary.reasonCounts) || "none"; + const lines = [ + "OpenClaw test skip inventory", + `Scanned files: ${report.summary.scannedFileCount}`, + `Findings: ${report.summary.findingCount} in ${report.summary.touchedFileCount} file(s)`, + `Reasons: ${reasonCounts}`, + "", + ]; + + if (report.findings.length === 0) { + lines.push("Findings: none"); + } else { + lines.push("Findings:"); + lines.push(...renderFindingGroups(report.findings, limit)); + } + + return `${lines.join("\n")}\n`; +} + +function readNonNegativeIntArg(raw: string | undefined): number { + if (!raw || raw.startsWith("--") || !/^\d+$/u.test(raw)) { + throw new Error("--limit expects a non-negative integer"); + } + const value = Number(raw); + if (!Number.isSafeInteger(value)) { + throw new Error("--limit expects a non-negative integer"); + } + return value; +} + +function parseArgs(argv: string[]): { + help: boolean; + json: boolean; + limit: number; + repoRoot: string; +} { + let help = false; + let json = false; + let limit = 120; + let repoRoot = process.cwd(); + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--") { + continue; + } + if (arg === "--help" || arg === "-h") { + help = true; + continue; + } + if (arg === "--json") { + json = true; + continue; + } + if (arg === "--limit") { + limit = readNonNegativeIntArg(argv[index + 1]); + index += 1; + continue; + } + if (arg === "--repo-root") { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error("--repo-root expects a path"); + } + repoRoot = value; + index += 1; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + return { help, json, limit, repoRoot }; +} + +function printHelp(): void { + process.stdout.write(`OpenClaw test skip inventory + +Usage: + pnpm test:skip-inventory:report [options] + +Options: + --json Print the full JSON report + --limit Maximum text findings to print; use 0 for all (default: 120) + --repo-root Repository root to scan (default: current working directory) + --help, -h Show this help message +`); +} + +export function main(argv = process.argv.slice(2)): number { + const args = parseArgs(argv); + if (args.help) { + printHelp(); + return 0; + } + + const report = collectTestSkipInventoryReport({ repoRoot: args.repoRoot }); + if (args.json) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + process.stdout.write(renderTestSkipInventoryReport(report, { limit: args.limit })); + } + return 0; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + process.exitCode = main(); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + } +} diff --git a/test/scripts/test-skip-inventory.test.ts b/test/scripts/test-skip-inventory.test.ts new file mode 100644 index 000000000000..0d58da7d208c --- /dev/null +++ b/test/scripts/test-skip-inventory.test.ts @@ -0,0 +1,272 @@ +// Test Skip Inventory tests cover skipped, conditional, todo, and focused test reporting. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + collectTestSkipInventoryReport, + renderTestSkipInventoryReport, + type TestSkipInventoryReport, +} from "../../scripts/test-skip-inventory.js"; +import { createScriptTestHarness } from "./test-helpers.js"; + +const { createTempDir } = createScriptTestHarness(); + +function writeRepoFile(repoRoot: string, relativePath: string, value: string): void { + const filePath = path.join(repoRoot, relativePath); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, value, "utf8"); +} + +function makeSkipInventoryFixture(): string { + const repoRoot = createTempDir("openclaw-test-skip-inventory-"); + writeRepoFile( + repoRoot, + "src/runtime.ts", + ` +describe.skip("ignored because this is not a test file", () => {}); +`, + ); + writeRepoFile( + repoRoot, + "src/example.test.ts", + ` +describe.skip("temporarily disabled suite", () => {}); +it.skipIf(process.platform === "win32")("uses POSIX shell", () => {}); +test.todo("covers migration edge"); +it.only("focused during local debugging", () => {}); +const posixIt = process.platform === "win32" ? it.skip : it; +(process.platform === "win32" ? it.skip : it)("uses terminal fixtures", () => {}); +it.runIf(process.platform !== "win32")("runs only on POSIX", () => {}); +it.runIf(process.platform !== "win32").each(["SIGTERM"] as const)("handles %s", () => {}); +const runIfPowerShell = powershell ? it : it.skip; +const fixture = 'describe.skip("not code", () => {})'; +`, + ); + writeRepoFile( + repoRoot, + "test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts", + ` +reader.skip(wire); +`, + ); + writeRepoFile( + repoRoot, + "test/scripts/test-live.test.ts", + ` +const posixIt = process.platform === "win32" ? it.skip : it; +`, + ); + writeRepoFile( + repoRoot, + "extensions/provider/live.test.ts", + ` +const LIVE = process.env.OPENCLAW_LIVE_TEST === "1"; +const describeLive = LIVE ? describe : describe.skip; +describeLive("provider live", () => {}); +`, + ); + writeRepoFile( + repoRoot, + "ui/src/ui/e2e/chat-flow.e2e.test.ts", + ` +const describeControlUiE2e = chromiumAvailable || !allowMissingChromium ? describe : describe.skip; +describeControlUiE2e("control UI", () => {}); +`, + ); + return repoRoot; +} + +describe("collectTestSkipInventoryReport", () => { + it("reports skipped, conditional, todo, and focused test inventory", () => { + const report = collectTestSkipInventoryReport({ repoRoot: makeSkipInventoryFixture() }); + + expect( + report.findings.map((finding) => ({ + file: finding.file, + kind: finding.kind, + method: finding.method, + reason: finding.reason, + target: finding.target, + })), + ).toEqual([ + { + file: "extensions/provider/live.test.ts", + kind: "alias", + method: "skip", + reason: "live-gate", + target: "describe", + }, + { + file: "src/example.test.ts", + kind: "call", + method: "skip", + reason: "explicit-skip", + target: "describe", + }, + { + file: "src/example.test.ts", + kind: "call", + method: "skipIf", + reason: "platform-gate", + target: "it", + }, + { + file: "src/example.test.ts", + kind: "call", + method: "todo", + reason: "todo", + target: "test", + }, + { + file: "src/example.test.ts", + kind: "call", + method: "only", + reason: "focused-only", + target: "it", + }, + { + file: "src/example.test.ts", + kind: "alias", + method: "skip", + reason: "platform-gate", + target: "it", + }, + { + file: "src/example.test.ts", + kind: "call", + method: "skip", + reason: "platform-gate", + target: "it", + }, + { + file: "src/example.test.ts", + kind: "call", + method: "runIf", + reason: "platform-gate", + target: "it", + }, + { + file: "src/example.test.ts", + kind: "call", + method: "runIf", + reason: "platform-gate", + target: "it", + }, + { + file: "src/example.test.ts", + kind: "alias", + method: "skip", + reason: "conditional-skip", + target: "it", + }, + { + file: "test/scripts/test-live.test.ts", + kind: "alias", + method: "skip", + reason: "platform-gate", + target: "it", + }, + { + file: "ui/src/ui/e2e/chat-flow.e2e.test.ts", + kind: "alias", + method: "skip", + reason: "optional-dependency", + target: "describe", + }, + ]); + expect(report.summary).toMatchObject({ + findingCount: 12, + reasonCounts: { + "conditional-skip": 1, + "explicit-skip": 1, + "focused-only": 1, + "live-gate": 1, + "optional-dependency": 1, + "platform-gate": 6, + todo: 1, + }, + touchedFileCount: 4, + }); + }); + + it("renders a compact non-blocking text report", () => { + const report = collectTestSkipInventoryReport({ repoRoot: makeSkipInventoryFixture() }); + + const rendered = renderTestSkipInventoryReport(report, { limit: 4 }); + + expect(rendered).toContain("OpenClaw test skip inventory"); + expect(rendered).toContain("Findings: 12 in 4 file(s)"); + expect(rendered).toContain("platform-gate: 6"); + expect(rendered).toContain("- src/example.test.ts (9)"); + expect(rendered).toContain("L2 describe.skip explicit-skip"); + expect(rendered).toContain("... 8 more finding(s) not shown"); + }); + + it("prints JSON from the CLI and exits successfully", () => { + const repoRoot = makeSkipInventoryFixture(); + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + path.join(process.cwd(), "scripts/test-skip-inventory.ts"), + "--", + "--repo-root", + repoRoot, + "--json", + ], + { + encoding: "utf8", + }, + ); + + expect(result.status).toBe(0); + const report = JSON.parse(result.stdout) as TestSkipInventoryReport; + expect(report.summary.findingCount).toBe(12); + expect(report.summary.reasonCounts["focused-only"]).toBe(1); + }); + + it("prints CLI help without scanning the repository", () => { + const result = spawnSync( + process.execPath, + ["--import", "tsx", path.join(process.cwd(), "scripts/test-skip-inventory.ts"), "--help"], + { + encoding: "utf8", + }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("Usage:"); + expect(result.stdout).toContain("pnpm test:skip-inventory:report"); + expect(result.stdout).not.toContain("Scanned files:"); + expect(result.stderr).toBe(""); + }); + + it("rejects missing CLI repo roots and loose limits before scanning", () => { + for (const args of [ + ["--repo-root", "--json"], + ["--limit", "1e3", "--repo-root", createTempDir("openclaw-skip-limit-")], + ]) { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + path.join(process.cwd(), "scripts/test-skip-inventory.ts"), + "--", + ...args, + ], + { + encoding: "utf8", + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toMatch( + /--repo-root expects a path|--limit expects a non-negative integer/u, + ); + expect(result.stdout).not.toContain("Scanned files:"); + } + }); +});