From 3564c1c79545b325312361c2b6ad946af96751e8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 19 Jul 2026 20:04:14 -0700 Subject: [PATCH] fix(test): bound Matrix test process memory (#111607) * test: bound Matrix test process memory * test: align Matrix runner mock contract --- .../memory-core/src/memory/index.test.ts | 20 +-- scripts/lib/extension-test-plan.d.mts | 8 + scripts/lib/extension-test-plan.mjs | 50 ++++++ scripts/test-extension-batch.mjs | 48 ++++-- scripts/test-extension.mjs | 35 +++- scripts/test-projects.test-support.mjs | 60 ++++++- test/scripts/test-extension.test.ts | 154 +++++++++++++++++- test/scripts/test-projects.test.ts | 126 +++++++++++--- 8 files changed, 439 insertions(+), 62 deletions(-) diff --git a/extensions/memory-core/src/memory/index.test.ts b/extensions/memory-core/src/memory/index.test.ts index 9ac78e240cc9..9aa61e1f5661 100644 --- a/extensions/memory-core/src/memory/index.test.ts +++ b/extensions/memory-core/src/memory/index.test.ts @@ -382,7 +382,7 @@ describe("memory index", () => { enabled: boolean; vectorWeight?: number; textWeight?: number; - temporalDecay?: { enabled: boolean; halfLifeDays: number }; + temporalDecay?: { enabled: boolean }; }; }): TestCfg { return { @@ -2581,7 +2581,7 @@ describe("memory index", () => { await fs.writeFile(freshFooPath, "Unrelated fresh candidate."); await fs.writeFile(staleBarPath, "bar md bar md bar md strongest stale body"); await fs.writeFile(path.join(freshDir, "bar.md"), "bar md fresh body"); - const staleMtime = new Date(Date.now() - 30 * 24 * 60 * 60_000); + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); await Promise.all([ fs.utimes(staleFooPath, staleMtime, staleMtime), fs.utimes(staleBarPath, staleMtime, staleMtime), @@ -2591,7 +2591,7 @@ describe("memory index", () => { minScore: 0, hybrid: { enabled: true, - temporalDecay: { enabled: true, halfLifeDays: 1 }, + temporalDecay: { enabled: true }, }, }); const result = await getMemorySearchManager({ cfg, agentId: "main" }); @@ -2613,7 +2613,7 @@ describe("memory index", () => { it("applies temporal decay after the exact-path candidate cap", async () => { forceNoProvider = true; - const staleMtime = new Date(Date.now() - 30 * 24 * 60 * 60_000); + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); const extraPaths: string[] = []; for (let index = 0; index < 5; index += 1) { const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; @@ -2632,7 +2632,7 @@ describe("memory index", () => { minScore: 0, hybrid: { enabled: true, - temporalDecay: { enabled: true, halfLifeDays: 1 }, + temporalDecay: { enabled: true }, }, }); const result = await getMemorySearchManager({ cfg, agentId: "main" }); @@ -2651,7 +2651,7 @@ describe("memory index", () => { }); it("applies hybrid temporal decay beyond the content candidate cap", async () => { - const staleMtime = new Date(Date.now() - 30 * 24 * 60 * 60_000); + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); const extraPaths: string[] = []; for (let index = 0; index < 5; index += 1) { const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; @@ -2670,7 +2670,7 @@ describe("memory index", () => { minScore: 0, hybrid: { enabled: true, - temporalDecay: { enabled: true, halfLifeDays: 1 }, + temporalDecay: { enabled: true }, }, }); const manager = await getPersistentManager(cfg); @@ -2683,7 +2683,7 @@ describe("memory index", () => { }); it("keeps temporal decay when degraded hybrid search becomes keyword-only", async () => { - const staleMtime = new Date(Date.now() - 30 * 24 * 60 * 60_000); + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); const extraPaths: string[] = []; for (let index = 0; index < 5; index += 1) { const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; @@ -2702,7 +2702,7 @@ describe("memory index", () => { minScore: 0, hybrid: { enabled: true, - temporalDecay: { enabled: true, halfLifeDays: 1 }, + temporalDecay: { enabled: true }, }, }); const manager = await getPersistentManager(cfg); @@ -2833,7 +2833,7 @@ describe("memory index", () => { minScore: 0.35, hybrid: { enabled: true, - temporalDecay: { enabled: true, halfLifeDays: 1 }, + temporalDecay: { enabled: true }, }, }); const result = await getMemorySearchManager({ cfg, agentId: "main" }); diff --git a/scripts/lib/extension-test-plan.d.mts b/scripts/lib/extension-test-plan.d.mts index 87ddd5f8ed1c..6927e9ea4ab0 100644 --- a/scripts/lib/extension-test-plan.d.mts +++ b/scripts/lib/extension-test-plan.d.mts @@ -21,7 +21,15 @@ export type ExtensionTestShard = ExtensionBatchPlan & { }; export const DEFAULT_EXTENSION_TEST_SHARD_COUNT: number; +export function createExtensionTestProcessTargetChunks( + config: string, + roots: string[], + vitestArgs?: string[], +): string[][]; +export function listExtensionTestFilesForRoots(roots: string[]): string[]; export function listTrackedTestFilesForRoots(roots: string[]): string[]; +export function shouldSplitExtensionTestProcesses(config: string, vitestArgs?: string[]): boolean; +export function splitExtensionTestProcessTargets(config: string, targets: string[]): string[][]; export function resolveExtensionTestConfig(root: string): string; export function resolveExtensionTestPlan(params?: { cwd?: string; diff --git a/scripts/lib/extension-test-plan.mjs b/scripts/lib/extension-test-plan.mjs index 8a07f16a5feb..f736a4e06b09 100644 --- a/scripts/lib/extension-test-plan.mjs +++ b/scripts/lib/extension-test-plan.mjs @@ -69,6 +69,11 @@ const EXTENSION_TEST_COST_MULTIPLIERS = { // overstates its real wall-clock cost during CI shard planning. "test/vitest/vitest.extensions.config.ts": 1.1, }; +const EXTENSION_TEST_PROCESS_FILE_LIMITS = new Map([ + // The non-isolated Matrix suite intentionally shares module state within a process. + // Bound its lifetime so Vite's transformed module graph cannot grow across the whole suite. + ["test/vitest/vitest.extension-matrix.config.ts", 40], +]); const EXTENSION_TEST_CONFIG_ROUTES = [ [isActiveMemoryExtensionRoot, "test/vitest/vitest.extension-active-memory.config.ts"], [isAcpxExtensionRoot, "test/vitest/vitest.extension-acpx.config.ts"], @@ -200,6 +205,51 @@ export function listTrackedTestFilesForRoots(roots) { return [...new Set(files)].toSorted((left, right) => left.localeCompare(right)); } +/** List working-tree test files for extension roots, including new untracked tests. */ +export function listExtensionTestFilesForRoots(roots) { + const files = roots.flatMap((root) => listFilesystemTestFiles(path.join(repoRoot, root))); + return [...new Set(files)].toSorted((left, right) => left.localeCompare(right)); +} + +/** Split an extension config's test files across bounded process lifetimes when required. */ +export function splitExtensionTestProcessTargets(config, targets) { + const maxFilesPerProcess = EXTENSION_TEST_PROCESS_FILE_LIMITS.get(config); + const orderedTargets = [...new Set(targets)].toSorted((left, right) => left.localeCompare(right)); + if (!maxFilesPerProcess || orderedTargets.length <= maxFilesPerProcess) { + return [orderedTargets]; + } + + const chunkCount = Math.ceil(orderedTargets.length / maxFilesPerProcess); + const baseSize = Math.floor(orderedTargets.length / chunkCount); + const remainder = orderedTargets.length % chunkCount; + const chunks = []; + let offset = 0; + for (let index = 0; index < chunkCount; index += 1) { + const chunkSize = baseSize + (index < remainder ? 1 : 0); + chunks.push(orderedTargets.slice(offset, offset + chunkSize)); + offset += chunkSize; + } + return chunks; +} + +/** Whether a Vitest invocation can safely be split into independent one-shot processes. */ +export function shouldSplitExtensionTestProcesses(config, vitestArgs = []) { + // Passthrough options can carry suite-wide semantics such as bail thresholds, + // filtering, watch state, or shared artifacts. Only plain one-shot runs are splittable. + return EXTENSION_TEST_PROCESS_FILE_LIMITS.has(config) && vitestArgs.length === 0; +} + +/** Resolve process targets for an extension config, expanding roots only when it is bounded. */ +export function createExtensionTestProcessTargetChunks(config, roots, vitestArgs = []) { + if (!shouldSplitExtensionTestProcesses(config, vitestArgs)) { + return [roots]; + } + // Explicit file targets replace Vitest's root discovery, so inventory the working tree. + // Otherwise a newly authored untracked test would silently disappear from a broad run. + const testFiles = listExtensionTestFilesForRoots(roots); + return testFiles.length > 0 ? splitExtensionTestProcessTargets(config, testFiles) : [roots]; +} + function countTestFiles(rootPath) { const trackedFiles = listTrackedTestFiles(rootPath); if (trackedFiles) { diff --git a/scripts/test-extension-batch.mjs b/scripts/test-extension-batch.mjs index 2095bfa87d5b..9eace88d54bd 100644 --- a/scripts/test-extension-batch.mjs +++ b/scripts/test-extension-batch.mjs @@ -4,8 +4,11 @@ import path from "node:path"; import pMap from "p-map"; import { - listTrackedTestFilesForRoots, + createExtensionTestProcessTargetChunks, + listExtensionTestFilesForRoots, resolveExtensionBatchPlan, + shouldSplitExtensionTestProcesses, + splitExtensionTestProcessTargets, } from "./lib/extension-test-plan.mjs"; import { normalizeRelativePath, @@ -149,7 +152,7 @@ function resolveGroupTargets(group, exactExcludePaths) { return group.roots; } - const testFiles = listTrackedTestFilesForRoots(group.roots); + const testFiles = listExtensionTestFilesForRoots(group.roots); if (!testFiles) { return group.roots; } @@ -164,20 +167,33 @@ async function runPlanGroup(group, params) { return params.allowEmptyAfterExclude ? 0 : 1; } - console.log( - `[test-extension-batch] ${group.config}: ${group.extensionIds.join(", ")} (${targets.length} targets)`, - ); - return await params.runGroup({ - args: relativizeExtensionVitestArgs(params.vitestArgs), - config: group.config, - env: createGroupEnv({ - baseEnv: params.env, - group, - groupIndex: params.groupIndex, - useDedicatedCache: params.useDedicatedCache, - }), - targets: targets.map((target) => relativizeExtensionVitestPath(target)), - }); + const targetChunks = + params.exactExcludePaths.size > 0 + ? shouldSplitExtensionTestProcesses(group.config, params.vitestArgs) + ? splitExtensionTestProcessTargets(group.config, targets) + : [targets] + : createExtensionTestProcessTargetChunks(group.config, group.roots, params.vitestArgs); + let finalExitCode = 0; + for (const [index, chunk] of targetChunks.entries()) { + console.log( + `[test-extension-batch] ${group.config}: ${group.extensionIds.join(", ")} (${chunk.length} targets${targetChunks.length > 1 ? `, chunk ${index + 1}/${targetChunks.length}` : ""})`, + ); + const exitCode = await params.runGroup({ + args: relativizeExtensionVitestArgs(params.vitestArgs), + config: group.config, + env: createGroupEnv({ + baseEnv: params.env, + group, + groupIndex: params.groupIndex, + useDedicatedCache: params.useDedicatedCache, + }), + targets: chunk.map((target) => relativizeExtensionVitestPath(target)), + }); + if (exitCode !== 0 && finalExitCode === 0) { + finalExitCode = exitCode; + } + } + return finalExitCode; } /** diff --git a/scripts/test-extension.mjs b/scripts/test-extension.mjs index a9497bcc8f9f..28e755aa7b03 100644 --- a/scripts/test-extension.mjs +++ b/scripts/test-extension.mjs @@ -2,7 +2,10 @@ // Runs the Vitest plan for one bundled plugin by id or path. import { formatErrorMessage } from "./lib/error-format.mjs"; -import { resolveExtensionTestPlan } from "./lib/extension-test-plan.mjs"; +import { + createExtensionTestProcessTargetChunks, + resolveExtensionTestPlan, +} from "./lib/extension-test-plan.mjs"; import { relativizeExtensionVitestArgs, relativizeExtensionVitestPath, @@ -57,13 +60,29 @@ async function run() { } console.log(`[test-extension] Running ${plan.testFileCount} test files for ${plan.extensionId}`); - const exitCode = await runVitestBatch({ - args: relativizeExtensionVitestArgs(passthroughArgs), - config: plan.config, - env: process.env, - targets: plan.roots.map((target) => relativizeExtensionVitestPath(target)), - }); - process.exit(exitCode); + const targetChunks = createExtensionTestProcessTargetChunks( + plan.config, + plan.roots, + passthroughArgs, + ); + let finalExitCode = 0; + for (const [index, targets] of targetChunks.entries()) { + if (targetChunks.length > 1) { + console.log(`[test-extension] Process chunk ${index + 1}/${targetChunks.length}`); + } + const exitCode = await runVitestBatch({ + args: relativizeExtensionVitestArgs(passthroughArgs), + config: plan.config, + env: process.env, + targets: targets.map((target) => relativizeExtensionVitestPath(target)), + }); + if (exitCode !== 0 && finalExitCode === 0) { + finalExitCode = exitCode; + } + } + if (finalExitCode !== 0) { + process.exit(finalExitCode); + } } if (isDirectScriptRun(import.meta.url)) { diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 12bc20829675..5e55a04bca8c 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -24,7 +24,10 @@ import { isCodexExtensionRoot } from "../test/vitest/vitest.extension-codex-path import { isDiffsExtensionRoot } from "../test/vitest/vitest.extension-diffs-paths.mjs"; import { isFeishuExtensionRoot } from "../test/vitest/vitest.extension-feishu-paths.mjs"; import { isIrcExtensionRoot } from "../test/vitest/vitest.extension-irc-paths.mjs"; -import { isMatrixExtensionRoot } from "../test/vitest/vitest.extension-matrix-paths.mjs"; +import { + isMatrixExtensionRoot, + matrixExtensionTestRoots, +} from "../test/vitest/vitest.extension-matrix-paths.mjs"; import { isMattermostExtensionRoot } from "../test/vitest/vitest.extension-mattermost-paths.mjs"; import { isMediaExtensionRoot } from "../test/vitest/vitest.extension-media-paths.mjs"; import { isMemoryExtensionRoot } from "../test/vitest/vitest.extension-memory-paths.mjs"; @@ -67,6 +70,7 @@ import { listChangedPathsFromGit as listChangedPathsFromGitSource, } from "./changed-lanes.mjs"; import { getChangedPathFacts } from "./lib/changed-path-facts.mjs"; +import { createExtensionTestProcessTargetChunks } from "./lib/extension-test-plan.mjs"; import { isCiLikeEnv, resolveLocalFullSuiteProfile } from "./lib/vitest-local-scheduling.mjs"; import { DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS, @@ -2720,6 +2724,22 @@ function createBroadToolingScriptPlans({ config, forwardedArgs, includePatterns, : null; } +function createBoundedExtensionPlans({ config, forwardedArgs, roots, watchMode }) { + if (watchMode) { + return null; + } + const chunks = createExtensionTestProcessTargetChunks(config, roots, forwardedArgs); + if (chunks.length <= 1) { + return null; + } + return chunks.map((includePatterns) => ({ + config, + forwardedArgs, + includePatterns, + watchMode, + })); +} + function expandBroadToolingScriptTargets(targetArgs, cwd, watchMode) { if (watchMode) { return targetArgs; @@ -4419,12 +4439,19 @@ export function buildVitestRunPlans( ? [FULL_EXTENSIONS_VITEST_CONFIG] : listFullExtensionVitestProjectConfigs(); for (const config of configs) { - plans.push({ + const plan = { config, forwardedArgs: nonTargetArgs, includePatterns: null, watchMode, + }; + const boundedPlans = createBoundedExtensionPlans({ + config, + forwardedArgs: nonTargetArgs, + roots: matrixExtensionTestRoots, + watchMode, }); + plans.push(...(boundedPlans ?? [plan])); } continue; } @@ -4459,6 +4486,29 @@ export function buildVitestRunPlans( plans.push(...broadToolingScriptPlans); continue; } + const boundedExtensionRoots = grouped.flatMap((targetArg) => { + const root = toRepoRelativeTarget(targetArg, cwd); + return isMatrixExtensionRoot(root) && isExistingDirectoryTarget(targetArg, cwd) ? [root] : []; + }); + const boundedRootsCoverGroupedTargets = grouped.every((targetArg) => { + const relativeTarget = toRepoRelativeTarget(targetArg, cwd); + return boundedExtensionRoots.some( + (root) => relativeTarget === root || relativeTarget.startsWith(`${root}/`), + ); + }); + const boundedExtensionPlans = + boundedExtensionRoots.length > 0 && boundedRootsCoverGroupedTargets + ? createBoundedExtensionPlans({ + config, + forwardedArgs: forwardedPlanArgs, + roots: boundedExtensionRoots, + watchMode, + }) + : null; + if (boundedExtensionPlans) { + plans.push(...boundedExtensionPlans); + continue; + } plans.push({ config, forwardedArgs: forwardedPlanArgs, @@ -4525,6 +4575,12 @@ export function buildFullSuiteVitestRunPlans(args, cwd = process.cwd()) { resolveGatewayServerFullSuiteTargets(cwd), GATEWAY_SERVER_FULL_SUITE_TARGET_CHUNK_COUNT, ); + } else if (config === EXTENSION_MATRIX_VITEST_CONFIG) { + chunks = createExtensionTestProcessTargetChunks( + config, + matrixExtensionTestRoots, + forwardedArgs, + ); } if (chunks.length > 0) { return chunks.map((targets) => ({ diff --git a/test/scripts/test-extension.test.ts b/test/scripts/test-extension.test.ts index f72251efc14a..05777c05ce86 100644 --- a/test/scripts/test-extension.test.ts +++ b/test/scripts/test-extension.test.ts @@ -14,13 +14,16 @@ import { } from "../../scripts/lib/changed-extensions.mjs"; import { DEFAULT_EXTENSION_TEST_SHARD_COUNT, + createExtensionTestProcessTargetChunks, createExtensionTestShards, + listExtensionTestFilesForRoots, listTrackedTestFilesForRoots, resolveExtensionBatchPlan, resolveExtensionTestConfig, resolveExtensionTestPlan, } from "../../scripts/lib/extension-test-plan.mjs"; import { relativizeExtensionVitestArgs } from "../../scripts/lib/extension-vitest-paths.mjs"; +import type { VitestBatchRunParams } from "../../scripts/lib/vitest-batch-runner.mjs"; import { buildVitestBatchPnpmArgs } from "../../scripts/lib/vitest-batch-runner.mjs"; import { parseExtensionIds, @@ -34,12 +37,7 @@ import { extensionCatchAllExcludedTestRoots } from "../vitest/vitest.extensions. const scriptPath = path.join(process.cwd(), "scripts", "test-extension.mjs"); const posixIt = process.platform === "win32" ? it.skip : it; -type RunGroupParams = { - args: string[]; - config: string; - env: Record; - targets: string[]; -}; +type RunGroupParams = VitestBatchRunParams; function createConcurrentExtensionBatchPlan() { const groups = [ @@ -179,6 +177,63 @@ describe("scripts/test-extension.mjs", () => { expect(plan.hasTests).toBe(true); }); + it("bounds Matrix test files across balanced process lifetimes", () => { + const config = "test/vitest/vitest.extension-matrix.config.ts"; + const roots = [bundledPluginRoot("matrix")]; + const expectedFiles = listExtensionTestFilesForRoots(roots); + const chunks = createExtensionTestProcessTargetChunks(config, roots); + + expect(chunks).toHaveLength(3); + expect(chunks.every((chunk) => chunk.length <= 40)).toBe(true); + expect(Math.max(...chunks.map((chunk) => chunk.length))).toBeLessThanOrEqual( + Math.min(...chunks.map((chunk) => chunk.length)) + 1, + ); + expect(chunks.flat()).toEqual(expectedFiles); + expect(new Set(chunks.flat()).size).toBe(expectedFiles.length); + }); + + it("includes newly authored Matrix tests in bounded process targets", () => { + const root = mkdtempSync(path.join(process.cwd(), "extensions", ".extension-test-plan-")); + const relativeRoot = path.relative(process.cwd(), root); + const testFile = path.join(root, "newly-authored.test.ts"); + writeFileSync(testFile, "export {};\n"); + try { + const chunks = createExtensionTestProcessTargetChunks( + "test/vitest/vitest.extension-matrix.config.ts", + [relativeRoot], + ); + + expect(chunks.flat()).toEqual([ + path.relative(process.cwd(), testFile).split(path.sep).join("/"), + ]); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it.each([ + ["watch", ["--watch"]], + ["short watch", ["-w"]], + ["coverage", ["--coverage"]], + ["reporter", ["--reporter=json"]], + ["output file", ["--outputFile=results.json"]], + ["shard", ["--shard=1/2"]], + ["bail", ["--bail=2"]], + ["changed", ["--changed=origin/main"]], + ["exclude", ["--exclude=extensions/matrix/src/**"]], + ["retry", ["--retry=1"]], + ])("keeps Matrix %s runs in one process", (_name, vitestArgs) => { + const root = bundledPluginRoot("matrix"); + + expect( + createExtensionTestProcessTargetChunks( + "test/vitest/vitest.extension-matrix.config.ts", + [root], + vitestArgs, + ), + ).toEqual([[root]]); + }); + it("resolves telegram onto the telegram vitest config", () => { const plan = resolveExtensionTestPlan({ targetArg: "telegram", cwd: process.cwd() }); @@ -787,6 +842,31 @@ describe("scripts/test-extension.mjs", () => { } }); + posixIt("runs every single-extension Matrix chunk after an earlier chunk fails", () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-test-extension-chunks-")); + const fakePnpmPath = path.join(root, "pnpm"); + const countPath = path.join(root, "count"); + + writeFakePnpm(fakePnpmPath); + try { + const result = spawnSync(process.execPath, [scriptPath, "matrix"], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + OPENCLAW_FAKE_PNPM_CALL_COUNT_PATH: countPath, + OPENCLAW_FAKE_PNPM_EXIT_CODES: "1,0,0", + npm_execpath: fakePnpmPath, + }, + }); + + expect(result.status).toBe(1); + expect(readFileSync(countPath, "utf8")).toBe("3"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + posixIt( "preserves wrapper termination when the pnpm child exits cleanly after SIGTERM", async () => { @@ -874,6 +954,61 @@ describe("scripts/test-extension.mjs", () => { expect(runParams.targets).toContain("codex/src/app-server/client.test.ts"); }); + it("runs Matrix extension batches in bounded sequential processes", async () => { + const runGroup = vi.fn<(params: RunGroupParams) => Promise>().mockResolvedValue(0); + const expectedFiles = listExtensionTestFilesForRoots([bundledPluginRoot("matrix")]).map( + (file) => file.replace(/^extensions\//u, ""), + ); + + const result = await runExtensionBatchPlan( + resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds: ["matrix"] }), + { runGroup }, + ); + + expect(result).toBe(0); + expect(runGroup).toHaveBeenCalledTimes(3); + const calls = runGroup.mock.calls.map(([params]) => params as RunGroupParams); + expect(calls.every((call) => call.targets.length <= 40)).toBe(true); + expect(calls.flatMap((call) => call.targets)).toEqual(expectedFiles); + }); + + it("runs every Matrix process chunk after an earlier chunk fails", async () => { + const runGroup = vi + .fn<(params: RunGroupParams) => Promise>() + .mockResolvedValueOnce(1) + .mockResolvedValue(0); + + const result = await runExtensionBatchPlan( + resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds: ["matrix"] }), + { runGroup }, + ); + + expect(result).toBe(1); + expect(runGroup).toHaveBeenCalledTimes(3); + }); + + it.each([ + ["--watch"], + ["--coverage"], + ["--reporter=json"], + ["--outputFile=results.json"], + ["--bail=2"], + ["--changed=origin/main"], + ["--exclude=extensions/matrix/src/**"], + ["--retry=1"], + ])("keeps Matrix extension batch mode %s in one process", async (vitestArg) => { + const runGroup = vi.fn<() => Promise>().mockResolvedValue(0); + + const result = await runExtensionBatchPlan( + resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds: ["matrix"] }), + { runGroup, vitestArgs: [vitestArg] }, + ); + + expect(result).toBe(0); + expect(runGroup).toHaveBeenCalledOnce(); + expect(requireFirstMockArg(runGroup).targets).toEqual(["matrix"]); + }); + it("fails extension batch groups when exact excludes remove every test", async () => { const runGroup = vi.fn<() => Promise>().mockResolvedValue(0); const firecrawlTestFiles = listExtensionTestFiles("firecrawl"); @@ -964,6 +1099,13 @@ function writeFakePnpm(filePath: string): void { "#!/usr/bin/env node", 'const { spawn } = require("node:child_process");', 'const fs = require("node:fs");', + "if (process.env.OPENCLAW_FAKE_PNPM_EXIT_CODES) {", + " const countPath = process.env.OPENCLAW_FAKE_PNPM_CALL_COUNT_PATH;", + " const count = fs.existsSync(countPath) ? Number(fs.readFileSync(countPath, 'utf8')) : 0;", + " const exitCodes = process.env.OPENCLAW_FAKE_PNPM_EXIT_CODES.split(',').map(Number);", + " fs.writeFileSync(countPath, String(count + 1));", + " process.exit(exitCodes[count] || 0);", + "}", "if (process.env.OPENCLAW_FAKE_PNPM_ARGS_PATH) {", " fs.writeFileSync(process.env.OPENCLAW_FAKE_PNPM_ARGS_PATH, JSON.stringify(process.argv.slice(2)));", " process.exit(0);", diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index b35ea30d10a3..389ad34dfe1b 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import fg from "fast-glob"; import { beforeAll, describe, expect, it, vi } from "vitest"; +import { listExtensionTestFilesForRoots } from "../../scripts/lib/extension-test-plan.mjs"; import { CHANNEL_CONTRACT_CONFIG_PATTERNS, DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS, @@ -144,6 +145,23 @@ function listNormalFullSuiteTestFiles(): string[] { .toSorted((left, right) => left.localeCompare(right)); } +function listExpectedFullExtensionRunPlans() { + const matrixConfig = "test/vitest/vitest.extension-matrix.config.ts"; + const matrixPlans = buildVitestRunPlans(["extensions/matrix"], process.cwd()); + return listFullExtensionVitestProjectConfigs().flatMap((config) => + config === matrixConfig + ? matrixPlans + : [ + { + config, + forwardedArgs: [], + includePatterns: null, + watchMode: false, + }, + ], + ); +} + function hasGitGatewayFileListing(cwd: string): boolean { const result = spawnSync("git", ["ls-files", "--", "src/gateway"], { cwd, @@ -3390,12 +3408,7 @@ describe("scripts/test-projects changed-target routing", () => { includePatterns: ["src/plugin-sdk/provider-entry.test.ts"], watchMode: false, }, - ...listFullExtensionVitestProjectConfigs().map((config) => ({ - config, - forwardedArgs: [], - includePatterns: null, - watchMode: false, - })), + ...listExpectedFullExtensionRunPlans(), ]); }); @@ -3452,14 +3465,84 @@ describe("scripts/test-projects changed-target routing", () => { }); it("routes the top-level extensions target to every extension shard", () => { - expect(buildVitestRunPlans(["extensions"], process.cwd())).toEqual( - listFullExtensionVitestProjectConfigs().map((config) => ({ - config, + const matrixConfig = "test/vitest/vitest.extension-matrix.config.ts"; + const plans = buildVitestRunPlans(["extensions"], process.cwd()); + const matrixPlans = plans.filter((plan) => plan.config === matrixConfig); + + expect(plans.filter((plan) => plan.config !== matrixConfig)).toEqual( + listFullExtensionVitestProjectConfigs() + .filter((config) => config !== matrixConfig) + .map((config) => ({ + config, + forwardedArgs: [], + includePatterns: null, + watchMode: false, + })), + ); + expect(matrixPlans).toHaveLength(3); + expect(matrixPlans.every((plan) => (plan.includePatterns?.length ?? 0) <= 40)).toBe(true); + expect(matrixPlans.flatMap((plan) => plan.includePatterns ?? [])).toEqual( + listExtensionTestFilesForRoots(["extensions/matrix"]), + ); + expect(plans).toEqual(listExpectedFullExtensionRunPlans()); + }); + + it("bounds an explicit Matrix directory target across process lifetimes", () => { + const plans = buildVitestRunPlans(["extensions/matrix"], process.cwd()); + + expect(plans).toHaveLength(3); + expect( + plans.every((plan) => plan.config === "test/vitest/vitest.extension-matrix.config.ts"), + ).toBe(true); + expect(plans.every((plan) => (plan.includePatterns?.length ?? 0) <= 40)).toBe(true); + expect(plans.flatMap((plan) => plan.includePatterns ?? [])).toEqual( + listExtensionTestFilesForRoots(["extensions/matrix"]), + ); + }); + + it("keeps grouped Matrix targets covered when bounding the directory", () => { + const testFile = listExtensionTestFilesForRoots(["extensions/matrix"])[0]; + if (!testFile) { + throw new Error("expected a Matrix test fixture"); + } + + const plans = buildVitestRunPlans(["extensions/matrix", testFile], process.cwd()); + + expect(plans).toHaveLength(3); + expect(plans.flatMap((plan) => plan.includePatterns ?? [])).toEqual( + listExtensionTestFilesForRoots(["extensions/matrix"]), + ); + }); + + it("keeps a grouped Matrix config target in the unsplit plan", () => { + expect( + buildVitestRunPlans( + ["extensions/matrix", "test/vitest/vitest.extension-matrix.config.ts"], + process.cwd(), + ), + ).toEqual([ + { + config: "test/vitest/vitest.extension-matrix.config.ts", forwardedArgs: [], includePatterns: null, watchMode: false, - })), - ); + }, + ]); + }); + + it("keeps explicit Matrix files and watch runs unchunked", () => { + const testFile = listExtensionTestFilesForRoots(["extensions/matrix"])[0]; + expect(testFile).toBeDefined(); + + expect(buildVitestRunPlans([testFile!], process.cwd())).toHaveLength(1); + expect(buildVitestRunPlans(["--watch", "extensions/matrix"], process.cwd())).toEqual([ + { + config: "test/vitest/vitest.extension-matrix.config.ts", + forwardedArgs: [], + includePatterns: ["extensions/matrix/**/*.test.ts"], + watchMode: true, + }, + ]); }); it("narrows default-lane changed source files to affected tests", () => { @@ -3937,12 +4020,7 @@ describe("scripts/test-projects changed-target routing", () => { includePatterns: ["src/plugin-sdk/facade-runtime.test.ts"], watchMode: false, }, - ...listFullExtensionVitestProjectConfigs().map((config) => ({ - config, - forwardedArgs: [], - includePatterns: null, - watchMode: false, - })), + ...listExpectedFullExtensionRunPlans(), ]); }); @@ -4596,10 +4674,12 @@ describe("scripts/test-projects full-suite sharding", () => { const agentsCoreConfig = "test/vitest/vitest.agents-core.config.ts"; const toolingConfig = "test/vitest/vitest.tooling.config.ts"; const unitFastConfig = "test/vitest/vitest.unit-fast.config.ts"; + const matrixConfig = "test/vitest/vitest.extension-matrix.config.ts"; const plans = leafShardPlans; const agentsCorePlans = plans.filter((plan) => plan.config === agentsCoreConfig); const toolingPlans = plans.filter((plan) => plan.config === toolingConfig); const unitFastPlans = plans.filter((plan) => plan.config === unitFastConfig); + const matrixPlans = plans.filter((plan) => plan.config === matrixConfig); if (leafShardHasGitGatewayListing) { expect(leafShardGatewayTreeReads).toEqual([]); @@ -4678,7 +4758,7 @@ describe("scripts/test-projects full-suite sharding", () => { "test/vitest/vitest.extension-irc.config.ts", "test/vitest/vitest.extension-line.config.ts", "test/vitest/vitest.extension-mattermost.config.ts", - "test/vitest/vitest.extension-matrix.config.ts", + ...matrixPlans.map(() => matrixConfig), "test/vitest/vitest.extension-memory.config.ts", "test/vitest/vitest.extension-messaging.config.ts", "test/vitest/vitest.extension-msteams.config.ts", @@ -4734,13 +4814,18 @@ describe("scripts/test-projects full-suite sharding", () => { expect(toolingTargets.some((target) => target.endsWith(".live.test.ts"))).toBe(false); expect(toolingTargets).not.toContain("test/scripts/docker-build-helper.test.ts"); expect(toolingTargets).not.toContain("test/scripts/openclaw-e2e-instance.test.ts"); + const matrixTargets = matrixPlans.flatMap((plan) => plan.forwardedArgs); + expect(matrixPlans).toHaveLength(3); + expect(matrixPlans.every((plan) => plan.forwardedArgs.length <= 40)).toBe(true); + expect(matrixTargets).toEqual(listExtensionTestFilesForRoots(["extensions/matrix"])); expect( plans.filter( (plan) => plan.config !== gatewayServerConfig && plan.config !== agentsCoreConfig && plan.config !== toolingConfig && - plan.config !== unitFastConfig, + plan.config !== unitFastConfig && + plan.config !== matrixConfig, ), ).toEqual( plans @@ -4749,7 +4834,8 @@ describe("scripts/test-projects full-suite sharding", () => { plan.config !== gatewayServerConfig && plan.config !== agentsCoreConfig && plan.config !== toolingConfig && - plan.config !== unitFastConfig, + plan.config !== unitFastConfig && + plan.config !== matrixConfig, ) .map((plan) => ({ config: plan.config,