diff --git a/.github/workflows/plugin-prerelease.yml b/.github/workflows/plugin-prerelease.yml index 5099424d11e4..8a5a70d23fef 100644 --- a/.github/workflows/plugin-prerelease.yml +++ b/.github/workflows/plugin-prerelease.yml @@ -155,8 +155,9 @@ jobs: FULL_RELEASE_VALIDATION: ${{ inputs.full_release_validation && 'true' || 'false' }} run: | node --import tsx --input-type=module <<'EOF' - import { appendFileSync, existsSync } from "node:fs"; + import { appendFileSync, existsSync, globSync } from "node:fs"; import { execFileSync } from "node:child_process"; + import path from "node:path"; const createMatrix = (include) => ({ include }); const outputPath = process.env.GITHUB_OUTPUT; @@ -210,14 +211,27 @@ jobs: } try { - const { createExtensionTestShards, DEFAULT_EXTENSION_TEST_SHARD_COUNT } = await import( - targetPlanPath("extension-test-plan") - ); - extensionShards = createExtensionTestShards({ + const { + createExtensionTestShards, + DEFAULT_EXTENSION_TEST_SHARD_COUNT, + splitExtensionTestJobTargets, + } = await import(targetPlanPath("extension-test-plan")); + const allExtensionShards = createExtensionTestShards({ + shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT, + }); + const telegramPlanGroup = allExtensionShards + .find((shard) => shard.extensionIds.includes("telegram")) + ?.planGroups.find((group) => group.extensionIds.includes("telegram")); + const genericExtensionIds = allExtensionShards + .flatMap((shard) => shard.extensionIds) + .filter((extensionId) => extensionId !== "telegram"); + const batchShards = createExtensionTestShards({ + extensionIds: genericExtensionIds, shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT, }).map((shard) => ({ check_name: shard.checkName, extensions_csv: shard.extensionIds.join(","), + vitest_config: "", vitest_max_workers: shard.extensionIds.some((extensionId) => extensionId.startsWith("memory-"), ) @@ -225,12 +239,48 @@ jobs: : 1, runner: shard.extensionIds.some((extensionId) => extensionId.startsWith("memory-")) ? "blacksmith-16vcpu-ubuntu-2404" - : [0, 1, 2, 3].includes(shard.index) - ? "blacksmith-8vcpu-ubuntu-2404" - : "blacksmith-4vcpu-ubuntu-2404", + : [0, 1, 2, 3].includes(shard.index) + ? "blacksmith-8vcpu-ubuntu-2404" + : "blacksmith-4vcpu-ubuntu-2404", shard_index: shard.index + 1, task: "extensions-batch", })); + const telegramVitestConfig = telegramPlanGroup + ? (await import(`./${telegramPlanGroup.config}`)).default + : null; + const telegramTestConfig = telegramVitestConfig?.test ?? {}; + const telegramTestDir = telegramTestConfig.dir ?? process.cwd(); + const telegramTestExclude = (telegramTestConfig.exclude ?? []).map((pattern) => + path.isAbsolute(pattern) + ? path.relative(telegramTestDir, pattern).replaceAll("\\", "/") + : pattern, + ); + const telegramTestFiles = globSync(telegramTestConfig.include ?? [], { + cwd: telegramTestDir, + exclude: telegramTestExclude, + }) + .map((file) => + path + .relative(process.cwd(), path.resolve(telegramTestDir, file)) + .replaceAll("\\", "/"), + ) + .sort(); + const telegramJobTargets = telegramPlanGroup + ? splitExtensionTestJobTargets(telegramPlanGroup.config, telegramTestFiles) + : []; + const telegramShards = telegramPlanGroup + ? telegramJobTargets.map((includePatterns, index) => ({ + check_name: `checks-node-extensions-telegram-shard-${index + 1}`, + extensions_csv: "telegram", + includePatterns, + vitest_config: telegramPlanGroup.config, + vitest_max_workers: 1, + runner: "blacksmith-8vcpu-ubuntu-2404", + shard_index: DEFAULT_EXTENSION_TEST_SHARD_COUNT + index + 1, + task: "extension-file-shard", + })) + : []; + extensionShards = [...batchShards, ...telegramShards]; } catch (error) { const errorCode = error && typeof error === "object" && "code" in error ? error.code : ""; @@ -473,6 +523,7 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false + max-parallel: 12 matrix: ${{ fromJson(needs.preflight.outputs.plugin_prerelease_extension_matrix) }} steps: - name: Checkout @@ -496,7 +547,39 @@ jobs: OPENCLAW_EXTENSION_BATCH_PARALLEL: 2 OPENCLAW_VITEST_MAX_WORKERS: ${{ matrix.vitest_max_workers }} OPENCLAW_EXTENSION_BATCH: ${{ matrix.extensions_csv }} - run: pnpm test:extensions:batch "$OPENCLAW_EXTENSION_BATCH" -- --retry=1 --exclude extensions/codex/src/app-server/run-attempt.test.ts + OPENCLAW_EXTENSION_INCLUDE_PATTERNS_JSON: ${{ toJson(matrix.includePatterns) }} + OPENCLAW_EXTENSION_TASK: ${{ matrix.task }} + OPENCLAW_EXTENSION_VITEST_CONFIG: ${{ matrix.vitest_config }} + shell: bash + run: | + set -euo pipefail + case "$OPENCLAW_EXTENSION_TASK" in + extensions-batch) + pnpm test:extensions:batch "$OPENCLAW_EXTENSION_BATCH" -- --retry=1 --exclude extensions/codex/src/app-server/run-attempt.test.ts + ;; + extension-file-shard) + include_file="${RUNNER_TEMP}/telegram-test-include-${GITHUB_JOB}.json" + trap 'rm -f -- "$include_file"' EXIT + INCLUDE_FILE="$include_file" node --input-type=module <<'EOF' + import { writeFileSync } from "node:fs"; + + const patterns = JSON.parse(process.env.OPENCLAW_EXTENSION_INCLUDE_PATTERNS_JSON); + if (!Array.isArray(patterns) || patterns.length === 0) { + throw new Error("Invalid Telegram extension file shard"); + } + writeFileSync(process.env.INCLUDE_FILE, JSON.stringify(patterns), "utf8"); + EOF + OPENCLAW_TEST_PROJECTS_PARALLEL=2 \ + OPENCLAW_VITEST_INCLUDE_FILE="$include_file" \ + pnpm test -- "$OPENCLAW_EXTENSION_VITEST_CONFIG" + trap - EXIT + rm -f -- "$include_file" + ;; + *) + echo "Unknown extension test task: $OPENCLAW_EXTENSION_TASK" >&2 + exit 1 + ;; + esac plugin-prerelease-inspector: permissions: diff --git a/docs/ci.md b/docs/ci.md index 17d39df2231f..b31af5a8f70e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -649,7 +649,7 @@ The scheduled live/E2E workflow runs the full release-path Docker suite daily an ## Plugin Prerelease -`Plugin Prerelease` is more expensive product/package coverage, so it is a separate workflow dispatched by `Full Release Validation` or by an explicit operator. Normal pull requests, `main` pushes, and standalone manual CI dispatches keep that suite off. It balances bundled plugin tests across eight extension workers; those extension shard jobs run up to two plugin config groups at a time with one Vitest worker per group and a larger Node heap so import-heavy plugin batches do not create extra CI jobs. The release-only Docker prerelease path (enabled by the `full_release_validation` input) batches targeted Docker lanes in groups of four to avoid reserving dozens of runners for one-to-three-minute jobs. The workflow also uploads an informational `plugin-inspector-advisory` artifact from `@openclaw/plugin-inspector`; inspector findings are triage input and do not change the blocking Plugin Prerelease gate. +`Plugin Prerelease` is more expensive product/package coverage, so it is a separate workflow dispatched by `Full Release Validation` or by an explicit operator. Normal pull requests, `main` pushes, and standalone manual CI dispatches keep that suite off. It balances non-Telegram bundled plugin tests across eight generic extension workers; those jobs run up to two plugin config groups at a time with one Vitest worker per group and a larger Node heap. Telegram runs in dedicated shards of at most ten test files, preserving one-file Vitest processes while scheduling two processes concurrently. The combined extension matrix is capped at 12 concurrent jobs. The release-only Docker prerelease path (enabled by the `full_release_validation` input) batches targeted Docker lanes in groups of four to avoid reserving dozens of runners for one-to-three-minute jobs. The workflow also uploads an informational `plugin-inspector-advisory` artifact from `@openclaw/plugin-inspector`; inspector findings are triage input and do not change the blocking Plugin Prerelease gate. ## QA Lab diff --git a/test/scripts/plugin-prerelease-telegram-shards.test.ts b/test/scripts/plugin-prerelease-telegram-shards.test.ts new file mode 100644 index 000000000000..7ced1a8e54aa --- /dev/null +++ b/test/scripts/plugin-prerelease-telegram-shards.test.ts @@ -0,0 +1,234 @@ +import { spawnSync } from "node:child_process"; +import { globSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path, { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { + DEFAULT_EXTENSION_TEST_SHARD_COUNT, + createExtensionTestShards, + listTrackedTestFilesForRoots, + splitExtensionTestJobTargets, +} from "../../scripts/lib/extension-test-plan.mts"; +import { createVitestRunSpecs } from "../../scripts/test-projects.test-support.mts"; +import { createExtensionTelegramVitestConfig } from "../vitest/vitest.extension-telegram.config.ts"; + +type WorkflowStep = { + env?: Record; + name?: string; + run?: string; +}; + +type PluginPrereleaseMatrixRow = { + extensions_csv: string; + includePatterns: string[]; + task: string; +}; + +function readPluginPrereleaseWorkflow() { + return parse(readFileSync(".github/workflows/plugin-prerelease.yml", "utf8")); +} + +function listTelegramRunnableTestFiles() { + const testConfig = createExtensionTelegramVitestConfig({}).test ?? {}; + const dir = testConfig.dir ?? process.cwd(); + const exclude = (testConfig.exclude ?? []).map((pattern) => + path.isAbsolute(pattern) ? path.relative(dir, pattern).replaceAll("\\", "/") : pattern, + ); + return globSync(testConfig.include ?? [], { cwd: dir, exclude }) + .map((file) => path.relative(process.cwd(), path.resolve(dir, file)).replaceAll("\\", "/")) + .toSorted((left, right) => left.localeCompare(right)); +} + +function runPluginPrereleaseManifest() { + const workflow = readPluginPrereleaseWorkflow(); + const manifestStep = workflow.jobs.preflight.steps.find( + (step: WorkflowStep) => step.name === "Build plugin prerelease manifest", + ); + if (!manifestStep?.run) { + throw new Error("Missing plugin prerelease manifest step"); + } + const source = manifestStep.run.match( + /node --import tsx --input-type=module <<'EOF'\n([\s\S]*?)\nEOF/u, + )?.[1]; + if (!source) { + throw new Error("Missing plugin prerelease manifest source"); + } + + const root = mkdtempSync(join(tmpdir(), "openclaw-plugin-prerelease-telegram-shards-")); + const outputPath = join(root, "github-output"); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + EXPECTED_SHA: "", + FULL_RELEASE_VALIDATION: "false", + GITHUB_OUTPUT: outputPath, + }; + delete env.OPENCLAW_VITEST_INCLUDE_FILE; + const result = spawnSync(process.execPath, ["--import", "tsx", "--input-type=module"], { + cwd: process.cwd(), + encoding: "utf8", + env, + input: source, + }); + expect(result.status, result.stderr).toBe(0); + const output = new Map( + readFileSync(outputPath, "utf8") + .trim() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); + return JSON.parse(output.get("plugin_prerelease_extension_matrix") ?? "{}") as { + include: PluginPrereleaseMatrixRow[]; + }; + } finally { + rmSync(root, { force: true, recursive: true }); + } +} + +describe("plugin prerelease Telegram extension shards", () => { + it("keeps Telegram out of balanced batches and covers every extension exactly once", () => { + const allShards = createExtensionTestShards({ + cwd: process.cwd(), + shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT, + }); + const allExtensionIds = allShards.flatMap((shard) => shard.extensionIds); + const genericExtensionIds = allExtensionIds.filter((extensionId) => extensionId !== "telegram"); + const genericShards = createExtensionTestShards({ + cwd: process.cwd(), + extensionIds: genericExtensionIds, + shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT, + }); + + expect(genericShards).toHaveLength(DEFAULT_EXTENSION_TEST_SHARD_COUNT); + expect(genericShards.flatMap((shard) => shard.extensionIds)).not.toContain("telegram"); + expect( + genericShards + .flatMap((shard) => shard.extensionIds) + .toSorted((left, right) => left.localeCompare(right)), + ).toEqual(genericExtensionIds.toSorted((left, right) => left.localeCompare(right))); + expect(allExtensionIds.filter((extensionId) => extensionId === "telegram")).toEqual([ + "telegram", + ]); + expect( + allShards + .flatMap((shard) => shard.planGroups) + .find((group) => group.extensionIds.includes("telegram")), + ).toMatchObject({ + config: "test/vitest/vitest.extension-telegram.config.ts", + extensionIds: ["telegram"], + roots: ["extensions/telegram"], + }); + expect(new Set(genericShards.flatMap((shard) => shard.extensionIds)).size).toBe( + genericExtensionIds.length, + ); + }); + + it("keeps dedicated Telegram shards inside the existing aggregate job contract", () => { + const workflow = readPluginPrereleaseWorkflow(); + const extensionJob = workflow.jobs["plugin-prerelease-extension-shard"]; + const runStep = extensionJob.steps.find( + (step: WorkflowStep) => step.name === "Run extension shard", + ); + const suite = workflow.jobs["plugin-prerelease-suite"]; + const matrix = runPluginPrereleaseManifest(); + const genericRows = matrix.include.filter((row) => row.task === "extensions-batch"); + const telegramRows = matrix.include.filter((row) => row.task === "extension-file-shard"); + const trackedTelegramTestFiles = listTrackedTestFilesForRoots(["extensions/telegram"]); + const runnableTelegramTestFiles = listTelegramRunnableTestFiles(); + + expect(genericRows).toHaveLength(DEFAULT_EXTENSION_TEST_SHARD_COUNT); + expect(genericRows.some((row) => row.extensions_csv.split(",").includes("telegram"))).toBe( + false, + ); + const telegramConfig = "test/vitest/vitest.extension-telegram.config.ts"; + const expectedTelegramPartitions = splitExtensionTestJobTargets( + telegramConfig, + runnableTelegramTestFiles, + ); + expect(telegramRows).toHaveLength(expectedTelegramPartitions.length); + expect(telegramRows).toEqual( + expectedTelegramPartitions.map((includePatterns, index) => + expect.objectContaining({ + check_name: `checks-node-extensions-telegram-shard-${index + 1}`, + extensions_csv: "telegram", + includePatterns, + runner: "blacksmith-8vcpu-ubuntu-2404", + vitest_config: telegramConfig, + }), + ), + ); + const telegramPartitions = telegramRows.map((row) => { + expect(row.includePatterns).toBeInstanceOf(Array); + return row.includePatterns; + }); + expect(telegramPartitions.every((partition) => partition.length > 0)).toBe(true); + expect(telegramPartitions.flat().toSorted((left, right) => left.localeCompare(right))).toEqual( + runnableTelegramTestFiles, + ); + expect(new Set(telegramPartitions.flat()).size).toBe(runnableTelegramTestFiles.length); + expect(telegramPartitions.every((partition) => partition.length <= 10)).toBe(true); + expect(Math.max(...telegramPartitions.map((partition) => partition.length))).toBe(10); + expect( + trackedTelegramTestFiles.filter((file) => !runnableTelegramTestFiles.includes(file)).length, + ).toBeGreaterThan(0); + + const tempDir = mkdtempSync(join(tmpdir(), "openclaw-plugin-prerelease-telegram-specs-")); + try { + for (const [index, partition] of telegramPartitions.entries()) { + const includeFile = join(tempDir, `telegram-shard-${index + 1}.json`); + writeFileSync(includeFile, JSON.stringify(partition)); + const specs = createVitestRunSpecs(["test/vitest/vitest.extension-telegram.config.ts"], { + baseEnv: { + OPENCLAW_TEST_PROJECTS_PARALLEL: "2", + OPENCLAW_VITEST_INCLUDE_FILE: includeFile, + }, + }); + + expect(specs).toHaveLength(partition.length); + expect(specs.map((spec) => spec.includePatterns)).toEqual(partition.map((file) => [file])); + expect(new Set(specs.map((spec) => spec.env.OPENCLAW_VITEST_INCLUDE_FILE)).size).toBe( + partition.length, + ); + expect(specs.every((spec) => spec.env.OPENCLAW_VITEST_INCLUDE_FILE !== includeFile)).toBe( + true, + ); + } + } finally { + rmSync(tempDir, { force: true, recursive: true }); + } + + expect(extensionJob.strategy["fail-fast"]).toBe(false); + expect(extensionJob.strategy["max-parallel"]).toBe(12); + expect(extensionJob["timeout-minutes"]).toBe(60); + expect(extensionJob.strategy.matrix).toBe( + "${{ fromJson(needs.preflight.outputs.plugin_prerelease_extension_matrix) }}", + ); + expect(runStep?.env).toMatchObject({ + OPENCLAW_EXTENSION_INCLUDE_PATTERNS_JSON: "${{ toJson(matrix.includePatterns) }}", + OPENCLAW_EXTENSION_TASK: "${{ matrix.task }}", + OPENCLAW_EXTENSION_VITEST_CONFIG: "${{ matrix.vitest_config }}", + }); + expect(runStep?.run).toContain("extension-file-shard)"); + expect(runStep?.run).toContain("OPENCLAW_TEST_PROJECTS_PARALLEL=2"); + expect(runStep?.run).toContain('OPENCLAW_VITEST_INCLUDE_FILE="$include_file"'); + expect(runStep?.run).toContain('pnpm test -- "$OPENCLAW_EXTENSION_VITEST_CONFIG"'); + const shellCheck = spawnSync("bash", ["-n"], { + encoding: "utf8", + input: runStep?.run, + }); + expect(shellCheck.status, shellCheck.stderr).toBe(0); + expect(runStep?.run?.match(/extension-file-shard\)([\s\S]*?)\n\s*;;/u)?.[1]).not.toContain( + "--retry", + ); + expect(suite.needs).toContain("plugin-prerelease-extension-shard"); + expect( + suite.steps.find((step: WorkflowStep) => step.name === "Verify plugin prerelease suite").run, + ).toContain( + 'check_required "plugin-prerelease-extensions" "$RUN_EXTENSIONS" "$EXTENSIONS_RESULT"', + ); + }); +});