mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 03:45:46 -06:00
perf(ci): split Telegram prerelease tests (#128459)
* perf(ci): split Telegram prerelease tests * fix(ci): preserve Telegram test process recycling * fix(ci): shard runnable Telegram tests * fix(ci): cap Telegram prerelease jobs * fix(ci): correct Telegram shard heredoc * test(ci): type Telegram prerelease matrix * docs(ci): explain Telegram prerelease shards
This commit is contained in:
Vincent Koc
@@ -155,8 +155,9 @@ jobs:
|
||||
FULL_RELEASE_VALIDATION: ${{ inputs.full_release_validation && 'true' || 'false' }}
|
||||
run: |
|
||||
node --import tsx --input-type=module <<'EOF'
|
||||
import { appendFileSync, existsSync } from "node:fs";
|
||||
import { appendFileSync, existsSync, globSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
|
||||
const createMatrix = (include) => ({ include });
|
||||
const outputPath = process.env.GITHUB_OUTPUT;
|
||||
@@ -210,14 +211,27 @@ jobs:
|
||||
}
|
||||
|
||||
try {
|
||||
const { createExtensionTestShards, DEFAULT_EXTENSION_TEST_SHARD_COUNT } = await import(
|
||||
targetPlanPath("extension-test-plan")
|
||||
);
|
||||
extensionShards = createExtensionTestShards({
|
||||
const {
|
||||
createExtensionTestShards,
|
||||
DEFAULT_EXTENSION_TEST_SHARD_COUNT,
|
||||
splitExtensionTestJobTargets,
|
||||
} = await import(targetPlanPath("extension-test-plan"));
|
||||
const allExtensionShards = createExtensionTestShards({
|
||||
shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT,
|
||||
});
|
||||
const telegramPlanGroup = allExtensionShards
|
||||
.find((shard) => shard.extensionIds.includes("telegram"))
|
||||
?.planGroups.find((group) => group.extensionIds.includes("telegram"));
|
||||
const genericExtensionIds = allExtensionShards
|
||||
.flatMap((shard) => shard.extensionIds)
|
||||
.filter((extensionId) => extensionId !== "telegram");
|
||||
const batchShards = createExtensionTestShards({
|
||||
extensionIds: genericExtensionIds,
|
||||
shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT,
|
||||
}).map((shard) => ({
|
||||
check_name: shard.checkName,
|
||||
extensions_csv: shard.extensionIds.join(","),
|
||||
vitest_config: "",
|
||||
vitest_max_workers: shard.extensionIds.some((extensionId) =>
|
||||
extensionId.startsWith("memory-"),
|
||||
)
|
||||
@@ -225,12 +239,48 @@ jobs:
|
||||
: 1,
|
||||
runner: shard.extensionIds.some((extensionId) => extensionId.startsWith("memory-"))
|
||||
? "blacksmith-16vcpu-ubuntu-2404"
|
||||
: [0, 1, 2, 3].includes(shard.index)
|
||||
? "blacksmith-8vcpu-ubuntu-2404"
|
||||
: "blacksmith-4vcpu-ubuntu-2404",
|
||||
: [0, 1, 2, 3].includes(shard.index)
|
||||
? "blacksmith-8vcpu-ubuntu-2404"
|
||||
: "blacksmith-4vcpu-ubuntu-2404",
|
||||
shard_index: shard.index + 1,
|
||||
task: "extensions-batch",
|
||||
}));
|
||||
const telegramVitestConfig = telegramPlanGroup
|
||||
? (await import(`./${telegramPlanGroup.config}`)).default
|
||||
: null;
|
||||
const telegramTestConfig = telegramVitestConfig?.test ?? {};
|
||||
const telegramTestDir = telegramTestConfig.dir ?? process.cwd();
|
||||
const telegramTestExclude = (telegramTestConfig.exclude ?? []).map((pattern) =>
|
||||
path.isAbsolute(pattern)
|
||||
? path.relative(telegramTestDir, pattern).replaceAll("\\", "/")
|
||||
: pattern,
|
||||
);
|
||||
const telegramTestFiles = globSync(telegramTestConfig.include ?? [], {
|
||||
cwd: telegramTestDir,
|
||||
exclude: telegramTestExclude,
|
||||
})
|
||||
.map((file) =>
|
||||
path
|
||||
.relative(process.cwd(), path.resolve(telegramTestDir, file))
|
||||
.replaceAll("\\", "/"),
|
||||
)
|
||||
.sort();
|
||||
const telegramJobTargets = telegramPlanGroup
|
||||
? splitExtensionTestJobTargets(telegramPlanGroup.config, telegramTestFiles)
|
||||
: [];
|
||||
const telegramShards = telegramPlanGroup
|
||||
? telegramJobTargets.map((includePatterns, index) => ({
|
||||
check_name: `checks-node-extensions-telegram-shard-${index + 1}`,
|
||||
extensions_csv: "telegram",
|
||||
includePatterns,
|
||||
vitest_config: telegramPlanGroup.config,
|
||||
vitest_max_workers: 1,
|
||||
runner: "blacksmith-8vcpu-ubuntu-2404",
|
||||
shard_index: DEFAULT_EXTENSION_TEST_SHARD_COUNT + index + 1,
|
||||
task: "extension-file-shard",
|
||||
}))
|
||||
: [];
|
||||
extensionShards = [...batchShards, ...telegramShards];
|
||||
} catch (error) {
|
||||
const errorCode =
|
||||
error && typeof error === "object" && "code" in error ? error.code : "";
|
||||
@@ -473,6 +523,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 12
|
||||
matrix: ${{ fromJson(needs.preflight.outputs.plugin_prerelease_extension_matrix) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -496,7 +547,39 @@ jobs:
|
||||
OPENCLAW_EXTENSION_BATCH_PARALLEL: 2
|
||||
OPENCLAW_VITEST_MAX_WORKERS: ${{ matrix.vitest_max_workers }}
|
||||
OPENCLAW_EXTENSION_BATCH: ${{ matrix.extensions_csv }}
|
||||
run: pnpm test:extensions:batch "$OPENCLAW_EXTENSION_BATCH" -- --retry=1 --exclude extensions/codex/src/app-server/run-attempt.test.ts
|
||||
OPENCLAW_EXTENSION_INCLUDE_PATTERNS_JSON: ${{ toJson(matrix.includePatterns) }}
|
||||
OPENCLAW_EXTENSION_TASK: ${{ matrix.task }}
|
||||
OPENCLAW_EXTENSION_VITEST_CONFIG: ${{ matrix.vitest_config }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$OPENCLAW_EXTENSION_TASK" in
|
||||
extensions-batch)
|
||||
pnpm test:extensions:batch "$OPENCLAW_EXTENSION_BATCH" -- --retry=1 --exclude extensions/codex/src/app-server/run-attempt.test.ts
|
||||
;;
|
||||
extension-file-shard)
|
||||
include_file="${RUNNER_TEMP}/telegram-test-include-${GITHUB_JOB}.json"
|
||||
trap 'rm -f -- "$include_file"' EXIT
|
||||
INCLUDE_FILE="$include_file" node --input-type=module <<'EOF'
|
||||
import { writeFileSync } from "node:fs";
|
||||
|
||||
const patterns = JSON.parse(process.env.OPENCLAW_EXTENSION_INCLUDE_PATTERNS_JSON);
|
||||
if (!Array.isArray(patterns) || patterns.length === 0) {
|
||||
throw new Error("Invalid Telegram extension file shard");
|
||||
}
|
||||
writeFileSync(process.env.INCLUDE_FILE, JSON.stringify(patterns), "utf8");
|
||||
EOF
|
||||
OPENCLAW_TEST_PROJECTS_PARALLEL=2 \
|
||||
OPENCLAW_VITEST_INCLUDE_FILE="$include_file" \
|
||||
pnpm test -- "$OPENCLAW_EXTENSION_VITEST_CONFIG"
|
||||
trap - EXIT
|
||||
rm -f -- "$include_file"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown extension test task: $OPENCLAW_EXTENSION_TASK" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
plugin-prerelease-inspector:
|
||||
permissions:
|
||||
|
||||
+1
-1
@@ -649,7 +649,7 @@ The scheduled live/E2E workflow runs the full release-path Docker suite daily an
|
||||
|
||||
## Plugin Prerelease
|
||||
|
||||
`Plugin Prerelease` is more expensive product/package coverage, so it is a separate workflow dispatched by `Full Release Validation` or by an explicit operator. Normal pull requests, `main` pushes, and standalone manual CI dispatches keep that suite off. It balances bundled plugin tests across eight extension workers; those extension shard jobs run up to two plugin config groups at a time with one Vitest worker per group and a larger Node heap so import-heavy plugin batches do not create extra CI jobs. The release-only Docker prerelease path (enabled by the `full_release_validation` input) batches targeted Docker lanes in groups of four to avoid reserving dozens of runners for one-to-three-minute jobs. The workflow also uploads an informational `plugin-inspector-advisory` artifact from `@openclaw/plugin-inspector`; inspector findings are triage input and do not change the blocking Plugin Prerelease gate.
|
||||
`Plugin Prerelease` is more expensive product/package coverage, so it is a separate workflow dispatched by `Full Release Validation` or by an explicit operator. Normal pull requests, `main` pushes, and standalone manual CI dispatches keep that suite off. It balances non-Telegram bundled plugin tests across eight generic extension workers; those jobs run up to two plugin config groups at a time with one Vitest worker per group and a larger Node heap. Telegram runs in dedicated shards of at most ten test files, preserving one-file Vitest processes while scheduling two processes concurrently. The combined extension matrix is capped at 12 concurrent jobs. The release-only Docker prerelease path (enabled by the `full_release_validation` input) batches targeted Docker lanes in groups of four to avoid reserving dozens of runners for one-to-three-minute jobs. The workflow also uploads an informational `plugin-inspector-advisory` artifact from `@openclaw/plugin-inspector`; inspector findings are triage input and do not change the blocking Plugin Prerelease gate.
|
||||
|
||||
## QA Lab
|
||||
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { globSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path, { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import {
|
||||
DEFAULT_EXTENSION_TEST_SHARD_COUNT,
|
||||
createExtensionTestShards,
|
||||
listTrackedTestFilesForRoots,
|
||||
splitExtensionTestJobTargets,
|
||||
} from "../../scripts/lib/extension-test-plan.mts";
|
||||
import { createVitestRunSpecs } from "../../scripts/test-projects.test-support.mts";
|
||||
import { createExtensionTelegramVitestConfig } from "../vitest/vitest.extension-telegram.config.ts";
|
||||
|
||||
type WorkflowStep = {
|
||||
env?: Record<string, string>;
|
||||
name?: string;
|
||||
run?: string;
|
||||
};
|
||||
|
||||
type PluginPrereleaseMatrixRow = {
|
||||
extensions_csv: string;
|
||||
includePatterns: string[];
|
||||
task: string;
|
||||
};
|
||||
|
||||
function readPluginPrereleaseWorkflow() {
|
||||
return parse(readFileSync(".github/workflows/plugin-prerelease.yml", "utf8"));
|
||||
}
|
||||
|
||||
function listTelegramRunnableTestFiles() {
|
||||
const testConfig = createExtensionTelegramVitestConfig({}).test ?? {};
|
||||
const dir = testConfig.dir ?? process.cwd();
|
||||
const exclude = (testConfig.exclude ?? []).map((pattern) =>
|
||||
path.isAbsolute(pattern) ? path.relative(dir, pattern).replaceAll("\\", "/") : pattern,
|
||||
);
|
||||
return globSync(testConfig.include ?? [], { cwd: dir, exclude })
|
||||
.map((file) => path.relative(process.cwd(), path.resolve(dir, file)).replaceAll("\\", "/"))
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function runPluginPrereleaseManifest() {
|
||||
const workflow = readPluginPrereleaseWorkflow();
|
||||
const manifestStep = workflow.jobs.preflight.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Build plugin prerelease manifest",
|
||||
);
|
||||
if (!manifestStep?.run) {
|
||||
throw new Error("Missing plugin prerelease manifest step");
|
||||
}
|
||||
const source = manifestStep.run.match(
|
||||
/node --import tsx --input-type=module <<'EOF'\n([\s\S]*?)\nEOF/u,
|
||||
)?.[1];
|
||||
if (!source) {
|
||||
throw new Error("Missing plugin prerelease manifest source");
|
||||
}
|
||||
|
||||
const root = mkdtempSync(join(tmpdir(), "openclaw-plugin-prerelease-telegram-shards-"));
|
||||
const outputPath = join(root, "github-output");
|
||||
try {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
EXPECTED_SHA: "",
|
||||
FULL_RELEASE_VALIDATION: "false",
|
||||
GITHUB_OUTPUT: outputPath,
|
||||
};
|
||||
delete env.OPENCLAW_VITEST_INCLUDE_FILE;
|
||||
const result = spawnSync(process.execPath, ["--import", "tsx", "--input-type=module"], {
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
env,
|
||||
input: source,
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const output = new Map(
|
||||
readFileSync(outputPath, "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
const separator = line.indexOf("=");
|
||||
return [line.slice(0, separator), line.slice(separator + 1)];
|
||||
}),
|
||||
);
|
||||
return JSON.parse(output.get("plugin_prerelease_extension_matrix") ?? "{}") as {
|
||||
include: PluginPrereleaseMatrixRow[];
|
||||
};
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe("plugin prerelease Telegram extension shards", () => {
|
||||
it("keeps Telegram out of balanced batches and covers every extension exactly once", () => {
|
||||
const allShards = createExtensionTestShards({
|
||||
cwd: process.cwd(),
|
||||
shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT,
|
||||
});
|
||||
const allExtensionIds = allShards.flatMap((shard) => shard.extensionIds);
|
||||
const genericExtensionIds = allExtensionIds.filter((extensionId) => extensionId !== "telegram");
|
||||
const genericShards = createExtensionTestShards({
|
||||
cwd: process.cwd(),
|
||||
extensionIds: genericExtensionIds,
|
||||
shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT,
|
||||
});
|
||||
|
||||
expect(genericShards).toHaveLength(DEFAULT_EXTENSION_TEST_SHARD_COUNT);
|
||||
expect(genericShards.flatMap((shard) => shard.extensionIds)).not.toContain("telegram");
|
||||
expect(
|
||||
genericShards
|
||||
.flatMap((shard) => shard.extensionIds)
|
||||
.toSorted((left, right) => left.localeCompare(right)),
|
||||
).toEqual(genericExtensionIds.toSorted((left, right) => left.localeCompare(right)));
|
||||
expect(allExtensionIds.filter((extensionId) => extensionId === "telegram")).toEqual([
|
||||
"telegram",
|
||||
]);
|
||||
expect(
|
||||
allShards
|
||||
.flatMap((shard) => shard.planGroups)
|
||||
.find((group) => group.extensionIds.includes("telegram")),
|
||||
).toMatchObject({
|
||||
config: "test/vitest/vitest.extension-telegram.config.ts",
|
||||
extensionIds: ["telegram"],
|
||||
roots: ["extensions/telegram"],
|
||||
});
|
||||
expect(new Set(genericShards.flatMap((shard) => shard.extensionIds)).size).toBe(
|
||||
genericExtensionIds.length,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps dedicated Telegram shards inside the existing aggregate job contract", () => {
|
||||
const workflow = readPluginPrereleaseWorkflow();
|
||||
const extensionJob = workflow.jobs["plugin-prerelease-extension-shard"];
|
||||
const runStep = extensionJob.steps.find(
|
||||
(step: WorkflowStep) => step.name === "Run extension shard",
|
||||
);
|
||||
const suite = workflow.jobs["plugin-prerelease-suite"];
|
||||
const matrix = runPluginPrereleaseManifest();
|
||||
const genericRows = matrix.include.filter((row) => row.task === "extensions-batch");
|
||||
const telegramRows = matrix.include.filter((row) => row.task === "extension-file-shard");
|
||||
const trackedTelegramTestFiles = listTrackedTestFilesForRoots(["extensions/telegram"]);
|
||||
const runnableTelegramTestFiles = listTelegramRunnableTestFiles();
|
||||
|
||||
expect(genericRows).toHaveLength(DEFAULT_EXTENSION_TEST_SHARD_COUNT);
|
||||
expect(genericRows.some((row) => row.extensions_csv.split(",").includes("telegram"))).toBe(
|
||||
false,
|
||||
);
|
||||
const telegramConfig = "test/vitest/vitest.extension-telegram.config.ts";
|
||||
const expectedTelegramPartitions = splitExtensionTestJobTargets(
|
||||
telegramConfig,
|
||||
runnableTelegramTestFiles,
|
||||
);
|
||||
expect(telegramRows).toHaveLength(expectedTelegramPartitions.length);
|
||||
expect(telegramRows).toEqual(
|
||||
expectedTelegramPartitions.map((includePatterns, index) =>
|
||||
expect.objectContaining({
|
||||
check_name: `checks-node-extensions-telegram-shard-${index + 1}`,
|
||||
extensions_csv: "telegram",
|
||||
includePatterns,
|
||||
runner: "blacksmith-8vcpu-ubuntu-2404",
|
||||
vitest_config: telegramConfig,
|
||||
}),
|
||||
),
|
||||
);
|
||||
const telegramPartitions = telegramRows.map((row) => {
|
||||
expect(row.includePatterns).toBeInstanceOf(Array);
|
||||
return row.includePatterns;
|
||||
});
|
||||
expect(telegramPartitions.every((partition) => partition.length > 0)).toBe(true);
|
||||
expect(telegramPartitions.flat().toSorted((left, right) => left.localeCompare(right))).toEqual(
|
||||
runnableTelegramTestFiles,
|
||||
);
|
||||
expect(new Set(telegramPartitions.flat()).size).toBe(runnableTelegramTestFiles.length);
|
||||
expect(telegramPartitions.every((partition) => partition.length <= 10)).toBe(true);
|
||||
expect(Math.max(...telegramPartitions.map((partition) => partition.length))).toBe(10);
|
||||
expect(
|
||||
trackedTelegramTestFiles.filter((file) => !runnableTelegramTestFiles.includes(file)).length,
|
||||
).toBeGreaterThan(0);
|
||||
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "openclaw-plugin-prerelease-telegram-specs-"));
|
||||
try {
|
||||
for (const [index, partition] of telegramPartitions.entries()) {
|
||||
const includeFile = join(tempDir, `telegram-shard-${index + 1}.json`);
|
||||
writeFileSync(includeFile, JSON.stringify(partition));
|
||||
const specs = createVitestRunSpecs(["test/vitest/vitest.extension-telegram.config.ts"], {
|
||||
baseEnv: {
|
||||
OPENCLAW_TEST_PROJECTS_PARALLEL: "2",
|
||||
OPENCLAW_VITEST_INCLUDE_FILE: includeFile,
|
||||
},
|
||||
});
|
||||
|
||||
expect(specs).toHaveLength(partition.length);
|
||||
expect(specs.map((spec) => spec.includePatterns)).toEqual(partition.map((file) => [file]));
|
||||
expect(new Set(specs.map((spec) => spec.env.OPENCLAW_VITEST_INCLUDE_FILE)).size).toBe(
|
||||
partition.length,
|
||||
);
|
||||
expect(specs.every((spec) => spec.env.OPENCLAW_VITEST_INCLUDE_FILE !== includeFile)).toBe(
|
||||
true,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
rmSync(tempDir, { force: true, recursive: true });
|
||||
}
|
||||
|
||||
expect(extensionJob.strategy["fail-fast"]).toBe(false);
|
||||
expect(extensionJob.strategy["max-parallel"]).toBe(12);
|
||||
expect(extensionJob["timeout-minutes"]).toBe(60);
|
||||
expect(extensionJob.strategy.matrix).toBe(
|
||||
"${{ fromJson(needs.preflight.outputs.plugin_prerelease_extension_matrix) }}",
|
||||
);
|
||||
expect(runStep?.env).toMatchObject({
|
||||
OPENCLAW_EXTENSION_INCLUDE_PATTERNS_JSON: "${{ toJson(matrix.includePatterns) }}",
|
||||
OPENCLAW_EXTENSION_TASK: "${{ matrix.task }}",
|
||||
OPENCLAW_EXTENSION_VITEST_CONFIG: "${{ matrix.vitest_config }}",
|
||||
});
|
||||
expect(runStep?.run).toContain("extension-file-shard)");
|
||||
expect(runStep?.run).toContain("OPENCLAW_TEST_PROJECTS_PARALLEL=2");
|
||||
expect(runStep?.run).toContain('OPENCLAW_VITEST_INCLUDE_FILE="$include_file"');
|
||||
expect(runStep?.run).toContain('pnpm test -- "$OPENCLAW_EXTENSION_VITEST_CONFIG"');
|
||||
const shellCheck = spawnSync("bash", ["-n"], {
|
||||
encoding: "utf8",
|
||||
input: runStep?.run,
|
||||
});
|
||||
expect(shellCheck.status, shellCheck.stderr).toBe(0);
|
||||
expect(runStep?.run?.match(/extension-file-shard\)([\s\S]*?)\n\s*;;/u)?.[1]).not.toContain(
|
||||
"--retry",
|
||||
);
|
||||
expect(suite.needs).toContain("plugin-prerelease-extension-shard");
|
||||
expect(
|
||||
suite.steps.find((step: WorkflowStep) => step.name === "Verify plugin prerelease suite").run,
|
||||
).toContain(
|
||||
'check_required "plugin-prerelease-extensions" "$RUN_EXTENSIONS" "$EXTENSIONS_RESULT"',
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user