perf(ci): content-hash boundary artifact freshness and resize Telegram shards (#123649)

Hosted CI runners restored the boundary-artifact cache and rebuilt it anyway: fresh checkouts re-stamp every input mtime, so mtime freshness never passed. Stamp files now record the input content digest and byte-identical inputs skip the rebuild (~60s saved per hosted lint/boundary job, 0.17s verify). Telegram CI shards pack ten files per job instead of five now that per-file import cost is back to seconds (#123607), halving the ~42-job fanout.
This commit is contained in:
Ayaan Zaidi
2026-08-14 18:13:12 +05:30
committed by GitHub
parent 44dd983c0c
commit af0221bba6
3 changed files with 210 additions and 29 deletions
+5 -3
View File
@@ -91,7 +91,7 @@ const EXTENSION_TEST_COST_MULTIPLIERS: Record<string, number> = {
};
export const MATRIX_EXTENSION_TEST_PROCESS_FILE_LIMIT = 40;
export const TELEGRAM_EXTENSION_TEST_PROCESS_FILE_LIMIT = 1;
export const TELEGRAM_EXTENSION_TEST_JOB_FILE_LIMIT = 5;
export const TELEGRAM_EXTENSION_TEST_JOB_FILE_LIMIT = 10;
const EXTENSION_TEST_PROCESS_FILE_LIMITS = new Map<string, number>([
// The non-isolated Matrix suite intentionally shares module state within a process.
// Bound its lifetime so Vite's transformed module graph cannot grow across the whole suite.
@@ -105,8 +105,10 @@ const EXTENSION_TEST_PROCESS_FILE_LIMITS = new Map<string, number>([
],
]);
const EXTENSION_TEST_JOB_FILE_LIMITS = new Map<string, number>([
// Keep Telegram CI jobs at five files so isolate recycling stays inside one
// job instead of minting one runner per test file.
// Bound Telegram CI jobs so isolate recycling stays inside one job instead
// of minting one runner per test file. Ten files keeps the worst job near
// 3 minutes (observed 2026-08: ~45s runner setup + ~7-24s per file) while
// halving the ~42-job fanout a Telegram-touching diff produced at five.
["test/vitest/vitest.extension-telegram.config.ts", TELEGRAM_EXTENSION_TEST_JOB_FILE_LIMIT],
]);
const EXTENSION_TEST_CONFIG_ROUTES: Array<[(root: string) => boolean, string]> = [
@@ -7,6 +7,7 @@ import {
type StdioNull,
type StdioPipe,
} from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import { createRequire } from "node:module";
import path, { resolve } from "node:path";
@@ -49,13 +50,16 @@ type NodeStep = Pick<NodeStepParams, "abortKillGraceMs" | "env"> & {
args: string[];
label: string;
timeoutMs: number;
stamp?: ArtifactStamp;
};
type ArtifactFreshParams = {
includeFile?: (filePath: string) => boolean;
inputPaths: string[];
outputPaths: string[];
rootDir?: string;
hashStampPath?: string;
};
type ArtifactStamp = Pick<ArtifactFreshParams, "includeFile" | "inputPaths"> & { path: string };
type NodeStepOutput = {
on(event: "data", listener: (chunk: string) => void): unknown;
setEncoding(encoding: "utf8"): void;
@@ -420,20 +424,19 @@ export function resolveBoundaryRootShimsTimeoutMs(env: NodeJS.ProcessEnv = proce
return parsePositiveInt(raw, "OPENCLAW_PLUGIN_SDK_BOUNDARY_ROOT_SHIMS_TIMEOUT_MS");
}
function collectNewestMtime(
function collectInputFiles(
paths: string[],
params: Pick<ArtifactFreshParams, "rootDir" | "includeFile"> = {},
) {
const rootDir = params.rootDir ?? repoRoot;
const includeFile = params.includeFile ?? (() => true);
let newestMtimeMs = 0;
const files: string[] = [];
function visit(entryPath: string): void {
if (!fs.existsSync(entryPath)) {
return;
}
const stats = fs.statSync(entryPath);
if (stats.isDirectory()) {
if (fs.statSync(entryPath).isDirectory()) {
for (const child of fs.readdirSync(entryPath)) {
visit(path.join(entryPath, child));
}
@@ -442,16 +445,61 @@ function collectNewestMtime(
if (!includeFile(entryPath)) {
return;
}
newestMtimeMs = Math.max(newestMtimeMs, stats.mtimeMs);
files.push(entryPath);
}
for (const relativePath of paths) {
visit(resolve(rootDir, relativePath));
}
return files;
}
function collectNewestMtime(
paths: string[],
params: Pick<ArtifactFreshParams, "rootDir" | "includeFile"> = {},
) {
let newestMtimeMs = 0;
for (const filePath of collectInputFiles(paths, params)) {
newestMtimeMs = Math.max(newestMtimeMs, fs.statSync(filePath).mtimeMs);
}
return newestMtimeMs;
}
const inputFileDigestMemo = new Map<string, string>();
// Keyed by stat identity, not path alone: entry-shim inputs include
// .tsbuildinfo files that lane builds rewrite mid-run.
function digestInputFile(filePath: string) {
const stats = fs.statSync(filePath);
const memoKey = `${filePath}\0${stats.size}\0${stats.mtimeMs}`;
const memoized = inputFileDigestMemo.get(memoKey);
if (memoized) {
return memoized;
}
const digest = createHash("sha256").update(fs.readFileSync(filePath)).digest("hex");
inputFileDigestMemo.set(memoKey, digest);
return digest;
}
/**
* Digests the exact input file set the mtime scan sees, so freshness can
* survive checkouts that re-stamp every file mtime (hosted CI runners).
*/
export function computeArtifactInputsDigest(
params: Pick<ArtifactFreshParams, "inputPaths" | "rootDir" | "includeFile">,
) {
const rootDir = params.rootDir ?? repoRoot;
const digest = createHash("sha256");
for (const filePath of collectInputFiles(params.inputPaths, params).toSorted()) {
digest.update(path.relative(rootDir, filePath));
digest.update("\0");
digest.update(digestInputFile(filePath));
digest.update("\n");
}
return digest.digest("hex");
}
function collectOldestMtime(paths: string[], params: Pick<ArtifactFreshParams, "rootDir"> = {}) {
const rootDir = params.rootDir ?? repoRoot;
let oldestMtimeMs = Number.POSITIVE_INFINITY;
@@ -468,7 +516,11 @@ function collectOldestMtime(paths: string[], params: Pick<ArtifactFreshParams, "
}
/**
* Compares input and output mtimes to skip fresh generated artifacts.
* Compares input and output mtimes to skip fresh generated artifacts. When the
* mtime fast path fails and the lane has a hash stamp, falls back to content
* identity: a fresh checkout re-stamps every input mtime, so cache-restored
* artifacts never look mtime-fresh on hosted CI runners even when no input
* byte changed.
*/
export function isArtifactSetFresh(params: ArtifactFreshParams) {
const newestInputMtimeMs = collectNewestMtime(params.inputPaths, {
@@ -478,7 +530,36 @@ export function isArtifactSetFresh(params: ArtifactFreshParams) {
const oldestOutputMtimeMs = collectOldestMtime(params.outputPaths, {
rootDir: params.rootDir,
});
return oldestOutputMtimeMs !== null && oldestOutputMtimeMs >= newestInputMtimeMs;
if (oldestOutputMtimeMs !== null && oldestOutputMtimeMs >= newestInputMtimeMs) {
return true;
}
if (!params.hashStampPath || oldestOutputMtimeMs === null) {
return false;
}
const rootDir = params.rootDir ?? repoRoot;
const stampPath = resolve(rootDir, params.hashStampPath);
let recordedDigest: string;
try {
recordedDigest = fs.readFileSync(stampPath, "utf8").trim();
} catch {
return false;
}
if (!/^[0-9a-f]{64}$/.test(recordedDigest)) {
return false;
}
if (recordedDigest !== computeArtifactInputsDigest(params)) {
return false;
}
// Repair the mtime fast path so later invocations in this checkout skip
// without re-reading every input byte.
const now = new Date();
for (const relativePath of params.outputPaths) {
const outputPath = resolve(rootDir, relativePath);
if (fs.existsSync(outputPath)) {
fs.utimesSync(outputPath, now, now);
}
}
return true;
}
function hasMissingOutput(paths: string[]) {
@@ -493,10 +574,13 @@ function removeStaleIncrementalState({ tsBuildInfoPath }: { tsBuildInfoPath: str
fs.rmSync(resolve(repoRoot, tsBuildInfoPath), { force: true });
}
function writeStampFile(relativePath: string) {
const filePath = resolve(repoRoot, relativePath);
// The stamp records the lane's input digest so cache-restored artifacts stay
// fresh across checkouts that rewrite mtimes; writing it last also gives the
// mtime fast path a floor newer than the lane's build.
function writeStampFile(stamp: ArtifactStamp) {
const filePath = resolve(repoRoot, stamp.path);
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, `${new Date().toISOString()}\n`, "utf8");
fs.writeFileSync(filePath, `${computeArtifactInputsDigest(stamp)}\n`, "utf8");
}
/**
@@ -813,68 +897,82 @@ async function main(argv: string[] = process.argv.slice(2)) {
isArtifactSetFresh({
inputPaths: ROOT_DTS_INPUTS,
outputPaths: [ROOT_DTS_STAMP, ...ROOT_DTS_REQUIRED_OUTPUTS],
hashStampPath: ROOT_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(ROOT_DTS_REQUIRED_OUTPUTS);
const packageDtsFresh =
isArtifactSetFresh({
inputPaths: PACKAGE_DTS_INPUTS,
outputPaths: [PACKAGE_DTS_STAMP, ...PACKAGE_DTS_REQUIRED_OUTPUTS],
hashStampPath: PACKAGE_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(PACKAGE_DTS_REQUIRED_OUTPUTS);
const entryShimsFresh = isArtifactSetFresh({
const entryShimsStamp: ArtifactStamp = {
path: "dist/plugin-sdk/.boundary-entry-shims.stamp",
inputPaths: [
...ENTRY_SHIMS_INPUTS,
"dist/plugin-sdk/.tsbuildinfo",
"packages/plugin-sdk/dist/.tsbuildinfo",
],
};
const entryShimsFresh = isArtifactSetFresh({
inputPaths: entryShimsStamp.inputPaths,
outputPaths: [
"dist/plugin-sdk/.boundary-entry-shims.stamp",
entryShimsStamp.path,
...resolveBoundaryEntryShimRequiredOutputs({
...process.env,
OPENCLAW_BUILD_PRIVATE_QA: "1",
}),
],
hashStampPath: entryShimsStamp.path,
});
const qaChannelDtsFresh =
isArtifactSetFresh({
inputPaths: QA_CHANNEL_DTS_INPUTS,
outputPaths: [QA_CHANNEL_DTS_STAMP, ...QA_CHANNEL_DTS_REQUIRED_OUTPUTS],
hashStampPath: QA_CHANNEL_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(QA_CHANNEL_DTS_REQUIRED_OUTPUTS);
const memoryCoreDtsFresh =
isArtifactSetFresh({
inputPaths: MEMORY_CORE_DTS_INPUTS,
outputPaths: [MEMORY_CORE_DTS_STAMP, ...MEMORY_CORE_DTS_REQUIRED_OUTPUTS],
hashStampPath: MEMORY_CORE_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(MEMORY_CORE_DTS_REQUIRED_OUTPUTS);
const matrixDtsFresh =
isArtifactSetFresh({
inputPaths: MATRIX_DTS_INPUTS,
outputPaths: [MATRIX_DTS_STAMP, ...MATRIX_DTS_REQUIRED_OUTPUTS],
hashStampPath: MATRIX_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(MATRIX_DTS_REQUIRED_OUTPUTS);
const discordDtsFresh =
isArtifactSetFresh({
inputPaths: DISCORD_DTS_INPUTS,
outputPaths: [DISCORD_DTS_STAMP, ...DISCORD_DTS_REQUIRED_OUTPUTS],
hashStampPath: DISCORD_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(DISCORD_DTS_REQUIRED_OUTPUTS);
const slackDtsFresh =
isArtifactSetFresh({
inputPaths: SLACK_DTS_INPUTS,
outputPaths: [SLACK_DTS_STAMP, ...SLACK_DTS_REQUIRED_OUTPUTS],
hashStampPath: SLACK_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(SLACK_DTS_REQUIRED_OUTPUTS);
const telegramDtsFresh =
isArtifactSetFresh({
inputPaths: TELEGRAM_DTS_INPUTS,
outputPaths: [TELEGRAM_DTS_STAMP, ...TELEGRAM_DTS_REQUIRED_OUTPUTS],
hashStampPath: TELEGRAM_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(TELEGRAM_DTS_REQUIRED_OUTPUTS);
const whatsappDtsFresh =
isArtifactSetFresh({
inputPaths: WHATSAPP_DTS_INPUTS,
outputPaths: [WHATSAPP_DTS_STAMP, ...WHATSAPP_DTS_REQUIRED_OUTPUTS],
hashStampPath: WHATSAPP_DTS_STAMP,
includeFile: isRelevantTypeInput,
}) && !hasMissingOutput(WHATSAPP_DTS_REQUIRED_OUTPUTS);
@@ -890,7 +988,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
args: [runTsgoScript, "-p", "tsconfig.plugin-sdk.dts.json", "--declaration", "true"],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: ROOT_DTS_STAMP,
stamp: {
path: ROOT_DTS_STAMP,
inputPaths: ROOT_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[plugin-sdk boundary dts] fresh; skipping\n");
@@ -905,7 +1007,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
args: [runTsgoScript, "-p", "packages/plugin-sdk/tsconfig.json", "--declaration", "true"],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: PACKAGE_DTS_STAMP,
stamp: {
path: PACKAGE_DTS_STAMP,
inputPaths: PACKAGE_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[plugin-sdk package boundary dts] fresh; skipping\n");
@@ -936,7 +1042,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: QA_CHANNEL_DTS_STAMP,
stamp: {
path: QA_CHANNEL_DTS_STAMP,
inputPaths: QA_CHANNEL_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[qa-channel boundary dts] fresh; skipping\n");
@@ -966,7 +1076,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: MEMORY_CORE_DTS_STAMP,
stamp: {
path: MEMORY_CORE_DTS_STAMP,
inputPaths: MEMORY_CORE_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[memory-core boundary dts] fresh; skipping\n");
@@ -996,7 +1110,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: MATRIX_DTS_STAMP,
stamp: {
path: MATRIX_DTS_STAMP,
inputPaths: MATRIX_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[matrix boundary dts] fresh; skipping\n");
@@ -1026,7 +1144,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: DISCORD_DTS_STAMP,
stamp: {
path: DISCORD_DTS_STAMP,
inputPaths: DISCORD_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[discord boundary dts] fresh; skipping\n");
@@ -1056,7 +1178,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: SLACK_DTS_STAMP,
stamp: {
path: SLACK_DTS_STAMP,
inputPaths: SLACK_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[slack boundary dts] fresh; skipping\n");
@@ -1086,7 +1212,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: WHATSAPP_DTS_STAMP,
stamp: {
path: WHATSAPP_DTS_STAMP,
inputPaths: WHATSAPP_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[whatsapp boundary dts] fresh; skipping\n");
@@ -1116,7 +1246,11 @@ async function main(argv: string[] = process.argv.slice(2)) {
],
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
timeoutMs: 300_000,
stampPath: TELEGRAM_DTS_STAMP,
stamp: {
path: TELEGRAM_DTS_STAMP,
inputPaths: TELEGRAM_DTS_INPUTS,
includeFile: isRelevantTypeInput,
},
});
} else {
process.stdout.write("[telegram boundary dts] fresh; skipping\n");
@@ -1126,8 +1260,8 @@ async function main(argv: string[] = process.argv.slice(2)) {
if (prerequisiteSteps.length > 0) {
await runNodeSteps(prerequisiteSteps);
for (const step of prerequisiteSteps) {
if (step.stampPath) {
writeStampFile(step.stampPath);
if (step.stamp) {
writeStampFile(step.stamp);
}
}
}
@@ -1148,6 +1282,9 @@ async function main(argv: string[] = process.argv.slice(2)) {
},
},
);
// Overwrite the child's timestamp stamp with the input digest after the
// prerequisite tsbuildinfo files have settled.
writeStampFile(entryShimsStamp);
} else if (mode === "all") {
process.stdout.write("[plugin-sdk boundary root shims] fresh; skipping\n");
}
@@ -1155,8 +1292,8 @@ async function main(argv: string[] = process.argv.slice(2)) {
if (dependentSteps.length > 0) {
await runNodeSteps(dependentSteps);
for (const step of dependentSteps) {
if (step.stampPath) {
writeStampFile(step.stampPath);
if (step.stamp) {
writeStampFile(step.stamp);
}
}
}
@@ -1,6 +1,5 @@
// Prepare Extension Package Boundary Artifacts tests cover prepare extension package boundary artifacts script behavior.
import { spawn } from "node:child_process";
// Prepare Extension Package Boundary Artifacts tests cover prepare extension package boundary artifacts script behavior.
import { EventEmitter } from "node:events";
import fs from "node:fs";
import os from "node:os";
@@ -15,6 +14,7 @@ import {
} from "../../scripts/lib/plugin-sdk-entries.mjs";
import { resolveWindowsTaskkillPath } from "../../scripts/lib/windows-taskkill.mjs";
import {
computeArtifactInputsDigest,
createPrefixedOutputWriter,
isArtifactSetFresh,
parseMode,
@@ -562,6 +562,48 @@ describe("prepare-extension-package-boundary-artifacts", () => {
).toBe(false);
});
it("keeps mtime-stale artifacts fresh when the hash stamp matches the input digest", () => {
// Regression: fresh checkouts re-stamp every input mtime, so cache-restored
// artifacts must stay fresh by content identity, not build again per CI run.
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-boundary-hash-"));
tempRoots.add(rootDir);
const inputPath = path.join(rootDir, "src", "demo.ts");
const stampPath = path.join(rootDir, "dist", ".demo.stamp");
const outputPath = path.join(rootDir, "dist", "demo.d.ts");
fs.mkdirSync(path.dirname(inputPath), { recursive: true });
fs.mkdirSync(path.dirname(stampPath), { recursive: true });
fs.writeFileSync(inputPath, "export const demo = 1;\n", "utf8");
fs.writeFileSync(outputPath, "export declare const demo = 1;\n", "utf8");
fs.writeFileSync(
stampPath,
`${computeArtifactInputsDigest({ rootDir, inputPaths: ["src"] })}\n`,
"utf8",
);
// Simulate checkout: inputs newer than restored outputs, bytes unchanged.
fs.utimesSync(stampPath, new Date(1_000), new Date(1_000));
fs.utimesSync(outputPath, new Date(1_000), new Date(1_000));
const freshParams = {
rootDir,
inputPaths: ["src"],
outputPaths: ["dist/.demo.stamp", "dist/demo.d.ts"],
hashStampPath: "dist/.demo.stamp",
};
expect(isArtifactSetFresh(freshParams)).toBe(true);
// The hash match repairs output mtimes so the next check takes the fast path.
expect(fs.statSync(outputPath).mtimeMs).toBeGreaterThanOrEqual(fs.statSync(inputPath).mtimeMs);
fs.appendFileSync(inputPath, "export const demoTwo = 2;\n", "utf8");
fs.utimesSync(outputPath, new Date(1_000), new Date(1_000));
expect(isArtifactSetFresh(freshParams)).toBe(false);
// Legacy timestamp stamps never satisfy the hash fallback.
fs.writeFileSync(stampPath, `${new Date(5_000).toISOString()}\n`, "utf8");
fs.utimesSync(stampPath, new Date(1_000), new Date(1_000));
expect(isArtifactSetFresh(freshParams)).toBe(false);
});
it("requires generated entry-shim outputs in addition to the freshness stamp", () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-boundary-entry-shims-"));
tempRoots.add(rootDir);