diff --git a/scripts/lib/extension-test-plan.mts b/scripts/lib/extension-test-plan.mts index e9414f5ac352..b3c7f2cdef8d 100644 --- a/scripts/lib/extension-test-plan.mts +++ b/scripts/lib/extension-test-plan.mts @@ -91,7 +91,7 @@ const EXTENSION_TEST_COST_MULTIPLIERS: Record = { }; export const MATRIX_EXTENSION_TEST_PROCESS_FILE_LIMIT = 40; export const TELEGRAM_EXTENSION_TEST_PROCESS_FILE_LIMIT = 1; -export const TELEGRAM_EXTENSION_TEST_JOB_FILE_LIMIT = 5; +export const TELEGRAM_EXTENSION_TEST_JOB_FILE_LIMIT = 10; const EXTENSION_TEST_PROCESS_FILE_LIMITS = new Map([ // The non-isolated Matrix suite intentionally shares module state within a process. // Bound its lifetime so Vite's transformed module graph cannot grow across the whole suite. @@ -105,8 +105,10 @@ const EXTENSION_TEST_PROCESS_FILE_LIMITS = new Map([ ], ]); const EXTENSION_TEST_JOB_FILE_LIMITS = new Map([ - // Keep Telegram CI jobs at five files so isolate recycling stays inside one - // job instead of minting one runner per test file. + // Bound Telegram CI jobs so isolate recycling stays inside one job instead + // of minting one runner per test file. Ten files keeps the worst job near + // 3 minutes (observed 2026-08: ~45s runner setup + ~7-24s per file) while + // halving the ~42-job fanout a Telegram-touching diff produced at five. ["test/vitest/vitest.extension-telegram.config.ts", TELEGRAM_EXTENSION_TEST_JOB_FILE_LIMIT], ]); const EXTENSION_TEST_CONFIG_ROUTES: Array<[(root: string) => boolean, string]> = [ diff --git a/scripts/prepare-extension-package-boundary-artifacts.mts b/scripts/prepare-extension-package-boundary-artifacts.mts index 6e418396ab3e..2a1a232afe50 100644 --- a/scripts/prepare-extension-package-boundary-artifacts.mts +++ b/scripts/prepare-extension-package-boundary-artifacts.mts @@ -7,6 +7,7 @@ import { type StdioNull, type StdioPipe, } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import { createRequire } from "node:module"; import path, { resolve } from "node:path"; @@ -49,13 +50,16 @@ type NodeStep = Pick & { args: string[]; label: string; timeoutMs: number; + stamp?: ArtifactStamp; }; type ArtifactFreshParams = { includeFile?: (filePath: string) => boolean; inputPaths: string[]; outputPaths: string[]; rootDir?: string; + hashStampPath?: string; }; +type ArtifactStamp = Pick & { path: string }; type NodeStepOutput = { on(event: "data", listener: (chunk: string) => void): unknown; setEncoding(encoding: "utf8"): void; @@ -420,20 +424,19 @@ export function resolveBoundaryRootShimsTimeoutMs(env: NodeJS.ProcessEnv = proce return parsePositiveInt(raw, "OPENCLAW_PLUGIN_SDK_BOUNDARY_ROOT_SHIMS_TIMEOUT_MS"); } -function collectNewestMtime( +function collectInputFiles( paths: string[], params: Pick = {}, ) { const rootDir = params.rootDir ?? repoRoot; const includeFile = params.includeFile ?? (() => true); - let newestMtimeMs = 0; + const files: string[] = []; function visit(entryPath: string): void { if (!fs.existsSync(entryPath)) { return; } - const stats = fs.statSync(entryPath); - if (stats.isDirectory()) { + if (fs.statSync(entryPath).isDirectory()) { for (const child of fs.readdirSync(entryPath)) { visit(path.join(entryPath, child)); } @@ -442,16 +445,61 @@ function collectNewestMtime( if (!includeFile(entryPath)) { return; } - newestMtimeMs = Math.max(newestMtimeMs, stats.mtimeMs); + files.push(entryPath); } for (const relativePath of paths) { visit(resolve(rootDir, relativePath)); } + return files; +} + +function collectNewestMtime( + paths: string[], + params: Pick = {}, +) { + let newestMtimeMs = 0; + for (const filePath of collectInputFiles(paths, params)) { + newestMtimeMs = Math.max(newestMtimeMs, fs.statSync(filePath).mtimeMs); + } return newestMtimeMs; } +const inputFileDigestMemo = new Map(); + +// Keyed by stat identity, not path alone: entry-shim inputs include +// .tsbuildinfo files that lane builds rewrite mid-run. +function digestInputFile(filePath: string) { + const stats = fs.statSync(filePath); + const memoKey = `${filePath}\0${stats.size}\0${stats.mtimeMs}`; + const memoized = inputFileDigestMemo.get(memoKey); + if (memoized) { + return memoized; + } + const digest = createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); + inputFileDigestMemo.set(memoKey, digest); + return digest; +} + +/** + * Digests the exact input file set the mtime scan sees, so freshness can + * survive checkouts that re-stamp every file mtime (hosted CI runners). + */ +export function computeArtifactInputsDigest( + params: Pick, +) { + const rootDir = params.rootDir ?? repoRoot; + const digest = createHash("sha256"); + for (const filePath of collectInputFiles(params.inputPaths, params).toSorted()) { + digest.update(path.relative(rootDir, filePath)); + digest.update("\0"); + digest.update(digestInputFile(filePath)); + digest.update("\n"); + } + return digest.digest("hex"); +} + function collectOldestMtime(paths: string[], params: Pick = {}) { const rootDir = params.rootDir ?? repoRoot; let oldestMtimeMs = Number.POSITIVE_INFINITY; @@ -468,7 +516,11 @@ function collectOldestMtime(paths: string[], params: Pick= newestInputMtimeMs; + if (oldestOutputMtimeMs !== null && oldestOutputMtimeMs >= newestInputMtimeMs) { + return true; + } + if (!params.hashStampPath || oldestOutputMtimeMs === null) { + return false; + } + const rootDir = params.rootDir ?? repoRoot; + const stampPath = resolve(rootDir, params.hashStampPath); + let recordedDigest: string; + try { + recordedDigest = fs.readFileSync(stampPath, "utf8").trim(); + } catch { + return false; + } + if (!/^[0-9a-f]{64}$/.test(recordedDigest)) { + return false; + } + if (recordedDigest !== computeArtifactInputsDigest(params)) { + return false; + } + // Repair the mtime fast path so later invocations in this checkout skip + // without re-reading every input byte. + const now = new Date(); + for (const relativePath of params.outputPaths) { + const outputPath = resolve(rootDir, relativePath); + if (fs.existsSync(outputPath)) { + fs.utimesSync(outputPath, now, now); + } + } + return true; } function hasMissingOutput(paths: string[]) { @@ -493,10 +574,13 @@ function removeStaleIncrementalState({ tsBuildInfoPath }: { tsBuildInfoPath: str fs.rmSync(resolve(repoRoot, tsBuildInfoPath), { force: true }); } -function writeStampFile(relativePath: string) { - const filePath = resolve(repoRoot, relativePath); +// The stamp records the lane's input digest so cache-restored artifacts stay +// fresh across checkouts that rewrite mtimes; writing it last also gives the +// mtime fast path a floor newer than the lane's build. +function writeStampFile(stamp: ArtifactStamp) { + const filePath = resolve(repoRoot, stamp.path); fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync(filePath, `${new Date().toISOString()}\n`, "utf8"); + fs.writeFileSync(filePath, `${computeArtifactInputsDigest(stamp)}\n`, "utf8"); } /** @@ -813,68 +897,82 @@ async function main(argv: string[] = process.argv.slice(2)) { isArtifactSetFresh({ inputPaths: ROOT_DTS_INPUTS, outputPaths: [ROOT_DTS_STAMP, ...ROOT_DTS_REQUIRED_OUTPUTS], + hashStampPath: ROOT_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(ROOT_DTS_REQUIRED_OUTPUTS); const packageDtsFresh = isArtifactSetFresh({ inputPaths: PACKAGE_DTS_INPUTS, outputPaths: [PACKAGE_DTS_STAMP, ...PACKAGE_DTS_REQUIRED_OUTPUTS], + hashStampPath: PACKAGE_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(PACKAGE_DTS_REQUIRED_OUTPUTS); - const entryShimsFresh = isArtifactSetFresh({ + const entryShimsStamp: ArtifactStamp = { + path: "dist/plugin-sdk/.boundary-entry-shims.stamp", inputPaths: [ ...ENTRY_SHIMS_INPUTS, "dist/plugin-sdk/.tsbuildinfo", "packages/plugin-sdk/dist/.tsbuildinfo", ], + }; + const entryShimsFresh = isArtifactSetFresh({ + inputPaths: entryShimsStamp.inputPaths, outputPaths: [ - "dist/plugin-sdk/.boundary-entry-shims.stamp", + entryShimsStamp.path, ...resolveBoundaryEntryShimRequiredOutputs({ ...process.env, OPENCLAW_BUILD_PRIVATE_QA: "1", }), ], + hashStampPath: entryShimsStamp.path, }); const qaChannelDtsFresh = isArtifactSetFresh({ inputPaths: QA_CHANNEL_DTS_INPUTS, outputPaths: [QA_CHANNEL_DTS_STAMP, ...QA_CHANNEL_DTS_REQUIRED_OUTPUTS], + hashStampPath: QA_CHANNEL_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(QA_CHANNEL_DTS_REQUIRED_OUTPUTS); const memoryCoreDtsFresh = isArtifactSetFresh({ inputPaths: MEMORY_CORE_DTS_INPUTS, outputPaths: [MEMORY_CORE_DTS_STAMP, ...MEMORY_CORE_DTS_REQUIRED_OUTPUTS], + hashStampPath: MEMORY_CORE_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(MEMORY_CORE_DTS_REQUIRED_OUTPUTS); const matrixDtsFresh = isArtifactSetFresh({ inputPaths: MATRIX_DTS_INPUTS, outputPaths: [MATRIX_DTS_STAMP, ...MATRIX_DTS_REQUIRED_OUTPUTS], + hashStampPath: MATRIX_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(MATRIX_DTS_REQUIRED_OUTPUTS); const discordDtsFresh = isArtifactSetFresh({ inputPaths: DISCORD_DTS_INPUTS, outputPaths: [DISCORD_DTS_STAMP, ...DISCORD_DTS_REQUIRED_OUTPUTS], + hashStampPath: DISCORD_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(DISCORD_DTS_REQUIRED_OUTPUTS); const slackDtsFresh = isArtifactSetFresh({ inputPaths: SLACK_DTS_INPUTS, outputPaths: [SLACK_DTS_STAMP, ...SLACK_DTS_REQUIRED_OUTPUTS], + hashStampPath: SLACK_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(SLACK_DTS_REQUIRED_OUTPUTS); const telegramDtsFresh = isArtifactSetFresh({ inputPaths: TELEGRAM_DTS_INPUTS, outputPaths: [TELEGRAM_DTS_STAMP, ...TELEGRAM_DTS_REQUIRED_OUTPUTS], + hashStampPath: TELEGRAM_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(TELEGRAM_DTS_REQUIRED_OUTPUTS); const whatsappDtsFresh = isArtifactSetFresh({ inputPaths: WHATSAPP_DTS_INPUTS, outputPaths: [WHATSAPP_DTS_STAMP, ...WHATSAPP_DTS_REQUIRED_OUTPUTS], + hashStampPath: WHATSAPP_DTS_STAMP, includeFile: isRelevantTypeInput, }) && !hasMissingOutput(WHATSAPP_DTS_REQUIRED_OUTPUTS); @@ -890,7 +988,11 @@ async function main(argv: string[] = process.argv.slice(2)) { args: [runTsgoScript, "-p", "tsconfig.plugin-sdk.dts.json", "--declaration", "true"], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: ROOT_DTS_STAMP, + stamp: { + path: ROOT_DTS_STAMP, + inputPaths: ROOT_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[plugin-sdk boundary dts] fresh; skipping\n"); @@ -905,7 +1007,11 @@ async function main(argv: string[] = process.argv.slice(2)) { args: [runTsgoScript, "-p", "packages/plugin-sdk/tsconfig.json", "--declaration", "true"], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: PACKAGE_DTS_STAMP, + stamp: { + path: PACKAGE_DTS_STAMP, + inputPaths: PACKAGE_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[plugin-sdk package boundary dts] fresh; skipping\n"); @@ -936,7 +1042,11 @@ async function main(argv: string[] = process.argv.slice(2)) { ], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: QA_CHANNEL_DTS_STAMP, + stamp: { + path: QA_CHANNEL_DTS_STAMP, + inputPaths: QA_CHANNEL_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[qa-channel boundary dts] fresh; skipping\n"); @@ -966,7 +1076,11 @@ async function main(argv: string[] = process.argv.slice(2)) { ], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: MEMORY_CORE_DTS_STAMP, + stamp: { + path: MEMORY_CORE_DTS_STAMP, + inputPaths: MEMORY_CORE_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[memory-core boundary dts] fresh; skipping\n"); @@ -996,7 +1110,11 @@ async function main(argv: string[] = process.argv.slice(2)) { ], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: MATRIX_DTS_STAMP, + stamp: { + path: MATRIX_DTS_STAMP, + inputPaths: MATRIX_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[matrix boundary dts] fresh; skipping\n"); @@ -1026,7 +1144,11 @@ async function main(argv: string[] = process.argv.slice(2)) { ], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: DISCORD_DTS_STAMP, + stamp: { + path: DISCORD_DTS_STAMP, + inputPaths: DISCORD_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[discord boundary dts] fresh; skipping\n"); @@ -1056,7 +1178,11 @@ async function main(argv: string[] = process.argv.slice(2)) { ], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: SLACK_DTS_STAMP, + stamp: { + path: SLACK_DTS_STAMP, + inputPaths: SLACK_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[slack boundary dts] fresh; skipping\n"); @@ -1086,7 +1212,11 @@ async function main(argv: string[] = process.argv.slice(2)) { ], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: WHATSAPP_DTS_STAMP, + stamp: { + path: WHATSAPP_DTS_STAMP, + inputPaths: WHATSAPP_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[whatsapp boundary dts] fresh; skipping\n"); @@ -1116,7 +1246,11 @@ async function main(argv: string[] = process.argv.slice(2)) { ], env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, timeoutMs: 300_000, - stampPath: TELEGRAM_DTS_STAMP, + stamp: { + path: TELEGRAM_DTS_STAMP, + inputPaths: TELEGRAM_DTS_INPUTS, + includeFile: isRelevantTypeInput, + }, }); } else { process.stdout.write("[telegram boundary dts] fresh; skipping\n"); @@ -1126,8 +1260,8 @@ async function main(argv: string[] = process.argv.slice(2)) { if (prerequisiteSteps.length > 0) { await runNodeSteps(prerequisiteSteps); for (const step of prerequisiteSteps) { - if (step.stampPath) { - writeStampFile(step.stampPath); + if (step.stamp) { + writeStampFile(step.stamp); } } } @@ -1148,6 +1282,9 @@ async function main(argv: string[] = process.argv.slice(2)) { }, }, ); + // Overwrite the child's timestamp stamp with the input digest after the + // prerequisite tsbuildinfo files have settled. + writeStampFile(entryShimsStamp); } else if (mode === "all") { process.stdout.write("[plugin-sdk boundary root shims] fresh; skipping\n"); } @@ -1155,8 +1292,8 @@ async function main(argv: string[] = process.argv.slice(2)) { if (dependentSteps.length > 0) { await runNodeSteps(dependentSteps); for (const step of dependentSteps) { - if (step.stampPath) { - writeStampFile(step.stampPath); + if (step.stamp) { + writeStampFile(step.stamp); } } } diff --git a/test/scripts/prepare-extension-package-boundary-artifacts.test.ts b/test/scripts/prepare-extension-package-boundary-artifacts.test.ts index 921dacb709a0..a27013b7e16e 100644 --- a/test/scripts/prepare-extension-package-boundary-artifacts.test.ts +++ b/test/scripts/prepare-extension-package-boundary-artifacts.test.ts @@ -1,6 +1,5 @@ // Prepare Extension Package Boundary Artifacts tests cover prepare extension package boundary artifacts script behavior. import { spawn } from "node:child_process"; -// Prepare Extension Package Boundary Artifacts tests cover prepare extension package boundary artifacts script behavior. import { EventEmitter } from "node:events"; import fs from "node:fs"; import os from "node:os"; @@ -15,6 +14,7 @@ import { } from "../../scripts/lib/plugin-sdk-entries.mjs"; import { resolveWindowsTaskkillPath } from "../../scripts/lib/windows-taskkill.mjs"; import { + computeArtifactInputsDigest, createPrefixedOutputWriter, isArtifactSetFresh, parseMode, @@ -562,6 +562,48 @@ describe("prepare-extension-package-boundary-artifacts", () => { ).toBe(false); }); + it("keeps mtime-stale artifacts fresh when the hash stamp matches the input digest", () => { + // Regression: fresh checkouts re-stamp every input mtime, so cache-restored + // artifacts must stay fresh by content identity, not build again per CI run. + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-boundary-hash-")); + tempRoots.add(rootDir); + const inputPath = path.join(rootDir, "src", "demo.ts"); + const stampPath = path.join(rootDir, "dist", ".demo.stamp"); + const outputPath = path.join(rootDir, "dist", "demo.d.ts"); + fs.mkdirSync(path.dirname(inputPath), { recursive: true }); + fs.mkdirSync(path.dirname(stampPath), { recursive: true }); + fs.writeFileSync(inputPath, "export const demo = 1;\n", "utf8"); + fs.writeFileSync(outputPath, "export declare const demo = 1;\n", "utf8"); + fs.writeFileSync( + stampPath, + `${computeArtifactInputsDigest({ rootDir, inputPaths: ["src"] })}\n`, + "utf8", + ); + + // Simulate checkout: inputs newer than restored outputs, bytes unchanged. + fs.utimesSync(stampPath, new Date(1_000), new Date(1_000)); + fs.utimesSync(outputPath, new Date(1_000), new Date(1_000)); + const freshParams = { + rootDir, + inputPaths: ["src"], + outputPaths: ["dist/.demo.stamp", "dist/demo.d.ts"], + hashStampPath: "dist/.demo.stamp", + }; + + expect(isArtifactSetFresh(freshParams)).toBe(true); + // The hash match repairs output mtimes so the next check takes the fast path. + expect(fs.statSync(outputPath).mtimeMs).toBeGreaterThanOrEqual(fs.statSync(inputPath).mtimeMs); + + fs.appendFileSync(inputPath, "export const demoTwo = 2;\n", "utf8"); + fs.utimesSync(outputPath, new Date(1_000), new Date(1_000)); + expect(isArtifactSetFresh(freshParams)).toBe(false); + + // Legacy timestamp stamps never satisfy the hash fallback. + fs.writeFileSync(stampPath, `${new Date(5_000).toISOString()}\n`, "utf8"); + fs.utimesSync(stampPath, new Date(1_000), new Date(1_000)); + expect(isArtifactSetFresh(freshParams)).toBe(false); + }); + it("requires generated entry-shim outputs in addition to the freshness stamp", () => { const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-boundary-entry-shims-")); tempRoots.add(rootDir);