diff --git a/scripts/postinstall-bundled-plugins.mjs b/scripts/postinstall-bundled-plugins.mjs index 0fd184114c79..39dc35c42b1c 100644 --- a/scripts/postinstall-bundled-plugins.mjs +++ b/scripts/postinstall-bundled-plugins.mjs @@ -8,6 +8,7 @@ import { closeSync, existsSync, lstatSync, + opendirSync, openSync, readdirSync, readFileSync, @@ -29,6 +30,7 @@ const DEFAULT_PACKAGE_ROOT = join(scriptDir, ".."); const DISABLE_POSTINSTALL_ENV = "OPENCLAW_DISABLE_BUNDLED_PLUGIN_POSTINSTALL"; const DISABLE_PLUGIN_REGISTRY_MIGRATION_ENV = "OPENCLAW_DISABLE_PLUGIN_REGISTRY_MIGRATION"; const DIST_INVENTORY_PATH = "dist/postinstall-inventory.json"; +export const MAX_INSTALLED_DIST_SCAN_ENTRIES = 25_000; const LEGACY_PLUGIN_RUNTIME_DEPS_DIR = "plugin-runtime-deps"; const BAILEYS_MEDIA_FILE = join("node_modules", "baileys", "lib", "Utils", "messages-media.js"); const BAILEYS_MEDIA_HOTFIX_NEEDLE = [ @@ -107,6 +109,8 @@ const BAILEYS_MEDIA_ASYNC_CONTEXT_RE = /async\s+function\s+encryptedStream|encryptedStream\s*=\s*async/u; const NODE_COMPILE_CACHE_VERSION_DIR_RE = /^v\d+\.\d+\.\d+-/u; +class InstalledDistScanLimitError extends Error {} + function hasEnvFlag(env, key) { const value = env?.[key]?.trim().toLowerCase(); return Boolean(value && value !== "0" && value !== "false" && value !== "no"); @@ -197,8 +201,57 @@ function assertSafeInstalledDistPath(relativePath, params) { return candidatePath; } +function createInstalledDistScanBudget(params = {}) { + return { + entries: 0, + limit: params.maxDistScanEntries ?? MAX_INSTALLED_DIST_SCAN_ENTRIES, + }; +} + +function resolveInstalledDistScanBudget(params = {}) { + return params.distScanBudget ?? createInstalledDistScanBudget(params); +} + +function countInstalledDistScanEntry(budget) { + budget.entries += 1; + if (budget.entries > budget.limit) { + throw new InstalledDistScanLimitError( + `installed dist scan exceeded ${budget.limit} filesystem entries; refusing to scan unbounded package contents`, + ); + } +} + +function* iterateInstalledDistEntries(currentDir, params = {}) { + if (params.readdirSync) { + yield* params.readdirSync(currentDir, { withFileTypes: true }); + return; + } + + const dir = opendirSync(currentDir); + try { + while (true) { + const entry = dir.readSync(); + if (!entry) { + break; + } + yield entry; + } + } finally { + dir.closeSync(); + } +} + +function* iterateOptionalInstalledDistEntries(currentDir, params = {}) { + try { + yield* iterateInstalledDistEntries(currentDir, params); + } catch (error) { + if (error instanceof InstalledDistScanLimitError) { + throw error; + } + } +} + function listInstalledDistFiles(params = {}) { - const readDir = params.readdirSync ?? readdirSync; const distRoot = resolveInstalledDistRoot(params); if (distRoot === null) { return []; @@ -206,12 +259,14 @@ function listInstalledDistFiles(params = {}) { const packageRoot = params.packageRoot ?? DEFAULT_PACKAGE_ROOT; const pending = [distRoot.distDir]; const files = []; + const budget = resolveInstalledDistScanBudget(params); while (pending.length > 0) { const currentDir = pending.pop(); if (!currentDir) { continue; } - for (const entry of readDir(currentDir, { withFileTypes: true })) { + for (const entry of iterateInstalledDistEntries(currentDir, params)) { + countInstalledDistScanEntry(budget); const entryPath = join(currentDir, entry.name); if (entry.isSymbolicLink()) { throw new Error( @@ -236,7 +291,6 @@ function listInstalledDistFiles(params = {}) { } function pruneEmptyDistDirectories(params = {}) { - const readDir = params.readdirSync ?? readdirSync; const removeDirectory = params.rmdirSync ?? rmdirSync; const distRoot = resolveInstalledDistRoot(params); if (distRoot === null) { @@ -244,9 +298,21 @@ function pruneEmptyDistDirectories(params = {}) { } const packageRoot = params.packageRoot ?? DEFAULT_PACKAGE_ROOT; const pathLstat = params.lstatSync ?? lstatSync; + const budget = resolveInstalledDistScanBudget(params); + + function isDirectoryEmpty(currentDir) { + for (const entry of iterateInstalledDistEntries(currentDir, params)) { + void entry; + countInstalledDistScanEntry(budget); + return false; + } + return true; + } function prune(currentDir) { - for (const entry of readDir(currentDir, { withFileTypes: true })) { + const childDirs = []; + for (const entry of iterateInstalledDistEntries(currentDir, params)) { + countInstalledDistScanEntry(budget); if (entry.isSymbolicLink()) { throw new Error( `unsafe dist entry: ${normalizeRelativePath(relative(packageRoot, join(currentDir, entry.name)))}`, @@ -255,7 +321,10 @@ function pruneEmptyDistDirectories(params = {}) { if (!entry.isDirectory()) { continue; } - prune(join(currentDir, entry.name)); + childDirs.push(join(currentDir, entry.name)); + } + for (const childDir of childDirs) { + prune(childDir); } if (currentDir === distRoot.distDir) { return; @@ -266,7 +335,7 @@ function pruneEmptyDistDirectories(params = {}) { `unsafe dist directory: ${normalizeRelativePath(relative(packageRoot, currentDir))}`, ); } - if (readDir(currentDir).length === 0) { + if (isDirectoryEmpty(currentDir)) { removeDirectory( assertSafeInstalledDistPath(normalizeRelativePath(relative(packageRoot, currentDir)), { packageRoot, @@ -285,45 +354,42 @@ function isLegacyInstalledPluginDependencyDirName(name) { } function pruneLegacyInstalledPluginDependencyDirs(params) { - const readDir = params.readdirSync ?? readdirSync; const removePath = params.rmSync ?? rmSync; const packageRoot = params.packageRoot ?? DEFAULT_PACKAGE_ROOT; const extensionsDir = join(packageRoot, "dist", "extensions"); + const budget = resolveInstalledDistScanBudget(params); const removed = []; - let pluginEntries; - try { - pluginEntries = readDir(extensionsDir, { withFileTypes: true }); - } catch { - return removed; - } - for (const pluginEntry of pluginEntries) { + for (const pluginEntry of iterateOptionalInstalledDistEntries(extensionsDir, params)) { + countInstalledDistScanEntry(budget); if (!pluginEntry.isDirectory() || pluginEntry.isSymbolicLink()) { continue; } const pluginDir = join(extensionsDir, pluginEntry.name); - let pluginChildren; - try { - pluginChildren = readDir(pluginDir, { withFileTypes: true }); - } catch { - continue; - } - for (const childEntry of pluginChildren) { + const dependencyDirNames = []; + for (const childEntry of iterateOptionalInstalledDistEntries(pluginDir, params)) { + countInstalledDistScanEntry(budget); if (!isLegacyInstalledPluginDependencyDirName(childEntry.name)) { continue; } - const safePluginDir = assertSafeInstalledDistPath( - normalizeRelativePath(relative(packageRoot, pluginDir)), - { - packageRoot, - distDirReal: params.distDirReal, - realpathSync: params.realpathSync, - }, - ); + dependencyDirNames.push(childEntry.name); + } + if (dependencyDirNames.length === 0) { + continue; + } + const safePluginDir = assertSafeInstalledDistPath( + normalizeRelativePath(relative(packageRoot, pluginDir)), + { + packageRoot, + distDirReal: params.distDirReal, + realpathSync: params.realpathSync, + }, + ); + for (const dependencyDirName of dependencyDirNames) { const relativePath = normalizeRelativePath( - relative(packageRoot, join(pluginDir, childEntry.name)), + relative(packageRoot, join(pluginDir, dependencyDirName)), ); - removePath(join(safePluginDir, childEntry.name), { recursive: true, force: true }); + removePath(join(safePluginDir, dependencyDirName), { recursive: true, force: true }); removed.push(relativePath); } } @@ -502,11 +568,13 @@ export function pruneInstalledPackageDist(params = {}) { if (distRoot === null) { return []; } + const distScanBudget = createInstalledDistScanBudget(params); + const distScanParams = { ...params, distScanBudget }; const removedLegacyDependencyDirs = pruneLegacyInstalledPluginDependencyDirs({ + ...distScanParams, packageRoot, distDirReal: distRoot.distDirReal, realpathSync: params.realpathSync, - readdirSync: params.readdirSync, rmSync: params.rmSync, }); let expectedFiles = params.expectedFiles ?? null; @@ -521,7 +589,7 @@ export function pruneInstalledPackageDist(params = {}) { return []; } } - const installedFiles = listInstalledDistFiles(params); + const installedFiles = listInstalledDistFiles(distScanParams); const readFile = params.readFileSync ?? readFileSync; expectedFiles = new Set( expandPackageDistImportClosure({ @@ -555,7 +623,7 @@ export function pruneInstalledPackageDist(params = {}) { removed.push(relativePath); } - pruneEmptyDistDirectories(params); + pruneEmptyDistDirectories(distScanParams); if (removed.length > 0) { log.log(`[postinstall] pruned stale dist files: ${removed.join(", ")}`); diff --git a/test/scripts/postinstall-bundled-plugins.test.ts b/test/scripts/postinstall-bundled-plugins.test.ts index 229ec0c50599..40727bb72a27 100644 --- a/test/scripts/postinstall-bundled-plugins.test.ts +++ b/test/scripts/postinstall-bundled-plugins.test.ts @@ -9,6 +9,7 @@ import { collectLegacyPluginRuntimeDepsStateRoots, isSourceCheckoutRoot, isDirectPostinstallInvocation, + MAX_INSTALLED_DIST_SCAN_ENTRIES, pruneOpenClawCompileCache, pruneInstalledPackageDist, pruneLegacyPluginRuntimeDepsState, @@ -953,6 +954,142 @@ describe("bundled plugin postinstall", () => { ).toThrow("unsafe dist entry: dist/escape"); }); + it("rejects packaged dist scans that exceed the filesystem entry limit", () => { + expect(() => + pruneInstalledPackageDist({ + packageRoot: "/pkg", + expectedFiles: new Set(), + existsSync: vi.fn(() => true), + lstatSync: vi.fn(() => ({ + isDirectory: () => true, + isSymbolicLink: () => false, + })), + maxDistScanEntries: 1, + realpathSync: vi.fn((filePath) => filePath), + readdirSync: vi.fn((filePath, options) => { + if (filePath === "/pkg/dist" && options?.withFileTypes) { + return [ + { + name: "first.js", + isDirectory: () => false, + isFile: () => true, + isSymbolicLink: () => false, + }, + { + name: "second.js", + isDirectory: () => false, + isFile: () => true, + isSymbolicLink: () => false, + }, + ]; + } + return []; + }), + rmSync: vi.fn(), + log: { log: vi.fn(), warn: vi.fn() }, + }), + ).toThrow( + "installed dist scan exceeded 1 filesystem entries; refusing to scan unbounded package contents", + ); + expect(MAX_INSTALLED_DIST_SCAN_ENTRIES).toBeGreaterThan(1); + }); + + it("uses one packaged dist scan budget across listing and pruning phases", () => { + expect(() => + pruneInstalledPackageDist({ + packageRoot: "/pkg", + expectedFiles: new Set(["dist/kept.js"]), + existsSync: vi.fn(() => true), + lstatSync: vi.fn(() => ({ + isDirectory: () => true, + isSymbolicLink: () => false, + })), + maxDistScanEntries: 1, + readFileSync: vi.fn(() => "export {};\n"), + realpathSync: vi.fn((filePath) => filePath), + readdirSync: vi.fn((filePath, options) => { + if (filePath === "/pkg/dist" && options?.withFileTypes) { + return [ + { + name: "kept.js", + isDirectory: () => false, + isFile: () => true, + isSymbolicLink: () => false, + }, + ]; + } + return []; + }), + rmSync: vi.fn(), + log: { log: vi.fn(), warn: vi.fn() }, + }), + ).toThrow( + "installed dist scan exceeded 1 filesystem entries; refusing to scan unbounded package contents", + ); + }); + + it("applies the packaged dist scan budget to legacy dependency debris prepass", () => { + expect(() => + pruneInstalledPackageDist({ + packageRoot: "/pkg", + expectedFiles: new Set(), + existsSync: vi.fn(() => true), + lstatSync: vi.fn(() => ({ + isDirectory: () => true, + isSymbolicLink: () => false, + })), + maxDistScanEntries: 1, + realpathSync: vi.fn((filePath) => filePath), + readdirSync: vi.fn((filePath, options) => { + if (filePath === "/pkg/dist/extensions" && options?.withFileTypes) { + return [ + { + name: "slack", + isDirectory: () => true, + isFile: () => false, + isSymbolicLink: () => false, + }, + ]; + } + if (filePath === "/pkg/dist/extensions/slack" && options?.withFileTypes) { + return [ + { + name: "node_modules", + isDirectory: () => true, + isFile: () => false, + isSymbolicLink: () => false, + }, + ]; + } + return []; + }), + rmSync: vi.fn(), + log: { log: vi.fn(), warn: vi.fn() }, + }), + ).toThrow( + "installed dist scan exceeded 1 filesystem entries; refusing to scan unbounded package contents", + ); + }); + + it("prunes sibling empty dist directories after closing parent scans", async () => { + const packageRoot = await createTempDirAsync("openclaw-packaged-install-empty-dirs-"); + const firstEmptyDir = path.join(packageRoot, "dist", "empty-a"); + const secondEmptyDir = path.join(packageRoot, "dist", "empty-b"); + await fs.mkdir(firstEmptyDir, { recursive: true }); + await fs.mkdir(secondEmptyDir, { recursive: true }); + + expect( + pruneInstalledPackageDist({ + packageRoot, + expectedFiles: new Set(), + log: { log: vi.fn(), warn: vi.fn() }, + }), + ).toEqual([]); + + await expectPathMissing(firstEmptyDir); + await expectPathMissing(secondEmptyDir); + }); + it("prunes stale bundled plugin dependency debris from packaged dist", async () => { const packageRoot = await createTempDirAsync("openclaw-packaged-install-dist-prune-"); const staleFile = path.join(packageRoot, "dist", "stale-runtime.js");