diff --git a/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs b/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs index e332fba0e002..4465bebf4bb9 100644 --- a/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs +++ b/scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs @@ -9,7 +9,7 @@ import { import { assertOpenAiEnvAuthProfileStore } from "../auth-profile-store-assertions.mjs"; import { readPositiveIntEnv } from "../env-limits.mjs"; import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs"; -import { readTextFileTail } from "../text-file-utils.mjs"; +import { readTextFileBounded, readTextFileTail } from "../text-file-utils.mjs"; const command = process.argv[2]; const ERROR_DETAIL_TAIL_BYTES = 16 * 1024; @@ -23,34 +23,12 @@ const STATUS_TEXT_MAX_BYTES = readPositiveIntEnv( ); const ansiEscapePattern = new RegExp(String.raw`\u001b\[[0-?]*[ -/]*[@-~]`, "g"); -function readTextFileBounded(file, label, maxBytes) { - const stat = fs.statSync(file); - if (!stat.isFile()) { - throw new Error(`${label} is not a file: ${file}`); - } - if (stat.size > maxBytes) { - throw new Error( - `${label} exceeded ${maxBytes} bytes: ${file} (${stat.size} bytes). Tail: ${readTextFileTail( - file, - ERROR_DETAIL_TAIL_BYTES, - )}`, - ); - } - const text = fs.readFileSync(file, "utf8"); - const bytes = Buffer.byteLength(text, "utf8"); - if (bytes > maxBytes) { - throw new Error( - `${label} exceeded ${maxBytes} bytes: ${file} (${bytes} bytes). Tail: ${readTextFileTail( - file, - ERROR_DETAIL_TAIL_BYTES, - )}`, - ); - } - return text; -} - function readJson(file) { - return JSON.parse(readTextFileBounded(file, "JSON artifact", JSON_ARTIFACT_MAX_BYTES)); + return JSON.parse( + readTextFileBounded(file, "JSON artifact", JSON_ARTIFACT_MAX_BYTES, { + tailBytes: ERROR_DETAIL_TAIL_BYTES, + }), + ); } function stripAnsi(text) { @@ -242,6 +220,7 @@ function assertStatusSurfaces() { statusTextPath, "plain status output", STATUS_TEXT_MAX_BYTES, + { tailBytes: ERROR_DETAIL_TAIL_BYTES }, ); const statusTail = readTextFileTail(statusTextPath, ERROR_DETAIL_TAIL_BYTES); const configuredChannels = Array.isArray(channelsStatus.configuredChannels) diff --git a/scripts/e2e/lib/plugin-index-sqlite.mjs b/scripts/e2e/lib/plugin-index-sqlite.mjs index 61ab765b94d9..066880da72d3 100644 --- a/scripts/e2e/lib/plugin-index-sqlite.mjs +++ b/scripts/e2e/lib/plugin-index-sqlite.mjs @@ -2,8 +2,15 @@ import fs from "node:fs"; import path from "node:path"; import { DatabaseSync } from "node:sqlite"; +import { readPositiveIntEnv } from "./env-limits.mjs"; +import { readTextFileBounded } from "./text-file-utils.mjs"; const INDEX_KEY = "installed-plugin-index"; +const ERROR_DETAIL_TAIL_BYTES = 16 * 1024; +const JSON_ARTIFACT_MAX_BYTES = readPositiveIntEnv( + "OPENCLAW_PLUGIN_INDEX_JSON_MAX_BYTES", + 1024 * 1024, +); export function stateDir() { return process.env.OPENCLAW_STATE_DIR || path.join(process.env.HOME, ".openclaw"); @@ -14,13 +21,46 @@ export function configPath() { } function readJsonMaybe(file) { + let text; try { - return JSON.parse(fs.readFileSync(file, "utf8")); + text = readTextFileBounded(file, "plugin index JSON artifact", JSON_ARTIFACT_MAX_BYTES, { + tailBytes: ERROR_DETAIL_TAIL_BYTES, + }); + } catch (error) { + if (error?.code === "ETOOBIG") { + throw error; + } + return {}; + } + try { + return JSON.parse(text); } catch { return {}; } } +function textTooLargeError(message) { + return Object.assign(new Error(message), { code: "ETOOBIG" }); +} + +function parseIndexJsonText(text, label) { + const bytes = Buffer.byteLength(text, "utf8"); + if (bytes > JSON_ARTIFACT_MAX_BYTES) { + throw textTooLargeError(`${label} exceeded ${JSON_ARTIFACT_MAX_BYTES} bytes (${bytes} bytes)`); + } + return JSON.parse(text); +} + +function assertIndexJsonByteLength(bytesRaw, label) { + const bytes = Number(bytesRaw); + if (!Number.isFinite(bytes) || bytes < 0) { + throw new Error(`${label} byte length was invalid: ${String(bytesRaw)}`); + } + if (bytes > JSON_ARTIFACT_MAX_BYTES) { + throw textTooLargeError(`${label} exceeded ${JSON_ARTIFACT_MAX_BYTES} bytes (${bytes} bytes)`); + } +} + function sqlitePath(root = stateDir()) { return path.join(root, "state", "openclaw.sqlite"); } @@ -37,6 +77,26 @@ function readSqlitePluginIndex(root = stateDir()) { let db; try { db = new DatabaseSync(dbPath, { readOnly: true }); + const lengths = db + .prepare( + ` + SELECT octet_length(install_records_json) AS install_records_json_bytes, + octet_length(plugins_json) AS plugins_json_bytes, + octet_length(diagnostics_json) AS diagnostics_json_bytes + FROM installed_plugin_index + WHERE index_key = ? + `, + ) + .get(INDEX_KEY); + if (!lengths) { + return {}; + } + assertIndexJsonByteLength( + lengths.install_records_json_bytes, + "plugin index install_records_json", + ); + assertIndexJsonByteLength(lengths.plugins_json_bytes, "plugin index plugins_json"); + assertIndexJsonByteLength(lengths.diagnostics_json_bytes, "plugin index diagnostics_json"); const row = db .prepare( ` @@ -60,11 +120,17 @@ function readSqlitePluginIndex(root = stateDir()) { policyHash: row.policy_hash, generatedAtMs: Number(row.generated_at_ms), ...(row.refresh_reason ? { refreshReason: row.refresh_reason } : {}), - installRecords: JSON.parse(row.install_records_json), - plugins: JSON.parse(row.plugins_json), - diagnostics: JSON.parse(row.diagnostics_json), + installRecords: parseIndexJsonText( + row.install_records_json, + "plugin index install_records_json", + ), + plugins: parseIndexJsonText(row.plugins_json, "plugin index plugins_json"), + diagnostics: parseIndexJsonText(row.diagnostics_json, "plugin index diagnostics_json"), }; - } catch { + } catch (error) { + if (error?.code === "ETOOBIG") { + throw error; + } return {}; } finally { db?.close(); diff --git a/scripts/e2e/lib/text-file-utils.mjs b/scripts/e2e/lib/text-file-utils.mjs index 8eb70210f59c..0799b21b6819 100644 --- a/scripts/e2e/lib/text-file-utils.mjs +++ b/scripts/e2e/lib/text-file-utils.mjs @@ -21,12 +21,52 @@ export function readTextFileTail(file, maxBytes) { const length = Math.min(maxBytes, stat.size); const start = stat.size - length; - const fd = fs.openSync(file, "r"); + let fd; try { + fd = fs.openSync(file, "r"); const buffer = Buffer.alloc(length); const bytesRead = fs.readSync(fd, buffer, 0, length, start); return buffer.subarray(0, bytesRead).toString("utf8"); + } catch { + return ""; } finally { - fs.closeSync(fd); + if (fd !== undefined) { + try { + fs.closeSync(fd); + } catch { + // Tail diagnostics are best-effort; callers may be preserving a richer error. + } + } } } + +function textFileTooLargeError(message) { + return Object.assign(new Error(message), { code: "ETOOBIG" }); +} + +export function readTextFileBounded(file, label, maxBytes, options = {}) { + const tailBytes = options.tailBytes ?? 16 * 1024; + const stat = fs.statSync(file); + if (!stat.isFile()) { + throw new Error(`${label} is not a file: ${file}`); + } + if (stat.size > maxBytes) { + throw textFileTooLargeError( + `${label} exceeded ${maxBytes} bytes: ${file} (${stat.size} bytes). Tail: ${readTextFileTail( + file, + tailBytes, + )}`, + ); + } + const text = fs.readFileSync(file, "utf8"); + const bytes = Buffer.byteLength(text, "utf8"); + if (bytes > maxBytes) { + throw textFileTooLargeError( + `${label} exceeded ${maxBytes} bytes: ${file} (${bytes} bytes). Tail: ${readTextFileTail( + file, + tailBytes, + )}`, + ); + } + return text; +} diff --git a/test/scripts/plugin-index-sqlite.test.ts b/test/scripts/plugin-index-sqlite.test.ts new file mode 100644 index 000000000000..4d29c9404b63 --- /dev/null +++ b/test/scripts/plugin-index-sqlite.test.ts @@ -0,0 +1,156 @@ +// Plugin Index SQLite tests cover shared E2E install-index readers. +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { pathToFileURL } from "node:url"; +import { describe, expect, it } from "vitest"; + +const MODULE_URL = pathToFileURL(path.resolve("scripts/e2e/lib/plugin-index-sqlite.mjs")).href; +let importCounter = 0; + +async function loadPluginIndex(env: Record = {}) { + const previous = new Map(Object.keys(env).map((key) => [key, process.env[key]])); + Object.assign(process.env, env); + try { + return await import(`${MODULE_URL}?case=${importCounter++}`); + } finally { + for (const [key, value] of previous) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + } +} + +function writeLegacyIndex(root: string, text: string) { + const file = path.join(root, "plugins", "installs.json"); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, text, "utf8"); +} + +function configPath(root: string) { + return path.join(root, "openclaw.json"); +} + +function writeSqliteIndex(root: string, installRecordsJson: string) { + const dbPath = path.join(root, "state", "openclaw.sqlite"); + mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new DatabaseSync(dbPath); + try { + db.exec(` + CREATE TABLE installed_plugin_index ( + index_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + host_contract_version TEXT NOT NULL, + compat_registry_version TEXT NOT NULL, + migration_version INTEGER NOT NULL, + policy_hash TEXT NOT NULL, + generated_at_ms INTEGER NOT NULL, + refresh_reason TEXT, + install_records_json TEXT NOT NULL, + plugins_json TEXT NOT NULL, + diagnostics_json TEXT NOT NULL, + warning TEXT, + updated_at_ms INTEGER NOT NULL + ); + `); + db.prepare( + ` + INSERT INTO installed_plugin_index ( + index_key, version, host_contract_version, compat_registry_version, + migration_version, policy_hash, generated_at_ms, refresh_reason, + install_records_json, plugins_json, diagnostics_json, warning, updated_at_ms + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ).run( + "installed-plugin-index", + 1, + "1", + "1", + 1, + "hash", + Date.now(), + null, + installRecordsJson, + "{}", + "{}", + null, + Date.now(), + ); + } finally { + db.close(); + } +} + +describe("plugin index SQLite E2E helpers", () => { + it("reads legacy install records when SQLite index state is absent", async () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-")); + try { + writeLegacyIndex( + root, + JSON.stringify({ records: { demo: { installPath: "/tmp/demo", source: "npm" } } }), + ); + + const { readPluginInstallRecords } = await loadPluginIndex(); + + expect(readPluginInstallRecords({ stateDir: root, configPath: configPath(root) })).toEqual({ + demo: { installPath: "/tmp/demo", source: "npm" }, + }); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("keeps malformed legacy install JSON as an empty fallback", async () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-")); + try { + writeLegacyIndex(root, "{not-json"); + + const { readPluginInstallRecords } = await loadPluginIndex(); + + expect(readPluginInstallRecords({ stateDir: root, configPath: configPath(root) })).toEqual( + {}, + ); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("rejects oversized legacy install JSON before parsing it", async () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-")); + try { + writeLegacyIndex(root, JSON.stringify({ records: {}, filler: "x".repeat(128) })); + + const { readPluginInstallRecords } = await loadPluginIndex({ + OPENCLAW_PLUGIN_INDEX_JSON_MAX_BYTES: "64", + }); + + expect(() => + readPluginInstallRecords({ stateDir: root, configPath: configPath(root) }), + ).toThrow("plugin index JSON artifact exceeded 64 bytes"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("rejects oversized SQLite install index JSON before parsing it", async () => { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-")); + try { + writeSqliteIndex(root, JSON.stringify({ filler: "x".repeat(128) })); + + const { readPluginInstallIndex } = await loadPluginIndex({ + OPENCLAW_PLUGIN_INDEX_JSON_MAX_BYTES: "64", + }); + + expect(() => + readPluginInstallIndex({ stateDir: root, configPath: configPath(root) }), + ).toThrow("plugin index install_records_json exceeded 64 bytes"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); +});