fix(scripts): bound plugin install index artifacts

2026-08-12 21:53:00 -06:00 · 2026-06-16 06:42:36 +02:00
parent 88c9e4d644
commit 484ee14273
4 changed files with 276 additions and 35 deletions
@@ -9,7 +9,7 @@ import {
 import { assertOpenAiEnvAuthProfileStore } from "../auth-profile-store-assertions.mjs";
 import { readPositiveIntEnv } from "../env-limits.mjs";
 import { applyMockOpenAiModelConfig } from "../fixtures/mock-openai-config.mjs";
-import { readTextFileTail } from "../text-file-utils.mjs";
+import { readTextFileBounded, readTextFileTail } from "../text-file-utils.mjs";

 const command = process.argv[2];
 const ERROR_DETAIL_TAIL_BYTES = 16 * 1024;
@@ -23,34 +23,12 @@ const STATUS_TEXT_MAX_BYTES = readPositiveIntEnv(
 );
 const ansiEscapePattern = new RegExp(String.raw`\u001b\[[0-?]*[ -/]*[@-~]`, "g");

-function readTextFileBounded(file, label, maxBytes) {
-  const stat = fs.statSync(file);
-  if (!stat.isFile()) {
-    throw new Error(`${label} is not a file: ${file}`);
-  }
-  if (stat.size > maxBytes) {
-    throw new Error(
-      `${label} exceeded ${maxBytes} bytes: ${file} (${stat.size} bytes). Tail: ${readTextFileTail(
-        file,
-        ERROR_DETAIL_TAIL_BYTES,
-      )}`,
-    );
-  }
-  const text = fs.readFileSync(file, "utf8");
-  const bytes = Buffer.byteLength(text, "utf8");
-  if (bytes > maxBytes) {
-    throw new Error(
-      `${label} exceeded ${maxBytes} bytes: ${file} (${bytes} bytes). Tail: ${readTextFileTail(
-        file,
-        ERROR_DETAIL_TAIL_BYTES,
-      )}`,
-    );
-  }
-  return text;
-}
-
 function readJson(file) {
-  return JSON.parse(readTextFileBounded(file, "JSON artifact", JSON_ARTIFACT_MAX_BYTES));
+  return JSON.parse(
+    readTextFileBounded(file, "JSON artifact", JSON_ARTIFACT_MAX_BYTES, {
+      tailBytes: ERROR_DETAIL_TAIL_BYTES,
+    }),
+  );
 }

 function stripAnsi(text) {
@@ -242,6 +220,7 @@ function assertStatusSurfaces() {
    statusTextPath,
    "plain status output",
    STATUS_TEXT_MAX_BYTES,
+    { tailBytes: ERROR_DETAIL_TAIL_BYTES },
  );
  const statusTail = readTextFileTail(statusTextPath, ERROR_DETAIL_TAIL_BYTES);
  const configuredChannels = Array.isArray(channelsStatus.configuredChannels)
@@ -2,8 +2,15 @@
 import fs from "node:fs";
 import path from "node:path";
 import { DatabaseSync } from "node:sqlite";
+import { readPositiveIntEnv } from "./env-limits.mjs";
+import { readTextFileBounded } from "./text-file-utils.mjs";

 const INDEX_KEY = "installed-plugin-index";
+const ERROR_DETAIL_TAIL_BYTES = 16 * 1024;
+const JSON_ARTIFACT_MAX_BYTES = readPositiveIntEnv(
+  "OPENCLAW_PLUGIN_INDEX_JSON_MAX_BYTES",
+  1024 * 1024,
+);

 export function stateDir() {
  return process.env.OPENCLAW_STATE_DIR || path.join(process.env.HOME, ".openclaw");
@@ -14,13 +21,46 @@ export function configPath() {
 }

 function readJsonMaybe(file) {
+  let text;
  try {
-    return JSON.parse(fs.readFileSync(file, "utf8"));
+    text = readTextFileBounded(file, "plugin index JSON artifact", JSON_ARTIFACT_MAX_BYTES, {
+      tailBytes: ERROR_DETAIL_TAIL_BYTES,
+    });
+  } catch (error) {
+    if (error?.code === "ETOOBIG") {
+      throw error;
+    }
+    return {};
+  }
+  try {
+    return JSON.parse(text);
  } catch {
    return {};
  }
 }

+function textTooLargeError(message) {
+  return Object.assign(new Error(message), { code: "ETOOBIG" });
+}
+
+function parseIndexJsonText(text, label) {
+  const bytes = Buffer.byteLength(text, "utf8");
+  if (bytes > JSON_ARTIFACT_MAX_BYTES) {
+    throw textTooLargeError(`${label} exceeded ${JSON_ARTIFACT_MAX_BYTES} bytes (${bytes} bytes)`);
+  }
+  return JSON.parse(text);
+}
+
+function assertIndexJsonByteLength(bytesRaw, label) {
+  const bytes = Number(bytesRaw);
+  if (!Number.isFinite(bytes) || bytes < 0) {
+    throw new Error(`${label} byte length was invalid: ${String(bytesRaw)}`);
+  }
+  if (bytes > JSON_ARTIFACT_MAX_BYTES) {
+    throw textTooLargeError(`${label} exceeded ${JSON_ARTIFACT_MAX_BYTES} bytes (${bytes} bytes)`);
+  }
+}
+
 function sqlitePath(root = stateDir()) {
  return path.join(root, "state", "openclaw.sqlite");
 }
@@ -37,6 +77,26 @@ function readSqlitePluginIndex(root = stateDir()) {
  let db;
  try {
    db = new DatabaseSync(dbPath, { readOnly: true });
+    const lengths = db
+      .prepare(
+        `
+          SELECT octet_length(install_records_json) AS install_records_json_bytes,
+                 octet_length(plugins_json) AS plugins_json_bytes,
+                 octet_length(diagnostics_json) AS diagnostics_json_bytes
+            FROM installed_plugin_index
+           WHERE index_key = ?
+        `,
+      )
+      .get(INDEX_KEY);
+    if (!lengths) {
+      return {};
+    }
+    assertIndexJsonByteLength(
+      lengths.install_records_json_bytes,
+      "plugin index install_records_json",
+    );
+    assertIndexJsonByteLength(lengths.plugins_json_bytes, "plugin index plugins_json");
+    assertIndexJsonByteLength(lengths.diagnostics_json_bytes, "plugin index diagnostics_json");
    const row = db
      .prepare(
        `
@@ -60,11 +120,17 @@ function readSqlitePluginIndex(root = stateDir()) {
      policyHash: row.policy_hash,
      generatedAtMs: Number(row.generated_at_ms),
      ...(row.refresh_reason ? { refreshReason: row.refresh_reason } : {}),
-      installRecords: JSON.parse(row.install_records_json),
-      plugins: JSON.parse(row.plugins_json),
-      diagnostics: JSON.parse(row.diagnostics_json),
+      installRecords: parseIndexJsonText(
+        row.install_records_json,
+        "plugin index install_records_json",
+      ),
+      plugins: parseIndexJsonText(row.plugins_json, "plugin index plugins_json"),
+      diagnostics: parseIndexJsonText(row.diagnostics_json, "plugin index diagnostics_json"),
    };
-  } catch {
+  } catch (error) {
+    if (error?.code === "ETOOBIG") {
+      throw error;
+    }
    return {};
  } finally {
    db?.close();
@@ -21,12 +21,52 @@ export function readTextFileTail(file, maxBytes) {

  const length = Math.min(maxBytes, stat.size);
  const start = stat.size - length;
-  const fd = fs.openSync(file, "r");
+  let fd;
  try {
+    fd = fs.openSync(file, "r");
    const buffer = Buffer.alloc(length);
    const bytesRead = fs.readSync(fd, buffer, 0, length, start);
    return buffer.subarray(0, bytesRead).toString("utf8");
+  } catch {
+    return "";
  } finally {
-    fs.closeSync(fd);
+    if (fd !== undefined) {
+      try {
+        fs.closeSync(fd);
+      } catch {
+        // Tail diagnostics are best-effort; callers may be preserving a richer error.
+      }
+    }
  }
 }
+
+function textFileTooLargeError(message) {
+  return Object.assign(new Error(message), { code: "ETOOBIG" });
+}
+
+export function readTextFileBounded(file, label, maxBytes, options = {}) {
+  const tailBytes = options.tailBytes ?? 16 * 1024;
+  const stat = fs.statSync(file);
+  if (!stat.isFile()) {
+    throw new Error(`${label} is not a file: ${file}`);
+  }
+  if (stat.size > maxBytes) {
+    throw textFileTooLargeError(
+      `${label} exceeded ${maxBytes} bytes: ${file} (${stat.size} bytes). Tail: ${readTextFileTail(
+        file,
+        tailBytes,
+      )}`,
+    );
+  }
+  const text = fs.readFileSync(file, "utf8");
+  const bytes = Buffer.byteLength(text, "utf8");
+  if (bytes > maxBytes) {
+    throw textFileTooLargeError(
+      `${label} exceeded ${maxBytes} bytes: ${file} (${bytes} bytes). Tail: ${readTextFileTail(
+        file,
+        tailBytes,
+      )}`,
+    );
+  }
+  return text;
+}
@@ -0,0 +1,156 @@
+// Plugin Index SQLite tests cover shared E2E install-index readers.
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { pathToFileURL } from "node:url";
+import { describe, expect, it } from "vitest";
+
+const MODULE_URL = pathToFileURL(path.resolve("scripts/e2e/lib/plugin-index-sqlite.mjs")).href;
+let importCounter = 0;
+
+async function loadPluginIndex(env: Record<string, string> = {}) {
+  const previous = new Map(Object.keys(env).map((key) => [key, process.env[key]]));
+  Object.assign(process.env, env);
+  try {
+    return await import(`${MODULE_URL}?case=${importCounter++}`);
+  } finally {
+    for (const [key, value] of previous) {
+      if (value === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = value;
+      }
+    }
+  }
+}
+
+function writeLegacyIndex(root: string, text: string) {
+  const file = path.join(root, "plugins", "installs.json");
+  mkdirSync(path.dirname(file), { recursive: true });
+  writeFileSync(file, text, "utf8");
+}
+
+function configPath(root: string) {
+  return path.join(root, "openclaw.json");
+}
+
+function writeSqliteIndex(root: string, installRecordsJson: string) {
+  const dbPath = path.join(root, "state", "openclaw.sqlite");
+  mkdirSync(path.dirname(dbPath), { recursive: true });
+  const db = new DatabaseSync(dbPath);
+  try {
+    db.exec(`
+      CREATE TABLE installed_plugin_index (
+        index_key TEXT NOT NULL PRIMARY KEY,
+        version INTEGER NOT NULL,
+        host_contract_version TEXT NOT NULL,
+        compat_registry_version TEXT NOT NULL,
+        migration_version INTEGER NOT NULL,
+        policy_hash TEXT NOT NULL,
+        generated_at_ms INTEGER NOT NULL,
+        refresh_reason TEXT,
+        install_records_json TEXT NOT NULL,
+        plugins_json TEXT NOT NULL,
+        diagnostics_json TEXT NOT NULL,
+        warning TEXT,
+        updated_at_ms INTEGER NOT NULL
+      );
+    `);
+    db.prepare(
+      `
+        INSERT INTO installed_plugin_index (
+          index_key, version, host_contract_version, compat_registry_version,
+          migration_version, policy_hash, generated_at_ms, refresh_reason,
+          install_records_json, plugins_json, diagnostics_json, warning, updated_at_ms
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `,
+    ).run(
+      "installed-plugin-index",
+      1,
+      "1",
+      "1",
+      1,
+      "hash",
+      Date.now(),
+      null,
+      installRecordsJson,
+      "{}",
+      "{}",
+      null,
+      Date.now(),
+    );
+  } finally {
+    db.close();
+  }
+}
+
+describe("plugin index SQLite E2E helpers", () => {
+  it("reads legacy install records when SQLite index state is absent", async () => {
+    const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-"));
+    try {
+      writeLegacyIndex(
+        root,
+        JSON.stringify({ records: { demo: { installPath: "/tmp/demo", source: "npm" } } }),
+      );
+
+      const { readPluginInstallRecords } = await loadPluginIndex();
+
+      expect(readPluginInstallRecords({ stateDir: root, configPath: configPath(root) })).toEqual({
+        demo: { installPath: "/tmp/demo", source: "npm" },
+      });
+    } finally {
+      rmSync(root, { force: true, recursive: true });
+    }
+  });
+
+  it("keeps malformed legacy install JSON as an empty fallback", async () => {
+    const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-"));
+    try {
+      writeLegacyIndex(root, "{not-json");
+
+      const { readPluginInstallRecords } = await loadPluginIndex();
+
+      expect(readPluginInstallRecords({ stateDir: root, configPath: configPath(root) })).toEqual(
+        {},
+      );
+    } finally {
+      rmSync(root, { force: true, recursive: true });
+    }
+  });
+
+  it("rejects oversized legacy install JSON before parsing it", async () => {
+    const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-"));
+    try {
+      writeLegacyIndex(root, JSON.stringify({ records: {}, filler: "x".repeat(128) }));
+
+      const { readPluginInstallRecords } = await loadPluginIndex({
+        OPENCLAW_PLUGIN_INDEX_JSON_MAX_BYTES: "64",
+      });
+
+      expect(() =>
+        readPluginInstallRecords({ stateDir: root, configPath: configPath(root) }),
+      ).toThrow("plugin index JSON artifact exceeded 64 bytes");
+    } finally {
+      rmSync(root, { force: true, recursive: true });
+    }
+  });
+
+  it("rejects oversized SQLite install index JSON before parsing it", async () => {
+    const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-index-"));
+    try {
+      writeSqliteIndex(root, JSON.stringify({ filler: "x".repeat(128) }));
+
+      const { readPluginInstallIndex } = await loadPluginIndex({
+        OPENCLAW_PLUGIN_INDEX_JSON_MAX_BYTES: "64",
+      });
+
+      expect(() =>
+        readPluginInstallIndex({ stateDir: root, configPath: configPath(root) }),
+      ).toThrow("plugin index install_records_json exceeded 64 bytes");
+    } finally {
+      rmSync(root, { force: true, recursive: true });
+    }
+  });
+});