From 820d167d21d541eaa6e119bc7960714fc3d6331d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 26 Aug 2026 14:33:01 -0700 Subject: [PATCH] fix: allow plugin state values up to 1 MiB (#130387) * fix: allow plugin state values up to 1 MiB * test: type-check plugin state write rejection cases * fix(ci): keep tar archive paths local on Windows --- docs/plugins/sdk-runtime.md | 2 +- scripts/package-openclaw-for-docker.mts | 40 ++++++++++----- .../plugin-state-store.e2e.test.ts | 51 +++++++++++-------- src/plugin-state/plugin-state-store.sqlite.ts | 2 +- src/plugin-state/plugin-state-store.test.ts | 5 +- 5 files changed, 65 insertions(+), 35 deletions(-) diff --git a/docs/plugins/sdk-runtime.md b/docs/plugins/sdk-runtime.md index 6065dff4da39..caf6da97657d 100644 --- a/docs/plugins/sdk-runtime.md +++ b/docs/plugins/sdk-runtime.md @@ -991,7 +991,7 @@ snapshots; OpenClaw owns all persistence and lifecycle coordination. const blob = await blobs.lookup("artifact-1"); ``` - Keyed stores survive restarts and are isolated by the runtime-bound plugin id. Use `registerIfAbsent(...)` for atomic dedupe claims: it returns `true` when the key was missing or expired and registered, or `false` when a live value already exists without overwriting its value, creation time, or TTL. Use `deleteIf(...)` when cleanup must remove only the value previously observed; its synchronous predicate and deletion run in one SQLite transaction. Limits: `maxEntries` per namespace, 50,000 live rows per plugin, JSON values under 64KB, and optional TTL expiry. By default, a write at either row limit sheds the oldest live rows from the namespace being written; sibling namespaces are not evicted for that write, and the write still fails if the namespace cannot free enough rows. Set `overflowPolicy: "reject-new"` for durable ownership records that must never be evicted: new keys fail at either limit, while existing keys remain updateable. + Keyed stores survive restarts and are isolated by the runtime-bound plugin id. Use `registerIfAbsent(...)` for atomic dedupe claims: it returns `true` when the key was missing or expired and registered, or `false` when a live value already exists without overwriting its value, creation time, or TTL. Use `deleteIf(...)` when cleanup must remove only the value previously observed; its synchronous predicate and deletion run in one SQLite transaction. Limits: `maxEntries` per namespace, 50,000 live rows per plugin, JSON values up to 1 MiB of UTF-8 encoded JSON, and optional TTL expiry. By default, a write at either row limit sheds the oldest live rows from the namespace being written; sibling namespaces are not evicted for that write, and the write still fails if the namespace cannot free enough rows. Set `overflowPolicy: "reject-new"` for durable ownership records that must never be evicted: new keys fail at either limit, while existing keys remain updateable. `openSyncKeyedStore(...)` returns the same store shape with synchronous methods (`register`, `registerIfAbsent`, `deleteIf`, `lookup`, `consume`, `clear` all return values directly instead of promises) for callers that cannot await. diff --git a/scripts/package-openclaw-for-docker.mts b/scripts/package-openclaw-for-docker.mts index 01f3fcf9b536..1f09804ecee5 100644 --- a/scripts/package-openclaw-for-docker.mts +++ b/scripts/package-openclaw-for-docker.mts @@ -641,12 +641,18 @@ export async function prepareBundledAiRuntimePackage( ((tarballPath: string, destination: string) => // Source-ref validation runs this trusted harness outside the candidate's dependency tree. // Keep extraction on the system tar contract so only the candidate checkout needs install. - run("tar", ["-xzf", tarballPath, "-C", destination, "--strip-components=1"], destination, { - timeoutMs: resolveTimeoutMs( - "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", - DEFAULT_PACKAGE_PACK_TIMEOUT_MS, - ), - })); + // Use an archive basename so GNU tar cannot treat a Windows drive as a remote host. + run( + "tar", + ["-xzf", path.basename(tarballPath), "-C", destination, "--strip-components=1"], + path.dirname(tarballPath), + { + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }, + )); const prepareManifest = packageOptions.prepareManifest ?? (async () => false); const restoreManifest = packageOptions.restoreManifest ?? (async () => false); const originalPackageJson = await fs.readFile(packageJsonPath, "utf8"); @@ -831,7 +837,12 @@ async function normalizeOpenClawTarballModes(tarballPath: string) { ); const stageDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-package-modes-")); try { - await run("tar", ["-xzf", tarballPath, "-C", stageDir], stageDir, { timeoutMs }); + await run( + "tar", + ["-xzf", path.basename(tarballPath), "-C", stageDir], + path.dirname(tarballPath), + { timeoutMs }, + ); let stagedFileCount = 0; const normalizeStagedModes = async (dir: string): Promise => { for (const entry of await fs.readdir(dir, { withFileTypes: true })) { @@ -855,11 +866,16 @@ async function normalizeOpenClawTarballModes(tarballPath: string) { const stageRootEntries = await fs.readdir(stageDir); const normalizedPath = `${tarballPath}.modes-tmp`; await fs.rm(normalizedPath, { force: true }); - await run("tar", ["-czf", normalizedPath, "-C", stageDir, ...stageRootEntries], stageDir, { - // macOS bsdtar must not add AppleDouble (._*) sidecar entries. - env: { ...process.env, COPYFILE_DISABLE: "1" }, - timeoutMs, - }); + await run( + "tar", + ["-czf", path.basename(normalizedPath), "-C", stageDir, ...stageRootEntries], + path.dirname(normalizedPath), + { + // macOS bsdtar must not add AppleDouble (._*) sidecar entries. + env: { ...process.env, COPYFILE_DISABLE: "1" }, + timeoutMs, + }, + ); await fs.rename(normalizedPath, tarballPath); } finally { await fs.rm(stageDir, { force: true, recursive: true }); diff --git a/src/plugin-state/plugin-state-store.e2e.test.ts b/src/plugin-state/plugin-state-store.e2e.test.ts index 98cdea841b40..504c334dd475 100644 --- a/src/plugin-state/plugin-state-store.e2e.test.ts +++ b/src/plugin-state/plugin-state-store.e2e.test.ts @@ -6,6 +6,7 @@ import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { closePluginStateDatabase, createPluginStateKeyedStore, + createPluginStateSyncKeyedStore, resetPluginStateStoreForTests, sweepExpiredPluginStateEntries, } from "./plugin-state-store.js"; @@ -149,31 +150,41 @@ describe("isolation", () => { // Limits // --------------------------------------------------------------------------- describe("limits", () => { - it("accepts a value at the 64 KB boundary", async () => { - await withOpenClawTestState({ label: "e2e-limit-accept" }, async () => { - const store = createPluginStateKeyedStore("fixture-plugin", { + it.each(["async", "sync"])("enforces the 1 MiB boundary across %s writes", async (mode) => { + await withOpenClawTestState({ label: "e2e-limit" }, async () => { + const createStore = + mode === "async" + ? createPluginStateKeyedStore + : createPluginStateSyncKeyedStore; + const store = createStore("fixture-plugin", { namespace: "size", maxEntries: 10, }); // JSON.stringify wraps a string in quotes (+2 bytes). - // 65 534 chars → 65 536 bytes of JSON → exactly at limit. - const boundary = "x".repeat(65_534); - await expect(store.register("big", boundary)).resolves.toBeUndefined(); - await expect(store.lookup("big")).resolves.toBe(boundary); - }); - }); + const boundary = "x".repeat(1_048_574); + const oversize = `${boundary}x`; + const update = expectDefined(store.update, "keyed store update support"); + await store.register("registered", boundary); + expect(await store.registerIfAbsent("claimed", boundary)).toBe(true); + await store.register("updated", "before"); + expect(await update("updated", () => boundary)).toBe(true); - it("rejects a value one byte over 64 KB", async () => { - await withOpenClawTestState({ label: "e2e-limit-reject" }, async () => { - const store = createPluginStateKeyedStore("fixture-plugin", { - namespace: "size", - maxEntries: 10, - }); - // 65 535 chars → 65 537 bytes of JSON → over limit. - const oversize = "x".repeat(65_535); - await expect(store.register("big", oversize)).rejects.toMatchObject({ - code: "PLUGIN_STATE_LIMIT_EXCEEDED", - }); + for (const write of [ + () => store.register("registered", oversize), + () => store.registerIfAbsent("rejected", oversize), + () => update("updated", () => oversize), + ]) { + await expect(async () => { + await write(); + }).rejects.toMatchObject({ + code: "PLUGIN_STATE_LIMIT_EXCEEDED", + }); + } + resetPluginStateStoreForTests(); + for (const key of ["registered", "claimed", "updated"]) { + expect(await store.lookup(key)).toBe(boundary); + } + expect(await store.lookup("rejected")).toBeUndefined(); }); }); }); diff --git a/src/plugin-state/plugin-state-store.sqlite.ts b/src/plugin-state/plugin-state-store.sqlite.ts index edbedf7f0db8..626c4cd3ae4c 100644 --- a/src/plugin-state/plugin-state-store.sqlite.ts +++ b/src/plugin-state/plugin-state-store.sqlite.ts @@ -30,7 +30,7 @@ import { } from "./plugin-state-store.types.js"; // Plugin-wide fuse only; namespace maxEntries still owns normal cache eviction. -export const MAX_PLUGIN_STATE_VALUE_BYTES = 65_536; +export const MAX_PLUGIN_STATE_VALUE_BYTES = 1_048_576; export const MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN = 50_000; const PLUGIN_STATE_EXPIRY_BATCH_ROWS = 1_024; let maxPluginStateEntriesPerPluginForTests: number | undefined; diff --git a/src/plugin-state/plugin-state-store.test.ts b/src/plugin-state/plugin-state-store.test.ts index 257094e19d66..86b1567e0a07 100644 --- a/src/plugin-state/plugin-state-store.test.ts +++ b/src/plugin-state/plugin-state-store.test.ts @@ -810,7 +810,10 @@ describe("plugin state keyed store", () => { await expect(store.register("non-enumerable", nonEnumerable)).rejects.toThrow( PluginStateStoreError, ); - await expectPluginStateStoreError(store.register("big", "x".repeat(65_537)), { + // UTF-8 bytes, including JSON quotes, determine the 1 MiB boundary. + const boundary = "é".repeat(524_287); + await expect(store.register("large", boundary)).resolves.toBeUndefined(); + await expectPluginStateStoreError(store.register("big", `${boundary}x`), { code: "PLUGIN_STATE_LIMIT_EXCEEDED", });