ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(fs): adopt fs-safe 0.5 core primitives (#113705)

Browse Source 
* refactor(fs): unify exclusive file publication

* fix(fs): fence stale lock reclamation

* refactor(fs): bound wiki scans and secret reads

* chore(fs): finalize fs-safe 0.5 compatibility

* fix(fs): preserve publication ownership and legacy mode

* fix(fs): fail closed on unverifiable lock owners

* fix(fs): preserve concurrent backup publications

* refactor(fs): preserve ambiguous backup outputs

* fix(fs): preserve mixed-version lock coordination

* refactor(file-transfer): adopt fs-safe archive extraction

* refactor(fs): add bounded walk and secret seams

* refactor(auth): replace proper-lockfile with fs-safe

* fix(fs): honor Windows mode override casing

* refactor(snapshot): adopt fs-safe publication

* refactor(memory-wiki): adopt prunable root walks

* refactor(fleet): adopt bounded archive restore

* fix(fs): preserve post-publication ownership receipts

* refactor(fs): harvest final fs-safe primitives

* style(fs): clean harvest lint

* chore(plugin-sdk): refresh move helper API baseline

* refactor(snapshot): adopt native Windows ACL facts

* refactor(fs): adopt hardened atomic outputs

* fix(fs): scope lock reentrancy to logical owners

* chore(config): lower env var count budget

* fix(deps): adopt published fs-safe 0.5.0

* fix(ci): align SDK surface ratchets

* fix(ci): regenerate SDK API baseline after rebase

* fix(fs): preserve owner-scoped file lock nesting

* fix(ci): refresh SDK API baseline for file locks

* fix(fs): separate SQLite and file lock reentrancy

* fix(imessage): bound pinned attachment reads

* fix(agents): narrow session-key lock options

* fix(fs): preserve fs-safe 0.5 compatibility contracts

* fix(windows): retain private SQLite directory owner

* refactor(sqlite): centralize exclusive coordinator

* refactor(snapshot): isolate Windows ACL policy

* fix(windows): retain snapshot ACL inspector

* chore(config): realign env budget after rebase

* test(agents): accept canonical sandbox escape error

* docs(changelog): defer fs-safe release note
This commit is contained in:
Peter Steinberger
2026-07-28 03:41:47 -04:00
committed by GitHub
parent 78d6c6c047
commit 0d7fb8eb39
89 changed files with 2058 additions and 1738 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/env-var-count-budget.txt
+1 -1
View File
@@ -1,3 +1,3 @@
# Distinct OPENCLAW_* names in production source under src, packages, and extensions.
# Ratchet: lower this number when cleanup removes names; never raise it.
520
519
docs/.generated/plugin-sdk-api-baseline.sha256
+8 -5
View File
@@ -4,7 +4,7 @@ cbf4e2c3088f8886a7c9ea91325a66e0f0846cea21f0b2891f36399b4811306c module/account
8e985f345f21a1c9a2b0e94304aaaad6a326bec1c1ce3b26027d2862804a366e module/account-resolution
e5e67ddf3cab38fcbf9220bc3160715897e2709d9a9ff6ff36f1ecc9453c2367 module/agent-config-primitives
74daa746deb548379d3f0d6eac3c4d082df1034c4360cc03bf51fee0f10a2e4d module/agent-harness
23461d43c31d8c143e9d45aa91e27a22938d3549093d3d637b3bb88cf400c8b8 module/agent-harness-runtime
5de3d5c7eb5b863c12453666725cfe6473d884a1788ad905fbddda64983ef100 module/agent-harness-runtime
5168648cd946abad8a92822889f13ceacc87ed502314a66190d0b1eb8ebe76ea module/agent-media-payload
2dcb4d62d90e5d71594f6b843c97534509a154784e378fb1c75bd86b5122b710 module/agent-runtime
56b6d5fb6af3d95af1200065aca2e7d4f59e5fa59740505fe6ff433077ef6646 module/allow-from
@@ -18,6 +18,7 @@ a453261fc6c0d45ad652992bfa3139a1295b6e6296878ee5fde2cf12b995516e module/approva
dad0906b607cfc044805279c25f656b65579b92b458c8a7c248b7cc8e297d99b module/approval-native-runtime
22db78ca5d8bb308d4ea35fdf0027d6bb6b7ceff8b5737e5bbd651b252d87b5f module/approval-reply-runtime
4c2ae57c01ec97f142666aba116d9469192f48b0eba8bc848ec985f1525698f2 module/approval-runtime
01ca912836b8dec672f705e294f72d346e778557e4c591317d67558ea7669c0b module/archive
ad60ccc4fe9084d47f0477e02d9296bacad32f26d7456e2a84be8d25a53a25c2 module/boolean-param
6e3b8ccee738f8eddde5d813b6e204d6f7c8541fda4cb279a3be3ead27f47e7d module/channel-actions
00d326fdff396dfcc657d1fd16a4015b81d60ae8bf7dad09ac1010b376814de6 module/channel-config-helpers
@@ -72,7 +73,7 @@ d117ebba8cc490501725778676a9d75855872b6e5fe2b2f64b1270d4808a2277 module/health
182dc685f2103ff66c1a4839a48f4f40d2eeb0070cd74e449b47aacc4e6f1c22 module/hook-runtime
f6e3c44e7d1090a97aca554a3c247219b8de78b3cb4399cac5efde8a0a6c1156 module/inbound-envelope
36721c58f479fe9ca32737f67850f4607bf8753ef6b193ff129e7a9e5fdbc6bd module/inbound-reply-dispatch
5aeb593609dbf7e104294d61c6f8081f94353126ed521c7413e62cd1143e18cf module/infra-runtime
9cdfb3584d6e381b964880694d80d9769ba325b8c4fac78feda37534549173b0 module/infra-runtime
2e717cccb3db127aed0287d4ea14c41a8e64e46d60c728638153e31e2fb0d296 module/ingress-effect-once
9389d91a090259f06e5ec2aae14c9857e45988d4edc4f91170cb8bc573b966a2 module/interactive-runtime
9dd66baf2def46386ad4706380e57f9068fa9bb3878be2d9ba961ec2f46d3d87 module/json-store
@@ -86,7 +87,7 @@ a5f59c9acbcaa3f82247bf806eb5ba08032373fb853719f0ec9457690f16fc70 module/media-m
c5e3eb1a584f4b8126d9d6c177a840ec9103671e8d1242634ee67db9b5b9e573 module/media-understanding
c0ffaed532578cf33493992e1ff806b2268b8e3774a92edbaede5cf5bda162a6 module/media-understanding-runtime
bebd2931dc51d67c063ff19fa1c278f8dcfe00ab23cfbd480d47329ea8e5088e module/meeting-runtime
6b396a29c4efcd562accbd22dfa474f7815ce039cb8314a4771c2e37b5431ab4 module/memory-core-host-engine-foundation
aec2225e0341aea994c2d5dd0e642e9c44281ba0aee445574e2355eda4225945 module/memory-core-host-engine-foundation
5d4d709d5ae573186459462fe5119bd253c13104554eb94aefbdab5f5c7ad46f module/memory-host-core
87b7a3206346c0d4b294fb3a2395cbaabc3e73ff8b1b9ea925bc3aade3e52687 module/messaging-targets
5011823e5530df577d800e2b910c4b4f49b59084aa1df3bc03e3962ad3aee279 module/model-session-runtime
@@ -107,6 +108,7 @@ aa2a56b4448c8ebdec9d06aac95d809995f533093d42fa32cd75e1d852967245 module/questio
ac2b199e95c5c8b1e2a65e62bd41d1b6322e531bca294ef4979a297a12640bce module/reply-history
f394fe4d5a7ed9e4d574063ae44e8d6af85c9a0e7d8b329f750ca16b0664325f module/reply-payload
ea18ab3eb4b2055f47ca75e146b2a359c0f904e513dc198f231ba33df16cbf60 module/reply-runtime
b4043b356372f6af64ee3c26e4d6a6d623b817e4d95346358dc0e64a3b61d1e0 module/root-walk
536341e301631a14ac67bd7e8d10d2ba770ff91c5f7a2c5dc8dd9dfd1c1c7ec4 module/routing
ff6cca86f54f94f238205f5b122af36666314e0a380f3ec7f0ccb9ed9208df31 module/run-command
53b0295cec105696a1664c5c7f5576a7b55d197eb95dcd9185486f010bd53750 module/runtime
@@ -114,15 +116,16 @@ fcf103d18c1d64ffa9719f1d5a1ca349df3a6125d00c9d54a1b76c195ba8691b module/runtime
b6b8edc50ecab8386c9acd8f374a207212b5a99c8f518538bbcf0c458dda3881 module/runtime-env
1b223b31f6f78d9eca4a07b92802e59684f24d7a1d040b83856ad165d0dd70fd module/runtime-group-policy
0d8f2c5f3a3325d7d190d2c395835f58ac57f1bea9b6fa516c6942b96d9bf605 module/runtime-store
aa8a411ad37c1d1143b67376bf2d20255b9eedff61d80815f42e4f8ed7bd8e58 module/secret-file
44adc2205f926172fcd3762ca8a96c1485beabcb1bef8b9acfd2233cefea2a6a module/secret-input
57dcb1462d4c4f9a98d934c4ca975b163d704758af9821a64001ff3ac05637c3 module/secret-input-runtime
e576b537880f63b3a91f3608f7e84c873bce6c6a3d9a0ba98c247f46de788d25 module/secret-ref-runtime
c81b9702c192d574413fc1df9a57c73128652a25de205eec55d6b47549349283 module/security-runtime
62ccaafc8e0677e850339f4a4333f9f16ae9fed979bcef003890b2a47507147f module/security-runtime
31b785e74f1f8f56241b7756ef6a5d86199c5ce177cbb1c234a261866972f270 module/session-discussion
f112bdabc51ba8659b37d0a6f6a32a2b1d471e5b49b56e108bf750ec55a7ea71 module/session-store-runtime
36affbe151431a6141664b6838e20f2d121ff210d57a3c1b4b41a8818b5c81d8 module/setup
21071e8c2ef020685aba09b5661e37e7415938ff6602053fa831ac9d58673248 module/setup-runtime
cd431f6ba8327b81438b7a63b1963120f200f5abd145fb6aa7c5c561339cb0b1 module/setup-tools
df263e722a36ffa168522fa739924bec6b4b122ac3f1e25497a30962bec30607 module/setup-tools
18e384ec43d9eaee52c8e286e127bda2048370e2337964a754d94b236724ca9e module/skill-commands-runtime
ae469f32799380e6b045abaefefee6eb3f00d714ffbf36b6eeef5025dc529472 module/speech-settings
9e521fe9073dfd1a6a6855f909fa6befe8613e18403f0a65faaba973a8b630c1 module/ssrf-policy
docs/cli/fleet.md
+1 -1
View File
@@ -223,7 +223,7 @@ Restore needs an existing stopped container because its inspected runtime profil
Both commands accept `--max-bytes <bytes>` to bound archived or extracted file data, and both apply the same fixed one-million budget of archive path segments so metadata-only archive bombs cannot exhaust host inodes and every accepted backup stays restorable. Backup accepts `--out <path>` and both commands support `--json`.
Archives contain regular files and directories only. Backup never follows or stores symlinks, hard links, sockets, or device nodes; skipped counts are reported in the result. Restore rejects archives containing any other entry type. Recreatable symlink trees such as workspace `node_modules` must be reinstalled inside the cell after a restore.
Archives contain regular files and directories only. Backup never follows or stores symlinks, hard links, sockets, or device nodes; skipped counts are reported in the result. Restore rejects archives containing any other entry type, ignores archived ownership, and clamps restored file and directory modes before applying the cell runtime owner. Recreatable symlink trees such as workspace `node_modules` must be reinstalled inside the cell after a restore.
## `fleet doctor`
docs/docs_map.md
+3 -3
View File
@@ -4131,9 +4131,9 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
- Route: /gateway/security/secure-file-operations
- Headings:
- H2: Default: no Python helper
- H2: What stays protected without Python
- H2: What Python adds
- H2: Default: JavaScript fallback
- H2: What stays protected without native acceleration
- H2: What native acceleration adds
- H2: Plugin and core guidance
## gateway/tailscale.md
docs/gateway/security/index.md
+2 -2
View File
@@ -716,8 +716,8 @@ The Control UI needs a secure context (HTTPS or localhost) to generate device id
## Deployment and host trust
- Full-disk encryption on the gateway host; prefer a dedicated OS user account for the Gateway if the host is shared.
- Dependency review and packaging: `pnpm-lock.yaml` is the committed product dependency review boundary; the ClawHub release toolchain keeps a separate reviewed npm project lock. Published plugins bundle runtime dependency files by default, while the root package and native-heavy plugins resolve exact-pinned direct dependencies at install time. OpenClaw packages ship no npm lockfiles. See [dependency locking](/gateway/security/dependency-locking).
- Secure file operations: OpenClaw uses `@openclaw/fs-safe` for root-bounded file access, atomic writes, archive extraction, temp workspaces, and secret-file helpers. The optional POSIX Python helper defaults **off**; set `OPENCLAW_FS_SAFE_PYTHON_MODE=auto` or `require` only when you want the extra fd-relative mutation hardening and can support a Python runtime. Details: [Secure file operations](/gateway/security/secure-file-operations).
- Published package dependency lock: source checkouts use `pnpm-lock.yaml`; the published `openclaw` npm package and OpenClaw-owned npm plugin packages include `npm-shrinkwrap.json` so installs use the reviewed transitive dependency graph from the release instead of resolving a fresh graph at install time. This is a supply-chain hardening and release reproducibility boundary, not a sandbox - see [npm shrinkwrap](/gateway/security/shrinkwrap).
- Secure file operations: OpenClaw uses `@openclaw/fs-safe` for root-bounded file access, atomic writes, archive extraction, temp workspaces, and secret-file helpers. Optional native acceleration defaults **off**; set `OPENCLAW_FS_SAFE_NATIVE_MODE=auto` to use an installed platform binding or `require` to fail closed when native support is unavailable. Details: [Secure file operations](/gateway/security/secure-file-operations).
- Shared Slack workspace risk: if everyone in Slack can message the bot, the core risk is delegated tool authority - any allowed sender can induce tool calls (`exec`, browser, network/file tools) within the agent's policy, prompt/content injection from one sender can affect shared state/devices/outputs, and if the shared agent has sensitive credentials/files, any allowed sender can potentially drive exfiltration via tool usage. Use separate agents/gateways with minimal tools for team workflows; keep personal-data agents private.
- Company-shared agent (acceptable pattern): fine when everyone using the agent is in the same trust boundary (for example one company team) and the agent is strictly business-scoped. Run it on a dedicated machine/VM/container, use a dedicated OS user + dedicated browser/profile/accounts, and do not sign that runtime into personal Apple/Google accounts or personal password-manager/browser profiles. Mixing personal and company identities on the same runtime collapses the separation and increases personal-data exposure risk.
docs/gateway/security/secure-file-operations.md
+22 -23
View File
@@ -1,5 +1,5 @@
---
summary: "How OpenClaw handles local file access safely, and why the optional fs-safe Python helper is off by default"
summary: "How OpenClaw handles local file access safely, and why optional fs-safe native acceleration is off by default"
read_when:
- Changing file access, archive extraction, workspace storage, or plugin filesystem helpers
title: "Secure file operations"
@@ -9,35 +9,34 @@ OpenClaw uses [`@openclaw/fs-safe`](https://github.com/openclaw/fs-safe) for sec
It is a **library guardrail** for trusted OpenClaw code that receives untrusted path names, not a sandbox. Host filesystem permissions, OS users, containers, and the agent/tool policy still define the real blast radius.
## Default: no Python helper
## Default: JavaScript fallback
OpenClaw sets the fs-safe POSIX Python helper to **off** by default:
OpenClaw sets fs-safe's optional native helper to **off** by default:
- the gateway should not spawn a persistent Python sidecar unless an operator opts in;
- most installs do not need the extra parent-directory mutation hardening;
- disabling Python keeps runtime behavior predictable across desktop, Docker, CI, and bundled-app environments.
- native platform packages are optional and may be absent from minimal installs;
- the guarded JavaScript paths support OpenClaw's normal filesystem operations;
- disabling native loading keeps runtime behavior deterministic across desktop, Docker, CI, and bundled-app environments.
OpenClaw only changes the _default_. An explicit setting always wins:
```bash
# Default OpenClaw behavior: Node-only fs-safe fallbacks.
OPENCLAW_FS_SAFE_PYTHON_MODE=off
# Default OpenClaw behavior: guarded JavaScript fs-safe paths.
OPENCLAW_FS_SAFE_NATIVE_MODE=off
# Opt into the helper when available, falling back if unavailable.
OPENCLAW_FS_SAFE_PYTHON_MODE=auto
# Prefer native primitives when the platform package is installed.
OPENCLAW_FS_SAFE_NATIVE_MODE=auto
# Fail closed if the helper cannot start.
OPENCLAW_FS_SAFE_PYTHON_MODE=require
# Optional explicit interpreter path.
OPENCLAW_FS_SAFE_PYTHON=/usr/bin/python3
# Fail closed when an operation needs native support and the binding is unavailable.
OPENCLAW_FS_SAFE_NATIVE_MODE=require
```
The generic fs-safe env names also work: `FS_SAFE_PYTHON_MODE` and `FS_SAFE_PYTHON`.
The generic fs-safe environment name also works: `FS_SAFE_NATIVE_MODE`.
Use `require` (not `auto`) when the helper is part of your security posture; `auto` silently falls back to Node-only behavior if the helper cannot start.
fs-safe 0.5 temporarily maps the retired `FS_SAFE_PYTHON_MODE` and `OPENCLAW_FS_SAFE_PYTHON_MODE` values to native modes and emits a deprecation warning. Migrate those names before fs-safe 0.6; Python interpreter path settings are no longer used.
## What stays protected without Python
Use `require` (not `auto`) when native primitives are part of your security posture. `auto` uses the guarded JavaScript implementation when the platform binding is unavailable.
## What stays protected without native acceleration
With the helper off, OpenClaw still gets fs-safe's Node-only guardrails:
@@ -51,16 +50,16 @@ With the helper off, OpenClaw still gets fs-safe's Node-only guardrails:
This covers OpenClaw's normal threat model: trusted gateway code handling untrusted model/plugin/channel path input inside a single trusted operator boundary.
## What Python adds
## What native acceleration adds
On POSIX, the optional helper keeps one persistent Python process and uses fd-relative filesystem operations for parent-directory mutations: rename, remove, mkdir, stat/list, and some write paths.
The optional platform package provides policy-free filesystem primitives used by fs-safe for create-only writes, guarded hard-link publication, asynchronous sidecar creation, and explicit no-replace rename publication. Linux uses `openat2` and `renameat2`; macOS uses descriptor-relative component checks and `renameatx_np`; Windows uses handle-relative operations and replacement-disabled rename.
That narrows same-UID race windows where another process swaps a parent directory between validation and mutation — defense in depth on hosts where untrusted local processes can modify the same directories OpenClaw operates in.
The TypeScript layer still owns policy, validation, retries, cleanup, and fallback decisions. Native support narrows filesystem race windows; it does not turn fs-safe into a sandbox.
If your deployment has that risk and Python is guaranteed to exist, set:
If your deployment requires those native primitives, install the matching optional platform package and set:
```bash
OPENCLAW_FS_SAFE_PYTHON_MODE=require
OPENCLAW_FS_SAFE_NATIVE_MODE=require
```
## Plugin and core guidance
docs/plugins/sdk-entrypoints.md
+3
View File
@@ -279,6 +279,9 @@ Pair `defineSetupPluginEntry(...)` with the narrow setup helper families:
| `openclaw/plugin-sdk/channel-setup` | Optional-install setup surfaces |
| `openclaw/plugin-sdk/channel-dm-policy` | Account-aware DM policy descriptors for setup flows |
| `openclaw/plugin-sdk/setup-tools` | Setup/install CLI, archive, and docs helpers |
| `openclaw/plugin-sdk/archive` | Bounded archive extraction and single-entry reads |
| `openclaw/plugin-sdk/root-walk` | Budgeted, root-bounded directory walking |
| `openclaw/plugin-sdk/secret-file` | Pinned secret reads and first-writer-wins creation |
Keep heavy SDKs, CLI registration, and long-lived runtime services in the
full entry.
docs/plugins/sdk-migration.md
+5
View File
@@ -468,6 +468,11 @@ For local media read policy, import `getAgentScopedMediaLocalRoots(...)` or
| Process-local async lock | `openclaw/plugin-sdk/async-lock-runtime` |
| File locks | `openclaw/plugin-sdk/file-lock` |
File-lock nesting is owner-scoped. Pass the same `reentrantOwner` only for
nested acquisitions in one logical operation; omit it for ordinary locking.
Never use a process-wide constant, because unrelated work would incorrectly
share the critical section.
Bundled plugins are scanner-guarded against `infra-runtime`, so repo code
cannot regress to the broad barrel.
docs/plugins/sdk-subpaths.md
+4 -1
View File
@@ -80,6 +80,9 @@ deprecated for new code; see the per-row notes below.
| `plugin-sdk/setup` | Shared setup wizard helpers, setup translator, allowlist prompts, setup status builders |
| `plugin-sdk/setup-runtime` | `defineChannelSetupContract`, `createSetupTranslator`, `createPatchedAccountSetupAdapter`, `createEnvPatchedAccountSetupAdapter`, `createSetupInputPresenceValidator`, `noteChannelLookupFailure`, `noteChannelLookupSummary`, `promptResolvedAllowFrom`, `splitSetupEntries`, `createAllowlistSetupWizardProxy`, `createDelegatedSetupWizardProxy` |
| `plugin-sdk/setup-tools` | `formatCliCommand`, `detectBinary`, `extractArchive`, `resolveBrewExecutable`, `formatDocsLink`, `CONFIG_DIR` |
| `plugin-sdk/archive` | `extractArchive`, `readArchiveEntry`, archive limits and entry kinds |
| `plugin-sdk/root-walk` | `walkRootDirectory`, root-walk options and entries |
| `plugin-sdk/secret-file` | `createSecretFileAtomic`, synchronous and asynchronous secret reads |
| `plugin-sdk/account-core` | Multi-account config/action-gate helpers, default-account fallback helpers |
| `plugin-sdk/account-id` | `DEFAULT_ACCOUNT_ID`, account-id normalization helpers |
| `plugin-sdk/account-resolution` | Account lookup + default-fallback helpers |
@@ -267,7 +270,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It
| `plugin-sdk/talk-config-runtime` | Private-local after July 2026; Talk provider config resolution helpers |
| `plugin-sdk/json-store` | Small JSON state read/write helpers |
| `plugin-sdk/json-unsafe-integers` | Private-local after July 2026; JSON parsing helpers that preserve unsafe integer literals as strings |
| `plugin-sdk/file-lock` | Private-local after July 2026; Re-entrant file-lock helpers plus Doctor-safe reclaim of definitely stale, unchanged retired lock sidecars |
| `plugin-sdk/file-lock` | Private-local after July 2026; Owner-scoped re-entrant file-lock helpers plus Doctor-safe reclaim of definitely stale, unchanged retired lock sidecars. Nested acquisitions share a refcount only when callers pass the same logical-operation `reentrantOwner`; ownerless or different-owner calls contend normally |
| `plugin-sdk/persistent-dedupe` | Disk-backed dedupe cache helpers |
| `plugin-sdk/ingress-effect-once` | Durable claim/commit guard for non-idempotent ingress side effects |
| `plugin-sdk/acp-runtime` | Private-local after July 2026; ACP runtime/session and reply-dispatch helpers |
extensions/browser/src/browser/extension-relay/relay-auth.test.ts
+17 -8
View File
@@ -31,8 +31,8 @@ describe("extension relay host-local secret", () => {
expect(resolveExtensionRelayToken()).toBeNull();
});
it("creates a 64-hex secret on ensure and persists it privately", () => {
const token = ensureExtensionRelayToken();
it("creates a 64-hex secret on ensure and persists it privately", async () => {
const token = await ensureExtensionRelayToken();
expect(token).toMatch(/^[0-9a-f]{64}$/);
const secretPath = path.join(stateDir, "credentials", "browser-extension-relay.secret");
expect(fs.existsSync(secretPath)).toBe(true);
@@ -41,19 +41,28 @@ describe("extension relay host-local secret", () => {
}
});
it("is stable across calls (does not rotate on read)", () => {
const first = ensureExtensionRelayToken();
expect(ensureExtensionRelayToken()).toBe(first);
it("is stable across calls (does not rotate on read)", async () => {
const first = await ensureExtensionRelayToken();
await expect(ensureExtensionRelayToken()).resolves.toBe(first);
expect(readExtensionRelayToken()).toBe(first);
});
it("gives different hosts (state dirs) different secrets", () => {
const a = ensureExtensionRelayToken();
it("adopts the first writer's token under concurrent creation", async () => {
const [first, second] = await Promise.all([
ensureExtensionRelayToken(),
ensureExtensionRelayToken(),
]);
expect(second).toBe(first);
expect(readExtensionRelayToken()).toBe(first);
});
it("gives different hosts (state dirs) different secrets", async () => {
const a = await ensureExtensionRelayToken();
const otherDir = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-relay-auth-2-")),
);
try {
const b = ensureExtensionRelayToken({ ...process.env, OPENCLAW_STATE_DIR: otherDir });
const b = await ensureExtensionRelayToken({ ...process.env, OPENCLAW_STATE_DIR: otherDir });
expect(b).not.toBe(a);
} finally {
fs.rmSync(otherDir, { recursive: true, force: true });
extensions/browser/src/browser/extension-relay/relay-auth.ts
+33 -11
View File
@@ -8,12 +8,17 @@
* Chrome, and no gateway credential ever has to travel to a node.
*/
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { tryReadSecretFileSync } from "openclaw/plugin-sdk/secret-file-runtime";
import {
createSecretFileAtomic,
readSecretFile,
tryReadSecretFileSync,
} from "openclaw/plugin-sdk/secret-file";
import { resolveOAuthDir } from "openclaw/plugin-sdk/state-paths";
const RELAY_SECRET_FILE = "browser-extension-relay.secret";
const RELAY_SECRET_REREAD_ATTEMPTS = 50;
const RELAY_SECRET_REREAD_DELAY_MS = 10;
// resolveOAuthDir returns `${stateDir}/credentials`, the shared credentials dir.
function resolveExtensionRelaySecretPath(env: NodeJS.ProcessEnv = process.env): string {
@@ -44,27 +49,44 @@ export function readExtensionRelayToken(env: NodeJS.ProcessEnv = process.env): s
* printed pairing string carries the other → 401). On EEXIST the winner's token
* is re-read.
*/
export function ensureExtensionRelayToken(env: NodeJS.ProcessEnv = process.env): string {
export async function ensureExtensionRelayToken(
env: NodeJS.ProcessEnv = process.env,
): Promise<string> {
const secretPath = resolveExtensionRelaySecretPath(env);
const existing = readExtensionRelayToken(env);
if (existing) {
return existing;
}
const token = crypto.randomBytes(32).toString("hex");
fs.mkdirSync(path.dirname(secretPath), { recursive: true, mode: 0o700 });
try {
fs.writeFileSync(secretPath, `${token}\n`, { mode: 0o600, flag: "wx" });
await createSecretFileAtomic({
rootDir: path.dirname(secretPath),
filePath: secretPath,
content: `${token}\n`,
});
return token;
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "EEXIST") {
if ((err as NodeJS.ErrnoException).code !== "secret-exists") {
throw err;
}
// Another process created it first; adopt its token.
const winner = readExtensionRelayToken(env);
if (!winner) {
throw new Error("extension relay secret exists but is unreadable/malformed", { cause: err });
// Another process created it first; its exclusive async write may still be
// finishing after the final name appears, so adopt it with a bounded reread.
for (let attempt = 0; attempt < RELAY_SECRET_REREAD_ATTEMPTS; attempt += 1) {
try {
const winner = normalizeToken(
await readSecretFile(secretPath, "browser extension relay secret"),
);
if (winner) {
return winner;
}
} catch {
// Retry only inside the bounded first-writer handoff window.
}
await new Promise<void>((resolve) => {
setTimeout(resolve, RELAY_SECRET_REREAD_DELAY_MS);
});
}
return winner;
throw new Error("extension relay secret exists but is unreadable/malformed", { cause: err });
}
}
extensions/browser/src/browser/extension-relay/relay-lifecycle.ts
+1 -1
View File
@@ -55,7 +55,7 @@ export async function ensureExtensionRelayForProfile(
// Resolve one canonical desired profile after applying that token so the
// intentional auth-derived cdpUrl change is not mistaken for config drift.
const { ensureExtensionRelayToken, readExtensionRelayToken } = await import("./relay-auth.js");
const token = readExtensionRelayToken() ?? ensureExtensionRelayToken();
const token = readExtensionRelayToken() ?? (await ensureExtensionRelayToken());
if (state.resolved.extensionRelayToken !== token) {
state.resolved = { ...state.resolved, extensionRelayToken: token };
}
extensions/browser/src/cli/browser-cli-extension.ts
+4 -4
View File
@@ -65,17 +65,17 @@ function buildRemoteGatewayRelayUrl(raw: string): string {
return url.toString();
}
function buildPairingString(gatewayUrl?: string): {
async function buildPairingString(gatewayUrl?: string): Promise<{
pairing: string;
relayPort: number;
remote: boolean;
} {
}> {
const cfg = getRuntimeConfig();
const resolved = resolveBrowserConfig(cfg.browser, cfg);
// Create the host-local relay secret if this host has not used the extension
// driver yet, so pairing works on a fresh gateway or node host before the
// relay has started. Pairing must run on the machine that hosts the browser.
const token = ensureExtensionRelayToken();
const token = await ensureExtensionRelayToken();
const profile = firstExtensionProfile();
const relayPort = profile?.relayPort ?? resolved.extensionRelayDefaultPort;
@@ -136,7 +136,7 @@ export function registerBrowserExtensionCommands(
await runCommandWithRuntime(
defaultRuntime,
async () => {
const result = buildPairingString(opts.gatewayUrl);
const result = await buildPairingString(opts.gatewayUrl);
if (opts.json === true) {
defaultRuntime.writeJson({
pairingString: result.pairing,
extensions/browser/src/control-service.ts
+1 -1
View File
@@ -52,7 +52,7 @@ async function startBrowserControlServiceUnlocked(): Promise<BrowserServerState
);
if (hasExtensionProfiles) {
const { ensureExtensionRelayToken } = await import("./browser/extension-relay/relay-auth.js");
ensureExtensionRelayToken();
await ensureExtensionRelayToken();
const refreshed = loadBrowserConfigForRuntimeRefresh();
resolved = resolveBrowserConfig(refreshed.browser, refreshed);
}
extensions/codex/src/app-server/auth-bridge.ts
+4 -1
View File
@@ -22,6 +22,7 @@ import {
type OAuthCredential,
} from "openclaw/plugin-sdk/agent-runtime";
import { hasUsableOAuthCredential } from "openclaw/plugin-sdk/provider-auth";
import { readSecretFile } from "openclaw/plugin-sdk/secret-file";
import { resolveCodexAppServerHomeDir, withEphemeralCodexAuthStore } from "./auth-start-options.js";
import type { CodexAppServerClient } from "./client.js";
import { ensureCodexComputerUseSharedPluginCache } from "./computer-use-cache.js";
@@ -755,7 +756,9 @@ function parseCodexCliAuthFileApiKey(raw: string): string | undefined {
async function readCodexCliAuthFileApiKey(env: NodeJS.ProcessEnv): Promise<string | undefined> {
try {
return parseCodexCliAuthFileApiKey(await fs.readFile(resolveCodexCliAuthFilePath(env), "utf8"));
return parseCodexCliAuthFileApiKey(
await readSecretFile(resolveCodexCliAuthFilePath(env), "Codex CLI auth file"),
);
} catch {
return undefined;
}
extensions/discord/src/send.voice.ts
+4 -1
View File
@@ -63,7 +63,10 @@ async function materializeVoiceMessageInput(
prefix: "voice-src-",
});
const filePath = await workspace.write(`input${ext}`, media.buffer);
return { filePath, cleanup: async () => await workspace.cleanup() };
return {
filePath,
cleanup: () => workspace.cleanup().then(() => undefined),
};
}
/**
extensions/file-transfer/src/tools/dir-fetch-tool.test.ts
+103 -164
View File
@@ -1,10 +1,10 @@
// File Transfer tests cover dir fetch tar validation through the tool boundary.
import crypto from "node:crypto";
import crypto, { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import * as tar from "tar";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { projectBoundedTextTail } from "../shared/append-bounded-text-tail.js";
let tmpRoot: string;
@@ -14,70 +14,37 @@ beforeEach(async () => {
afterEach(async () => {
vi.doUnmock("openclaw/plugin-sdk/media-store");
vi.doUnmock("openclaw/plugin-sdk/process-runtime");
vi.doUnmock("../shared/audit.js");
vi.doUnmock("./node-tool-invoke.js");
vi.resetModules();
await fs.rm(tmpRoot, { recursive: true, force: true });
});
function commandResult(overrides: Record<string, unknown> = {}) {
return {
stdout: "",
stderr: "",
code: 0,
signal: null,
killed: false,
termination: "exit",
...overrides,
};
async function createTarBuffer(params: {
entries: string[];
setup: (sourceDir: string) => Promise<void>;
}): Promise<Buffer> {
const sourceDir = path.join(tmpRoot, `source-${randomUUID()}`);
await fs.mkdir(sourceDir, { recursive: true });
await params.setup(sourceDir);
const chunks: Buffer[] = [];
for await (const chunk of tar.c({ cwd: sourceDir, gzip: true, portable: true }, params.entries)) {
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
type MockCommandResult = Record<string, unknown> & {
outputByteLength?: number;
};
async function importToolWithCommandResults(tarBuffer: Buffer, ...results: MockCommandResult[]) {
const runCommandWithTimeout = vi.fn();
for (const result of results) {
runCommandWithTimeout.mockImplementationOnce(
async (
_argv: string[],
options: { onOutputChunk?: (chunk: Buffer, stream: string) => boolean | void },
) => {
if (result.error instanceof Error && result.termination === "error") {
throw result.error;
}
let stopped = false;
const stdout = typeof result.stdout === "string" ? result.stdout : "";
if (stdout) {
stopped = options.onOutputChunk?.(Buffer.from(stdout), "stdout") === false;
} else if (typeof result.outputByteLength === "number") {
stopped =
options.onOutputChunk?.({ byteLength: result.outputByteLength } as Buffer, "stdout") ===
false;
}
return commandResult({
...result,
stdout: "",
...(stopped
? { code: null, killed: true, outputLimitExceeded: true, termination: "signal" }
: {}),
});
},
);
}
runCommandWithTimeout.mockResolvedValue(commandResult());
async function importTool(tarBuffer: Buffer) {
const archivePath = path.join(tmpRoot, `archive-${randomUUID()}.tar.gz`);
const appendFileTransferAudit = vi.fn(async () => undefined);
vi.resetModules();
vi.doMock("openclaw/plugin-sdk/process-runtime", () => ({
runCommandWithTimeout,
}));
vi.doMock("openclaw/plugin-sdk/media-store", () => ({
saveMediaBuffer: vi.fn(async () => ({ path: path.join(tmpRoot, "archive.tar.gz") })),
}));
vi.doMock("../shared/audit.js", () => ({
appendFileTransferAudit: vi.fn(async () => undefined),
saveMediaBuffer: vi.fn(async () => {
await fs.writeFile(archivePath, tarBuffer);
return { path: archivePath };
}),
}));
vi.doMock("../shared/audit.js", () => ({ appendFileTransferAudit }));
vi.doMock("./node-tool-invoke.js", () => ({
readRequiredNodePath: (params: Record<string, unknown>) => ({
node: String(params.node),
@@ -98,8 +65,9 @@ async function importToolWithCommandResults(tarBuffer: Buffer, ...results: MockC
})),
}));
return {
archivePath,
appendFileTransferAudit,
module: await import("./dir-fetch-tool.js"),
runCommandWithTimeout,
};
}
@@ -110,130 +78,101 @@ async function executeDirFetch(module: typeof import("./dir-fetch-tool.js")) {
});
}
const validListingResults = [{ stdout: "./ok.txt\n" }, { stdout: "-ok.txt\n" }] as const;
describe("dir.fetch tar validation", () => {
it("rejects an archive before extraction when expanded bytes exceed budget", async () => {
const { module } = await importToolWithCommandResults(
Buffer.from("archive"),
...validListingResults,
{ outputByteLength: 64 * 1024 * 1024 + 1 },
);
await expect(executeDirFetch(module)).rejects.toThrow(
"dir.fetch UNCOMPRESSED_TOO_LARGE: archive expands past uncompressed budget 67108864 bytes",
);
});
it("fails uncompressed budget checks closed on wrapper errors", async () => {
const { module, runCommandWithTimeout } = await importToolWithCommandResults(
Buffer.from("archive"),
...validListingResults,
{
code: null,
termination: "error",
error: new Error("budget read failed"),
describe("dir.fetch archive extraction", () => {
it("extracts a bounded tar and returns the plugin-side manifest", async () => {
const tarBuffer = await createTarBuffer({
entries: ["ok.txt"],
setup: async (sourceDir) => {
await fs.writeFile(path.join(sourceDir, "ok.txt"), "ok");
},
);
await expect(executeDirFetch(module)).rejects.toThrow(
"dir.fetch UNCOMPRESSED_TOO_LARGE: tar uncompressed budget validation error: budget read failed",
);
expect(runCommandWithTimeout).toHaveBeenLastCalledWith(
expect.any(Array),
expect.objectContaining({ tolerateOutputError: { stderr: true } }),
);
});
it("fails tar listing closed on wrapper errors", async () => {
const { module } = await importToolWithCommandResults(Buffer.from("archive"), {
code: null,
termination: "error",
error: new Error("listing read failed"),
});
const { appendFileTransferAudit, module } = await importTool(tarBuffer);
await expect(executeDirFetch(module)).rejects.toThrow(
"dir.fetch UNSAFE_ARCHIVE: tar -tzf error: listing read failed",
);
});
const result = await executeDirFetch(module);
it("accepts successful validation and unpack", async () => {
const { module, runCommandWithTimeout } = await importToolWithCommandResults(
Buffer.from("archive"),
...validListingResults,
{},
{},
);
await expect(executeDirFetch(module)).resolves.toMatchObject({
expect(result).toMatchObject({
details: {
path: "/tmp/project",
fileCount: 1,
files: [
{
relPath: "ok.txt",
size: 2,
sha256: crypto.createHash("sha256").update("ok").digest("hex"),
},
],
},
});
expect(runCommandWithTimeout).toHaveBeenLastCalledWith(
expect.any(Array),
expect.objectContaining({
outputCapture: { stdout: "discard", stderr: "tail" },
tolerateOutputError: { stderr: true },
}),
const localPath = (result.details as { files: Array<{ localPath: string }> }).files[0]
?.localPath;
await expect(fs.readFile(localPath!, "utf8")).resolves.toBe("ok");
expect(appendFileTransferAudit).toHaveBeenLastCalledWith(
expect.objectContaining({ decision: "allowed" }),
);
});
it("keeps tar exit diagnostics", async () => {
const { module } = await importToolWithCommandResults(Buffer.from("archive"), {
code: 2,
stderr: "invalid archive",
it.runIf(process.platform !== "win32")(
"settles promptly when a Fleet-shaped archive contains a symlink",
async () => {
const tarBuffer = await createTarBuffer({
entries: ["data", "auth"],
setup: async (sourceDir) => {
await fs.mkdir(path.join(sourceDir, "data"));
await fs.mkdir(path.join(sourceDir, "auth"));
await fs.writeFile(path.join(sourceDir, "data", "state.json"), "{}");
await fs.writeFile(path.join(sourceDir, "auth", "token"), "secret");
await fs.symlink("../auth/token", path.join(sourceDir, "data", "token-link"));
},
});
const { appendFileTransferAudit, archivePath, module } = await importTool(tarBuffer);
const settled = await Promise.race([
executeDirFetch(module).then(
() => ({ status: "resolved" as const }),
(error: unknown) => ({ status: "rejected" as const, error }),
),
new Promise<{ status: "timeout" }>((resolve) => {
setTimeout(() => resolve({ status: "timeout" }), 2_000);
}),
]);
expect(settled.status).toBe("rejected");
expect(settled.status === "rejected" ? String(settled.error) : "").toMatch(
/dir\.fetch UNSAFE_ARCHIVE:.*link/iu,
);
await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" });
expect(appendFileTransferAudit).toHaveBeenLastCalledWith(
expect.objectContaining({ decision: "error", errorCode: "UNSAFE_ARCHIVE" }),
);
},
);
it.runIf(process.platform !== "win32")("rejects backslash-containing entry names", async () => {
const tarBuffer = await createTarBuffer({
entries: ["dir\\escape.txt"],
setup: async (sourceDir) => {
await fs.writeFile(path.join(sourceDir, "dir\\escape.txt"), "blocked");
},
});
const { module } = await importTool(tarBuffer);
await expect(executeDirFetch(module)).rejects.toThrow(/dir\.fetch UNSAFE_ARCHIVE:.*filter/iu);
});
it("maps single-entry expansion limits to TREE_TOO_LARGE", async () => {
const tarBuffer = await createTarBuffer({
entries: ["large.bin"],
setup: async (sourceDir) => {
await fs.writeFile(path.join(sourceDir, "large.bin"), Buffer.alloc(16 * 1024 * 1024 + 1));
},
});
const { appendFileTransferAudit, module } = await importTool(tarBuffer);
await expect(executeDirFetch(module)).rejects.toThrow(
"dir.fetch UNSAFE_ARCHIVE: tar -tzf exited 2: invalid archive",
/dir\.fetch UNCOMPRESSED_TOO_LARGE: archive entry extracted size exceeds limit/iu,
);
});
it("stops name validation at the entry cap", async () => {
const tarLines = Array.from({ length: 5001 }, (_, index) => `file-${index}`).join("\n") + "\n";
const { module, runCommandWithTimeout } = await importToolWithCommandResults(
Buffer.from("archive"),
{ stdout: tarLines },
expect(appendFileTransferAudit).toHaveBeenLastCalledWith(
expect.objectContaining({ decision: "error", errorCode: "TREE_TOO_LARGE" }),
);
await expect(executeDirFetch(module)).rejects.toThrow(
"dir.fetch UNSAFE_ARCHIVE: archive contains 5001 entries; limit 5000",
);
expect(runCommandWithTimeout).toHaveBeenCalledOnce();
});
it("keeps recent tar stderr when listing fails noisily", async () => {
const oldNoise = "old-noise\n".repeat(600);
const recent = "recent-invalid-archive-details\n".repeat(12);
const { module } = await importToolWithCommandResults(Buffer.from("archive"), {
code: 2,
stderr: oldNoise + recent,
});
await expect(executeDirFetch(module)).rejects.toThrow(projectBoundedTextTail(recent, 200));
});
it("surfaces a UTF-16-safe tar stderr tail", async () => {
const oldNoise = "n".repeat(250);
const recent = "🤖" + "f".repeat(199);
const { module } = await importToolWithCommandResults(Buffer.from("archive"), {
code: 2,
stderr: oldNoise + recent,
});
let message = "";
try {
await executeDirFetch(module);
} catch (error) {
message = error instanceof Error ? error.message : String(error);
}
expect(message).toContain(projectBoundedTextTail(recent, 200));
expect(message).toContain("f".repeat(199));
expect(message).not.toContain("🤖");
expect(
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/.test(message),
).toBe(false);
});
});
extensions/file-transfer/src/tools/dir-fetch-tool.ts
+60 -351
View File
@@ -2,12 +2,14 @@
import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { StringDecoder } from "node:string_decoder";
import type { AnyAgentTool } from "openclaw/plugin-sdk/agent-harness-runtime";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
ARCHIVE_LIMIT_ERROR_CODE,
ArchiveLimitError,
extractArchive,
type ArchiveEntryKind,
} from "openclaw/plugin-sdk/archive";
import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store";
import { runCommandWithTimeout } from "openclaw/plugin-sdk/process-runtime";
import { projectBoundedTextTail } from "../shared/append-bounded-text-tail.js";
import { appendFileTransferAudit } from "../shared/audit.js";
import { IMAGE_MIME_INLINE_SET, mimeFromExtension } from "../shared/mime.js";
import { humanSize, readBoolean, readClampedInt } from "../shared/params.js";
@@ -24,7 +26,7 @@ import { invokeNodeToolPayload, readRequiredNodePath } from "./node-tool-invoke.
// with hundreds of attachments.
const MEDIA_URL_CAP = 25;
// Hard timeout for gateway-side tar processes.
// Hard timeout for gateway-side archive extraction.
const TAR_UNPACK_TIMEOUT_MS = 60_000;
// Cap on number of entries pre-validated. The compressed tar is already
@@ -32,88 +34,35 @@ const TAR_UNPACK_TIMEOUT_MS = 60_000;
// tree to compute hashes — TAR_UNPACK_MAX_ENTRIES bounds how much work
// that walk can do.
const TAR_UNPACK_MAX_ENTRIES = 5000;
const TAR_LIST_OUTPUT_MAX_CHARS = 32 * 1024 * 1024;
const TAR_STDERR_TAIL_CHARS = 4096;
const TAR_ERROR_REASON_STDERR_CHARS = 200;
const TAR_UNPACK_ERROR_STDERR_CHARS = 300;
// Hard caps on uncompressed extraction. Defends against decompression-bomb
// archives that compress to <16MB but expand to gigabytes. Both caps are
// enforced during the post-extract walk: total bytes summed across entries
// and per-file size to bound any single fs.stat / hash operation.
// enforced by fs-safe while extracting.
const DIR_FETCH_MAX_UNCOMPRESSED_BYTES = 64 * 1024 * 1024;
const DIR_FETCH_MAX_SINGLE_FILE_BYTES = 16 * 1024 * 1024;
async function listTarOutputLines<T>(input: {
args: string[];
label: string;
tarBuffer: Buffer;
mapLine: (line: string) => T;
maxValues: number;
}): Promise<{ ok: true; values: T[] } | { ok: false; reason: string }> {
const tarBin = process.platform !== "win32" ? "/usr/bin/tar" : "tar";
const decoder = new StringDecoder("utf8");
const values: T[] = [];
let pending = "";
let outputBytes = 0;
let outputTooLarge = false;
let valueLimitReached = false;
const appendLine = (line: string): boolean => {
if (!line) {
return true;
}
values.push(input.mapLine(line));
valueLimitReached = values.length >= input.maxValues;
return !valueLimitReached;
};
let result: Awaited<ReturnType<typeof runCommandWithTimeout>>;
try {
result = await runCommandWithTimeout([tarBin, ...input.args], {
input: input.tarBuffer,
maxOutputBytes: { stderr: TAR_STDERR_TAIL_CHARS },
onOutputChunk: (chunk, stream) => {
if (stream !== "stdout") {
return true;
}
outputBytes += chunk.byteLength;
if (outputBytes > TAR_LIST_OUTPUT_MAX_CHARS) {
outputTooLarge = true;
return false;
}
const lines = `${pending}${decoder.write(chunk)}`.split("\n");
pending = lines.pop() ?? "";
return lines.every(appendLine);
},
outputCapture: { stdout: "discard", stderr: "tail" },
tolerateOutputError: { stderr: true },
timeoutMs: 30_000,
});
} catch (error) {
return { ok: false, reason: `${input.label} error: ${formatErrorMessage(error)}` };
function filterDirFetchArchiveEntry(entry: {
path: string;
kind: ArchiveEntryKind;
}): "extract" | "skip" {
return (entry.kind === "file" || entry.kind === "directory") && !entry.path.includes("\\")
? "extract"
: "skip";
}
function classifyArchiveFailure(error: unknown): {
auditCode: "TREE_TOO_LARGE" | "UNSAFE_ARCHIVE";
publicCode: "UNCOMPRESSED_TOO_LARGE" | "UNSAFE_ARCHIVE";
reason: string;
} {
const reason = error instanceof Error ? error.message : String(error);
if (
error instanceof ArchiveLimitError &&
error.code !== ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT
) {
return { auditCode: "TREE_TOO_LARGE", publicCode: "UNCOMPRESSED_TOO_LARGE", reason };
}
if (result.termination === "timeout") {
return { ok: false, reason: `${input.label} timed out` };
}
if (valueLimitReached) {
return { ok: true, values };
}
if (outputTooLarge) {
return { ok: false, reason: `${input.label} output too large` };
}
if (result.termination !== "exit") {
return {
ok: false,
reason: `${input.label} error: ${result.termination}`,
};
}
if (result.code !== 0) {
return {
ok: false,
reason: `${input.label} exited ${result.code}: ${projectBoundedTextTail(result.stderr, TAR_ERROR_REASON_STDERR_CHARS)}`,
};
}
appendLine(pending + decoder.end());
return { ok: true, values };
return { auditCode: "UNSAFE_ARCHIVE", publicCode: "UNSAFE_ARCHIVE", reason };
}
async function computeFileSha256(filePath: string): Promise<string> {
@@ -138,160 +87,6 @@ async function computeFileSha256(filePath: string): Promise<string> {
return hash.digest("hex");
}
/**
* Run two passes against the buffer to enumerate entries BEFORE we extract:
*
* 1. `tar -tf -` produces names ONLY, one per line. This is whitespace-safe
* because each line is exactly one path; no parsing of fixed columns.
* Used to validate paths (reject absolute, '..' traversal).
* 2. `tar -tvf -` adds type info via the `ls -l`-style perm prefix.
* Used ONLY to detect symlinks / hardlinks / non-regular entries via
* the FIRST CHARACTER of each line, never the path column.
*
* Size limits are enforced at the *extraction* step instead — the tar
* unpack process is bounded by the maxBytes we already pass through, and
* the post-extract walkDir is hard-capped by TAR_UNPACK_MAX_ENTRIES.
* Trying to parse uncompressed sizes from `tar -tvf` output is fragile
* (filenames with whitespace shift the columns) and Aisle flagged that
* shape as a bypass primitive — drop it.
*/
async function listTarPaths(
tarBuffer: Buffer,
): Promise<{ ok: true; paths: string[] } | { ok: false; reason: string }> {
const result = await listTarOutputLines({
args: ["-tzf", "-"],
label: "tar -tzf",
tarBuffer,
mapLine: (line) => line,
maxValues: TAR_UNPACK_MAX_ENTRIES + 1,
});
return result.ok ? { ok: true, paths: result.values } : result;
}
async function listTarTypeChars(
tarBuffer: Buffer,
): Promise<{ ok: true; typeChars: string[] } | { ok: false; reason: string }> {
const result = await listTarOutputLines({
args: ["-tzvf", "-"],
label: "tar -tzvf",
tarBuffer,
mapLine: (line) => line.charAt(0),
maxValues: TAR_UNPACK_MAX_ENTRIES + 1,
});
return result.ok ? { ok: true, typeChars: result.values } : result;
}
async function preValidateTarball(
tarBuffer: Buffer,
): Promise<{ ok: true } | { ok: false; reason: string }> {
const namesResult = await listTarPaths(tarBuffer);
if (!namesResult.ok) {
return namesResult;
}
const paths = namesResult.paths;
if (paths.length > TAR_UNPACK_MAX_ENTRIES) {
return {
ok: false,
reason: `archive contains ${paths.length} entries; limit ${TAR_UNPACK_MAX_ENTRIES}`,
};
}
const typesResult = await listTarTypeChars(tarBuffer);
if (!typesResult.ok) {
return typesResult;
}
const typeChars = typesResult.typeChars;
// The two passes should report the same number of entries; if they
// don't, something exotic is going on (filenames with newlines, etc.)
// and we refuse defensively.
if (typeChars.length !== paths.length) {
return {
ok: false,
reason: `tar -tzf and tar -tzvf disagree on entry count (${paths.length} vs ${typeChars.length}); refusing`,
};
}
for (const [index, entryPath] of paths.entries()) {
const t = typeChars.at(index);
if (t === undefined) {
return {
ok: false,
reason: `tar -tzf and tar -tzvf disagree on entry count (${paths.length} vs ${typeChars.length}); refusing`,
};
}
if (t === "l" || t === "h") {
return { ok: false, reason: `archive contains link entry: ${entryPath}` };
}
if (t !== "-" && t !== "d") {
return { ok: false, reason: `archive contains non-regular entry type '${t}': ${entryPath}` };
}
if (path.isAbsolute(entryPath)) {
return { ok: false, reason: `archive contains absolute path: ${entryPath}` };
}
const norm = path.posix.normalize(entryPath);
if (norm === ".." || norm.startsWith("../") || norm.includes("/../")) {
return { ok: false, reason: `archive contains '..' traversal: ${entryPath}` };
}
// Reject backslash-containing names too — refuses Windows-style
// traversal in archives produced by an attacker on a Windows node.
if (entryPath.includes("\\")) {
return { ok: false, reason: `archive contains backslash in path: ${entryPath}` };
}
}
return { ok: true };
}
async function validateTarUncompressedBudget(
tarBuffer: Buffer,
maxBytes = DIR_FETCH_MAX_UNCOMPRESSED_BYTES,
): Promise<{ ok: true } | { ok: false; reason: string }> {
const tarBin = process.platform !== "win32" ? "/usr/bin/tar" : "tar";
let totalBytes = 0;
let budgetExceeded = false;
let result: Awaited<ReturnType<typeof runCommandWithTimeout>>;
try {
result = await runCommandWithTimeout([tarBin, "-xOzf", "-"], {
input: tarBuffer,
maxOutputBytes: { stderr: TAR_STDERR_TAIL_CHARS },
onOutputChunk: (chunk, stream) => {
if (stream !== "stdout") {
return true;
}
totalBytes += chunk.byteLength;
budgetExceeded = totalBytes > maxBytes;
return !budgetExceeded;
},
outputCapture: { stdout: "discard", stderr: "tail" },
tolerateOutputError: { stderr: true },
timeoutMs: TAR_UNPACK_TIMEOUT_MS,
});
} catch (error) {
return {
ok: false,
reason: `tar uncompressed budget validation error: ${formatErrorMessage(error)}`,
};
}
if (result.termination === "timeout") {
return { ok: false, reason: "tar uncompressed budget validation timed out" };
}
if (budgetExceeded) {
return { ok: false, reason: `archive expands past uncompressed budget ${maxBytes} bytes` };
}
if (result.termination !== "exit") {
return {
ok: false,
reason: `tar uncompressed budget validation error: ${result.termination}`,
};
}
if (result.code !== 0) {
return {
ok: false,
reason: `tar uncompressed budget validation exited ${result.code}: ${projectBoundedTextTail(result.stderr, TAR_ERROR_REASON_STDERR_CHARS)}`,
};
}
return { ok: true };
}
type UnpackedFileEntry = {
relPath: string;
size: number;
@@ -300,55 +95,6 @@ type UnpackedFileEntry = {
localPath: string;
};
/**
* Unpack a gzipped tarball into a target directory via `tar -xzf -`.
* Caller MUST have run `preValidateTarball` first — this function trusts
* that the archive contains only regular files / dirs with relative,
* non-traversing paths. Without that pre-validation, raw `tar -xzf` is
* unsafe (tarbomb, symlink-then-write tricks, decompression bomb).
*
* The `-P` flag is intentionally omitted so absolute paths in the
* archive are stripped to relative ones (defense-in-depth on top of the
* pre-validation rejection). A hard wall-clock timeout caps the unpack
* at TAR_UNPACK_TIMEOUT_MS to avoid hangs.
*
* BSD tar (macOS) and GNU tar disagree on flags: `--no-overwrite-dir` is
* GNU-only and BSD tar rejects it. We use only flags both implementations
* accept. Defense-in-depth comes from the pre-validation step instead.
*
* `--no-same-owner` and `--no-same-permissions` are accepted by both BSD
* and GNU tar. They prevent the archive from setting file ownership
* (uid/gid) and dangerous mode bits (setuid/setgid/world-writable) on
* the gateway filesystem. If the gateway is ever run as root or with
* elevated privileges, a malicious node could otherwise plant
* privileged executables here.
*/
async function unpackTar(tarBuffer: Buffer, destDir: string): Promise<void> {
await fs.mkdir(destDir, { recursive: true, mode: 0o700 });
const tarBin = process.platform !== "win32" ? "/usr/bin/tar" : "tar";
const result = await runCommandWithTimeout(
[tarBin, "-xzf", "-", "-C", destDir, "--no-same-owner", "--no-same-permissions"],
{
input: tarBuffer,
maxOutputBytes: { stderr: TAR_STDERR_TAIL_CHARS },
outputCapture: { stdout: "discard", stderr: "tail" },
tolerateOutputError: { stderr: true },
timeoutMs: TAR_UNPACK_TIMEOUT_MS,
},
);
if (result.termination === "timeout") {
throw new Error(`tar unpack timed out after ${TAR_UNPACK_TIMEOUT_MS}ms`);
}
if (result.termination !== "exit") {
throw new Error(`tar unpack failed: ${result.termination}`);
}
if (result.code !== 0) {
throw new Error(
`tar unpack exited ${result.code}: ${projectBoundedTextTail(result.stderr, TAR_UNPACK_ERROR_STDERR_CHARS)}`,
);
}
}
/**
* Walk a directory recursively, collecting file entries (skips directories).
* Skips symlinks — we don't want to follow links the archive might have
@@ -423,47 +169,6 @@ export function createDirFetchTool(): AnyAgentTool {
throw new Error("dir.fetch sha256 mismatch (integrity failure)");
}
// Pre-validate before extraction. The node is in the trust boundary
// for v1, but a malicious or compromised node should not be able to
// pivot into arbitrary file write on the gateway via tar tricks.
// Rejects: symlinks, hardlinks, absolute paths, ".." traversal,
// entry counts and uncompressed sizes above the caps.
const validation = await preValidateTarball(tarBuffer);
if (!validation.ok) {
await appendFileTransferAudit({
op: "dir.fetch",
nodeId,
nodeDisplayName,
requestedPath: dirPath,
canonicalPath,
decision: "error",
errorCode: "UNSAFE_ARCHIVE",
errorMessage: validation.reason,
sizeBytes: tarBytes,
sha256,
durationMs: Date.now() - startedAt,
});
throw new Error(`dir.fetch UNSAFE_ARCHIVE: ${validation.reason}`);
}
const budget = await validateTarUncompressedBudget(tarBuffer);
if (!budget.ok) {
await appendFileTransferAudit({
op: "dir.fetch",
nodeId,
nodeDisplayName,
requestedPath: dirPath,
canonicalPath,
decision: "error",
errorCode: "TREE_TOO_LARGE",
errorMessage: budget.reason,
sizeBytes: tarBytes,
sha256,
durationMs: Date.now() - startedAt,
});
throw new Error(`dir.fetch UNCOMPRESSED_TOO_LARGE: ${budget.reason}`);
}
// Save tarball under the file-transfer subdir (no 2-min TTL).
const savedTar = await saveMediaBuffer(
tarBuffer,
@@ -476,18 +181,30 @@ export function createDirFetchTool(): AnyAgentTool {
const tarBaseName = path.basename(savedTar.path, path.extname(savedTar.path));
const unpackId = `dir-fetch-${tarBaseName}`;
const rootDir = path.join(tarDir, unpackId);
await unpackTar(tarBuffer, rootDir);
const walked = await walkDir(rootDir, rootDir);
const files: UnpackedFileEntry[] = [];
// Defense-in-depth budget on the *uncompressed* extraction. Compressed
// tar is bounded upstream; an attacker can still send a highly
// compressible bomb (gigabytes of zeros) that fits under that cap.
// Stop walking + clean up if the unpacked tree busts the budget.
let totalUncompressed = 0;
const abortAndCleanup = async (reason: string): Promise<never> => {
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
await fs.mkdir(rootDir, { recursive: true, mode: 0o700 });
try {
await extractArchive({
archivePath: savedTar.path,
destDir: rootDir,
kind: "tar",
tarGzip: true,
timeoutMs: TAR_UNPACK_TIMEOUT_MS,
entryModes: "clamp",
entryFilter: filterDirFetchArchiveEntry,
onFiltered: "reject-archive",
limits: {
maxArchiveBytes: DIR_FETCH_HARD_MAX_BYTES,
maxEntries: TAR_UNPACK_MAX_ENTRIES,
maxExtractedBytes: DIR_FETCH_MAX_UNCOMPRESSED_BYTES,
maxEntryBytes: DIR_FETCH_MAX_SINGLE_FILE_BYTES,
},
});
} catch (error) {
await Promise.all([
fs.rm(rootDir, { recursive: true, force: true }).catch(() => undefined),
fs.rm(savedTar.path, { force: true }).catch(() => undefined),
]);
const failure = classifyArchiveFailure(error);
await appendFileTransferAudit({
op: "dir.fetch",
nodeId,
@@ -495,14 +212,17 @@ export function createDirFetchTool(): AnyAgentTool {
requestedPath: dirPath,
canonicalPath,
decision: "error",
errorCode: "TREE_TOO_LARGE",
errorMessage: reason,
errorCode: failure.auditCode,
errorMessage: failure.reason,
sizeBytes: tarBytes,
sha256,
durationMs: Date.now() - startedAt,
});
throw new Error(`dir.fetch UNCOMPRESSED_TOO_LARGE: ${reason}`);
};
throw new Error(`dir.fetch ${failure.publicCode}: ${failure.reason}`, { cause: error });
}
const walked = await walkDir(rootDir, rootDir);
const files: UnpackedFileEntry[] = [];
for (const { relPath, absPath } of walked) {
let size;
try {
@@ -511,17 +231,6 @@ export function createDirFetchTool(): AnyAgentTool {
} catch {
continue;
}
if (size > DIR_FETCH_MAX_SINGLE_FILE_BYTES) {
await abortAndCleanup(
`extracted file ${relPath} is ${size} bytes (limit ${DIR_FETCH_MAX_SINGLE_FILE_BYTES})`,
);
}
totalUncompressed += size;
if (totalUncompressed > DIR_FETCH_MAX_UNCOMPRESSED_BYTES) {
await abortAndCleanup(
`extracted tree exceeds uncompressed budget ${DIR_FETCH_MAX_UNCOMPRESSED_BYTES} bytes (decompression bomb?)`,
);
}
const mimeType = mimeFromExtension(relPath);
const fileSha256 = await computeFileSha256(absPath);
files.push({ relPath, size, mimeType, sha256: fileSha256, localPath: absPath });
extensions/imessage/src/monitor/media-staging.test-utils.ts
+92 -3
View File
@@ -2,6 +2,7 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { openLocalFileSafely } from "openclaw/plugin-sdk/security-runtime";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { stageIMessageAttachments } from "./media-staging.js";
@@ -52,6 +53,42 @@ describe("stageIMessageAttachments", () => {
);
});
it("reads from the pinned attachment when its pathname is replaced", async () => {
const sourcePath = await writeTempFile("photo.png", Buffer.from("original"));
const displacedPath = `${sourcePath}.displaced`;
const saveMediaBuffer = vi.fn(async () => ({
id: "saved.png",
path: "/state/media/inbound/saved.png",
size: 8,
contentType: "image/png",
}));
await stageIMessageAttachments(
[{ original_path: sourcePath, mime_type: "image/png", missing: false }],
{
maxBytes: 1024,
allowedRoots: [tempDir],
deps: {
saveMediaBuffer,
openLocalFileSafely: async (options) => {
const opened = await openLocalFileSafely(options);
await fs.rename(sourcePath, displacedPath);
await fs.writeFile(sourcePath, "replacement");
return opened;
},
},
},
);
expect(saveMediaBuffer).toHaveBeenCalledWith(
Buffer.from("original"),
"image/png",
"inbound",
1024,
"photo.png",
);
});
it("drops attachments whose canonical path escapes the allowed root", async () => {
const allowedRoot = path.join(tempDir, "allowed");
const outsideRoot = path.join(tempDir, "outside");
@@ -88,20 +125,39 @@ describe("stageIMessageAttachments", () => {
it("converts HEIC iMessage attachments to JPEG before staging", async () => {
const sourcePath = await writeTempFile("IMG_0001.HEIC", Buffer.from("heic-bytes"));
const displacedPath = `${sourcePath}.displaced`;
const saveMediaBuffer = vi.fn(async () => ({
id: "saved.jpg",
path: "/state/media/inbound/saved.jpg",
size: 10,
contentType: "image/jpeg",
}));
const convertHeicToJpeg = vi.fn(async () => Buffer.from("jpeg-bytes"));
const convertHeicToJpeg = vi.fn(async (pinnedPath: string) => {
await expect(fs.readFile(pinnedPath, "utf8")).resolves.toBe("heic-bytes");
return Buffer.from("jpeg-bytes");
});
await stageIMessageAttachments(
[{ original_path: sourcePath, mime_type: "image/heic", missing: false }],
{ maxBytes: 1024, deps: { saveMediaBuffer, convertHeicToJpeg } },
{
maxBytes: 1024,
deps: {
saveMediaBuffer,
convertHeicToJpeg,
openLocalFileSafely: async (options) => {
const opened = await openLocalFileSafely(options);
await fs.rename(sourcePath, displacedPath);
await fs.writeFile(sourcePath, "replacement");
return opened;
},
},
},
);
expect(convertHeicToJpeg).toHaveBeenCalledWith(sourcePath, 1024);
expect(convertHeicToJpeg).toHaveBeenCalledWith(
expect.stringContaining("attachment.heic"),
1024,
);
expect(saveMediaBuffer).toHaveBeenCalledWith(
Buffer.from("jpeg-bytes"),
"image/jpeg",
@@ -111,6 +167,39 @@ describe("stageIMessageAttachments", () => {
);
});
it("bounds pinned HEIC reads when the opened inode grows", async () => {
const sourcePath = await writeTempFile("IMG_0002.HEIC", Buffer.from("heic"));
const saveMediaBuffer = vi.fn();
const convertHeicToJpeg = vi.fn();
const logVerbose = vi.fn();
await expect(
stageIMessageAttachments(
[{ original_path: sourcePath, mime_type: "image/heic", missing: false }],
{
maxBytes: 4,
deps: {
saveMediaBuffer,
convertHeicToJpeg,
logVerbose,
openLocalFileSafely: async (options) => {
const opened = await openLocalFileSafely(options);
await fs.appendFile(sourcePath, Buffer.alloc(64));
return opened;
},
},
},
),
).resolves.toEqual({
attachments: [{ contentType: "image/heic", kind: "image" }],
unavailableCount: 1,
});
expect(convertHeicToJpeg).not.toHaveBeenCalled();
expect(saveMediaBuffer).not.toHaveBeenCalled();
expect(logVerbose).toHaveBeenCalledWith(expect.stringContaining("attachment exceeds"));
});
it("drops attachments over the inbound media limit", async () => {
const sourcePath = await writeTempFile("huge.png", Buffer.from("too large"));
const saveMediaBuffer = vi.fn();
extensions/imessage/src/monitor/media-staging.ts
+73 -57
View File
@@ -1,9 +1,11 @@
// Imessage plugin module implements media staging behavior.
import fs from "node:fs/promises";
import fs, { type FileHandle } from "node:fs/promises";
import path from "node:path";
import type { ChannelInboundMediaInput } from "openclaw/plugin-sdk/channel-inbound";
import { isInboundPathAllowed, kindFromMime } from "openclaw/plugin-sdk/media-runtime";
import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store";
import { openLocalFileSafely } from "openclaw/plugin-sdk/security-runtime";
import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path";
import { loadWebMedia } from "openclaw/plugin-sdk/web-media";
import type { IMessageAttachment } from "./types.js";
@@ -19,6 +21,7 @@ type SaveMediaBufferImpl = typeof saveMediaBuffer;
type StageIMessageAttachmentsDeps = {
saveMediaBuffer?: SaveMediaBufferImpl;
convertHeicToJpeg?: (sourcePath: string, maxBytes: number) => Promise<Buffer>;
openLocalFileSafely?: typeof openLocalFileSafely;
logVerbose?: (message: string) => void;
};
@@ -62,19 +65,33 @@ async function canonicalizeAllowedRoots(roots: readonly string[]): Promise<strin
return canonicalRoots;
}
async function resolveAllowedCanonicalAttachmentPath(params: {
attachmentPath: string;
async function assertAllowedCanonicalAttachmentPath(params: {
canonicalPath: string;
allowedRoots?: readonly string[];
}): Promise<string> {
}): Promise<void> {
if (!params.allowedRoots) {
return params.attachmentPath;
return;
}
const canonicalPath = await fs.realpath(params.attachmentPath);
const canonicalRoots = await canonicalizeAllowedRoots(params.allowedRoots);
if (!isInboundPathAllowed({ filePath: canonicalPath, roots: canonicalRoots })) {
if (!isInboundPathAllowed({ filePath: params.canonicalPath, roots: canonicalRoots })) {
throw new Error("attachment path resolves outside allowed roots");
}
return canonicalPath;
}
async function readPinnedAttachmentBytes(handle: FileHandle, maxBytes: number): Promise<Buffer> {
const chunks: Buffer[] = [];
let totalBytes = 0;
while (totalBytes <= maxBytes) {
const remaining = maxBytes + 1 - totalBytes;
const chunk = Buffer.allocUnsafe(Math.min(64 * 1024, remaining));
const { bytesRead } = await handle.read(chunk, 0, chunk.length, null);
if (bytesRead === 0) {
return Buffer.concat(chunks, totalBytes);
}
chunks.push(chunk.subarray(0, bytesRead));
totalBytes += bytesRead;
}
throw new Error(`attachment exceeds ${Math.round(maxBytes / (1024 * 1024))}MB limit`);
}
async function readAttachmentBuffer(params: {
@@ -84,58 +101,57 @@ async function readAttachmentBuffer(params: {
allowedRoots?: readonly string[];
deps: StageIMessageAttachmentsDeps;
}): Promise<{ buffer: Buffer; contentType?: string; originalFilename?: string }> {
const stat = await fs.lstat(params.attachmentPath);
if (stat.isSymbolicLink()) {
throw new Error("attachment path is a symlink");
}
if (!stat.isFile()) {
throw new Error("attachment path is not a file");
}
if (stat.size > params.maxBytes) {
throw new Error(`attachment exceeds ${Math.round(params.maxBytes / (1024 * 1024))}MB limit`);
}
const canonicalPath = await resolveAllowedCanonicalAttachmentPath({
attachmentPath: params.attachmentPath,
allowedRoots: params.allowedRoots,
const opened = await (params.deps.openLocalFileSafely ?? openLocalFileSafely)({
filePath: params.attachmentPath,
});
const canonicalStat = await fs.stat(canonicalPath);
if (!canonicalStat.isFile()) {
throw new Error("attachment path is not a file");
}
if (canonicalStat.size > params.maxBytes) {
throw new Error(`attachment exceeds ${Math.round(params.maxBytes / (1024 * 1024))}MB limit`);
}
if (isHeicAttachment(params.attachmentPath, params.mimeType)) {
try {
const convert = params.deps.convertHeicToJpeg;
const converted = convert
? {
buffer: await convert(canonicalPath, params.maxBytes),
fileName: jpegFilenameForAttachment(params.attachmentPath),
}
: await loadWebMedia(canonicalPath, {
maxBytes: params.maxBytes,
localRoots: [path.dirname(canonicalPath)],
});
return {
buffer: converted.buffer,
contentType: "image/jpeg",
originalFilename: converted.fileName ?? jpegFilenameForAttachment(params.attachmentPath),
};
} catch (err) {
params.deps.logVerbose?.(
`imessage: HEIC attachment conversion failed; staging original instead: ${String(err)}`,
);
try {
if (opened.stat.size > params.maxBytes) {
throw new Error(`attachment exceeds ${Math.round(params.maxBytes / (1024 * 1024))}MB limit`);
}
}
await assertAllowedCanonicalAttachmentPath({
canonicalPath: opened.realPath,
allowedRoots: params.allowedRoots,
});
// The inode can grow after the pinned open; keep the allocation bounded as well as the stat.
const buffer = await readPinnedAttachmentBytes(opened.handle, params.maxBytes);
return {
buffer: await fs.readFile(canonicalPath),
contentType: params.mimeType ?? undefined,
originalFilename: path.basename(params.attachmentPath),
};
if (isHeicAttachment(params.attachmentPath, params.mimeType)) {
try {
const convert = params.deps.convertHeicToJpeg;
const converted = await withTempWorkspace(
{ rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-imessage-heic-" },
async (workspace) => {
const pinnedPath = await workspace.write("attachment.heic", buffer);
return convert
? {
buffer: await convert(pinnedPath, params.maxBytes),
}
: await loadWebMedia(pinnedPath, {
maxBytes: params.maxBytes,
localRoots: [workspace.dir],
});
},
);
return {
buffer: converted.buffer,
contentType: "image/jpeg",
originalFilename: jpegFilenameForAttachment(params.attachmentPath),
};
} catch (err) {
params.deps.logVerbose?.(
`imessage: HEIC attachment conversion failed; staging original instead: ${String(err)}`,
);
}
}
return {
buffer,
contentType: params.mimeType ?? undefined,
originalFilename: path.basename(params.attachmentPath),
};
} finally {
await opened.handle.close().catch(() => undefined);
}
}
export async function stageIMessageAttachments(
extensions/memory-wiki/src/bounded-walk.test.ts
+68
View File
@@ -0,0 +1,68 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
const tempDirs: string[] = [];
afterEach(async () => {
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
});
async function createTempDir(): Promise<string> {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-walk-"));
tempDirs.push(dir);
return dir;
}
describe("walkMemoryWikiDirectory", () => {
it("fails instead of truncating at the entry budget", async () => {
const root = await createTempDir();
await Promise.all([
fs.writeFile(path.join(root, "one.md"), "one"),
fs.writeFile(path.join(root, "two.md"), "two"),
]);
await expect(walkMemoryWikiDirectory(root, "", { maxEntries: 1 })).rejects.toMatchObject({
code: "too-large",
});
});
it("treats a missing optional directory as empty", async () => {
const root = await createTempDir();
await expect(walkMemoryWikiDirectory(root, "missing")).resolves.toEqual([]);
});
it("prunes ignored subtrees before their descendants consume the budget", async () => {
const root = await createTempDir();
await fs.mkdir(path.join(root, "node_modules"));
await Promise.all(
Array.from({ length: 10 }, (_, index) =>
fs.writeFile(path.join(root, "node_modules", `ignored-${index}.md`), "ignored"),
),
);
await fs.writeFile(path.join(root, "visible.md"), "visible");
await expect(
walkMemoryWikiDirectory(root, "", {
maxEntries: 2,
entryFilter: (entry) =>
entry.kind === "directory" && path.basename(entry.relativePath) === "node_modules"
? "skip-subtree"
: "include",
}),
).resolves.toEqual([expect.objectContaining({ relativePath: "visible.md", kind: "file" })]);
});
it("reports directory failures when partial scans are requested", async () => {
const root = await createTempDir();
await expect(
walkMemoryWikiDirectory(root, "missing", { onDirectoryError: "skip-and-report" }),
).resolves.toEqual([
expect.objectContaining({ relativePath: "missing", kind: "directory-error" }),
]);
});
});
extensions/memory-wiki/src/bounded-walk.ts
+42
View File
@@ -0,0 +1,42 @@
import {
walkRootDirectory,
type RootWalkEntry,
type RootWalkOptions,
} from "openclaw/plugin-sdk/root-walk";
const MEMORY_WIKI_WALK_MAX_DEPTH = 128;
const MEMORY_WIKI_WALK_MAX_ENTRIES = 20_000;
type MemoryWikiWalkLimits = {
maxDepth?: number;
maxEntries?: number;
entryFilter?: RootWalkOptions["entryFilter"];
onDirectoryError?: RootWalkOptions["onDirectoryError"];
};
export async function walkMemoryWikiDirectory(
rootDir: string,
relativePath: string,
limits: MemoryWikiWalkLimits = {},
): Promise<RootWalkEntry[]> {
const entries: RootWalkEntry[] = [];
try {
for await (const entry of walkRootDirectory(rootDir, relativePath, {
maxDepth: limits.maxDepth ?? MEMORY_WIKI_WALK_MAX_DEPTH,
maxEntries: limits.maxEntries ?? MEMORY_WIKI_WALK_MAX_ENTRIES,
symlinkPolicy: "skip",
limitBehavior: "throw",
...(limits.entryFilter ? { entryFilter: limits.entryFilter } : {}),
...(limits.onDirectoryError ? { onDirectoryError: limits.onDirectoryError } : {}),
})) {
entries.push(entry);
}
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === "not-file" || code === "not-found") {
return [];
}
throw error;
}
return entries;
}
extensions/memory-wiki/src/compile.ts
+4 -9
View File
@@ -12,6 +12,7 @@ import {
normalizeLowercaseStringOrEmpty,
uniqueStrings,
} from "openclaw/plugin-sdk/string-coerce-runtime";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
import {
assessClaimFreshness,
assessPageFreshness,
@@ -375,16 +376,10 @@ export type RefreshMemoryWikiIndexesResult = {
};
async function collectMarkdownFiles(rootDir: string, relativeDir: string): Promise<string[]> {
const dirPath = path.join(rootDir, relativeDir);
const entries = await fs
.readdir(dirPath, { withFileTypes: true, recursive: true })
.catch(() => []);
const entries = await walkMemoryWikiDirectory(rootDir, relativeDir);
return entries
.filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
.map((entry) => {
const absPath = path.join(entry.parentPath ?? dirPath, entry.name);
return path.relative(rootDir, absPath).split(path.sep).join("/");
})
.filter((entry) => entry.kind === "file" && entry.relativePath.endsWith(".md"))
.map((entry) => entry.relativePath.split(path.sep).join("/"))
.filter((relativePath) => path.basename(relativePath) !== "index.md")
.toSorted((left, right) => left.localeCompare(right));
}
extensions/memory-wiki/src/import-runs-state.ts
+18 -11
View File
@@ -7,6 +7,7 @@ import type {
PluginStateKeyedStore,
} from "openclaw/plugin-sdk/plugin-state-runtime";
import pMap, { pMapSkip } from "p-map";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
const LEGACY_IMPORT_RUN_READ_CONCURRENCY = 16;
@@ -465,19 +466,25 @@ export async function readLegacyMemoryWikiImportRunRecords(
vaultRoot: string,
): Promise<ChatGptImportRunRecord[]> {
const importRunsDir = resolveMemoryWikiImportRunsDir(vaultRoot);
const entries = await fs
.readdir(importRunsDir, { withFileTypes: true })
.catch((error: unknown) => {
const code = asRecord(error)?.code;
if (code === "ENOENT") {
return [];
}
throw error;
});
const entries = await walkMemoryWikiDirectory(importRunsDir, "", {
maxDepth: 1,
entryFilter: (entry) =>
entry.kind === "directory"
? "skip-subtree"
: entry.kind === "file" && entry.relativePath.endsWith(".json")
? "include"
: "skip",
}).catch((error: unknown) => {
const code = asRecord(error)?.code;
if (code === "ENOENT") {
return [];
}
throw error;
});
return await pMap(
entries.filter((entry) => entry.isFile() && entry.name.endsWith(".json")),
entries.filter((entry) => entry.kind === "file"),
async (entry) => {
const raw = await fs.readFile(path.join(importRunsDir, entry.name), "utf8");
const raw = await fs.readFile(path.join(importRunsDir, entry.relativePath), "utf8");
return normalizeMemoryWikiImportRunRecord(JSON.parse(raw) as unknown) ?? pMapSkip;
},
{ concurrency: LEGACY_IMPORT_RUN_READ_CONCURRENCY, stopOnError: true },
extensions/memory-wiki/src/log.ts
+5 -15
View File
@@ -1,9 +1,9 @@
// Memory Wiki plugin module implements log behavior.
import { createHash, randomUUID } from "node:crypto";
import type { Dirent } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
type MemoryWikiLogEntry = {
type: "init" | "vault-generation" | "ingest" | "okf-import" | "compile" | "lint";
@@ -128,23 +128,13 @@ export async function resolveMemoryWikiVaultSourceGeneration(vaultRoot: string):
const files = (
await Promise.all(
COMPILED_SOURCE_DIRECTORIES.map(async (relativeDir) => {
const dirPath = path.join(vaultRoot, relativeDir);
let entries: Dirent[];
try {
entries = await fs.readdir(dirPath, { withFileTypes: true, recursive: true });
} catch (error) {
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
return [];
}
throw error;
}
const entries = await walkMemoryWikiDirectory(vaultRoot, relativeDir);
return entries
.filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
.filter((entry) => entry.kind === "file" && entry.relativePath.endsWith(".md"))
.map((entry) => {
const absolutePath = path.join(entry.parentPath ?? dirPath, entry.name);
return {
absolutePath,
relativePath: path.relative(vaultRoot, absolutePath).split(path.sep).join("/"),
absolutePath: path.join(vaultRoot, entry.relativePath),
relativePath: entry.relativePath.split(path.sep).join("/"),
};
})
.filter((entry) => path.basename(entry.relativePath) !== "index.md");
extensions/memory-wiki/src/okf.test.ts
+26
View File
@@ -193,6 +193,32 @@ describe("importMemoryWikiOkfBundle", () => {
expect(searchResults.map((searchResult) => searchResult.path)).toContain(ordersPath);
});
it("prunes repository metadata and dependency subtrees", async () => {
const rootDir = await createTempDir("memory-wiki-okf-prune-");
const bundlePath = path.join(rootDir, "pruned-okf");
await Promise.all([
fs.mkdir(path.join(bundlePath, "tables"), { recursive: true }),
fs.mkdir(path.join(bundlePath, ".git"), { recursive: true }),
fs.mkdir(path.join(bundlePath, "node_modules"), { recursive: true }),
]);
const concept = `---\ntype: BigQuery Table\ntitle: Included\n---\n\nIncluded body.\n`;
await Promise.all([
fs.writeFile(path.join(bundlePath, "tables", "included.md"), concept),
fs.writeFile(path.join(bundlePath, ".git", "ignored.md"), concept),
fs.writeFile(path.join(bundlePath, "node_modules", "ignored.md"), concept),
]);
const { config } = await createVault({ rootDir: path.join(rootDir, "vault") });
const result = await importMemoryWikiOkfBundle({
config,
bundlePath,
nowMs: Date.UTC(2026, 5, 12, 10, 0, 0),
});
expect(result.importedCount).toBe(1);
expect(result.pagePaths).toHaveLength(1);
});
it("caps generated concept filenames for long OKF concept paths", async () => {
const rootDir = await createTempDir("memory-wiki-okf-long-");
const bundlePath = path.join(rootDir, "long-okf");
extensions/memory-wiki/src/okf.ts
+28 -27
View File
@@ -8,6 +8,7 @@ import {
normalizeSingleOrTrimmedStringList,
uniqueStrings,
} from "openclaw/plugin-sdk/string-coerce-runtime";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
import { compileMemoryWikiVault } from "./compile.js";
import type { ResolvedMemoryWikiConfig } from "./config.js";
import { appendMemoryWikiLog } from "./log.js";
@@ -130,34 +131,30 @@ async function collectOkfMarkdownFiles(
rootDir: string,
warnings: ImportMemoryWikiOkfWarning[],
): Promise<string[]> {
async function walk(relativeDir: string): Promise<string[]> {
const absoluteDir = path.join(rootDir, relativeDir);
const entries = await fs.readdir(absoluteDir, { withFileTypes: true }).catch((err: unknown) => {
const entries = await walkMemoryWikiDirectory(rootDir, "", {
entryFilter: (entry) =>
entry.kind === "directory" &&
[".git", "node_modules"].includes(path.basename(entry.relativePath))
? "skip-subtree"
: "include",
onDirectoryError: "skip-and-report",
});
const files: string[] = [];
for (const entry of entries) {
if (entry.kind === "directory-error") {
warnings.push({
code: "unreadable-entry",
path: toPosixPath(relativeDir) || ".",
message: err instanceof Error ? err.message : "Unable to read OKF directory.",
path: toPosixPath(entry.relativePath) || ".",
message:
entry.error instanceof Error ? entry.error.message : "Unable to read OKF directory.",
});
return [];
});
const files: string[] = [];
for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) {
if (entry.name === ".git" || entry.name === "node_modules") {
continue;
}
const relativePath = path.join(relativeDir, entry.name);
if (entry.isDirectory()) {
files.push(...(await walk(relativePath)));
continue;
}
if (entry.isFile() && entry.name.endsWith(".md")) {
files.push(relativePath);
}
continue;
}
if (entry.kind === "file" && entry.relativePath.endsWith(".md")) {
files.push(toPosixPath(entry.relativePath));
}
return files;
}
return (await walk("")).map(toPosixPath).toSorted((left, right) => left.localeCompare(right));
return files.toSorted((left, right) => left.localeCompare(right));
}
function parseOkfMarkdown(
@@ -482,14 +479,18 @@ async function removeStaleOkfConceptPages(params: {
currentPagePaths: Set<string>;
}): Promise<string[]> {
const vault = await fsRoot(params.vaultRoot);
const conceptsDir = path.join(params.vaultRoot, "concepts");
const entries = await fs.readdir(conceptsDir, { withFileTypes: true }).catch(() => []);
const entries = await walkMemoryWikiDirectory(params.vaultRoot, "concepts", {
maxDepth: 0,
entryFilter: (entry) => (entry.kind === "directory" ? "skip-subtree" : "include"),
onDirectoryError: "skip-and-report",
});
const removedPagePaths: string[] = [];
for (const entry of entries) {
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
const entryName = path.basename(entry.relativePath);
if (entry.kind !== "file" || !entryName.endsWith(".md") || entryName === "index.md") {
continue;
}
const pagePath = `concepts/${entry.name}`;
const pagePath = `concepts/${entryName}`;
if (params.currentPagePaths.has(pagePath)) {
continue;
}
extensions/memory-wiki/src/query.ts
+7 -9
View File
@@ -20,6 +20,7 @@ import {
} from "openclaw/plugin-sdk/string-coerce-runtime";
import pMap, { pMapSkip } from "p-map";
import type { OpenClawConfig } from "../api.js";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
import { assessClaimFreshness, isClaimContestedStatus } from "./claim-health.js";
import {
loadMemoryWikiCompiledCache,
@@ -211,18 +212,15 @@ async function listWikiMarkdownFiles(rootDir: string): Promise<string[]> {
const files = (
await Promise.all(
QUERY_DIRS.map(async (relativeDir) => {
const dirPath = path.join(rootDir, relativeDir);
const entries = await fs
.readdir(dirPath, { withFileTypes: true, recursive: true })
.catch(() => []);
const entries = await walkMemoryWikiDirectory(rootDir, relativeDir);
return entries
.filter(
(entry) => entry.isFile() && entry.name.endsWith(".md") && entry.name !== "index.md",
(entry) =>
entry.kind === "file" &&
entry.relativePath.endsWith(".md") &&
path.basename(entry.relativePath) !== "index.md",
)
.map((entry) => {
const absPath = path.join(entry.parentPath ?? dirPath, entry.name);
return path.relative(rootDir, absPath).split(path.sep).join("/");
});
.map((entry) => entry.relativePath.split(path.sep).join("/"));
}),
)
).flat();
extensions/memory-wiki/src/status.ts
+9 -7
View File
@@ -4,6 +4,7 @@ import path from "node:path";
import { listActiveMemoryPublicArtifacts } from "openclaw/plugin-sdk/memory-host-core";
import { pathExists } from "openclaw/plugin-sdk/security-runtime";
import type { OpenClawConfig } from "../api.js";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
import { filterMemoryWikiBridgeArtifacts, resolveMemoryWikiVaultAgentId } from "./bridge.js";
import type { ResolvedMemoryWikiConfig } from "./config.js";
import { toWikiPageSummary, type WikiPageKind } from "./markdown.js";
@@ -91,16 +92,17 @@ async function collectVaultCounts(vaultPath: string): Promise<{
};
const dirs = ["entities", "concepts", "sources", "syntheses", "reports"] as const;
for (const dir of dirs) {
const dirPath = path.join(vaultPath, dir);
const entries = await fs
.readdir(dirPath, { withFileTypes: true, recursive: true })
.catch(() => []);
const entries = await walkMemoryWikiDirectory(vaultPath, dir);
for (const entry of entries) {
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
if (
entry.kind !== "file" ||
!entry.relativePath.endsWith(".md") ||
path.basename(entry.relativePath) === "index.md"
) {
continue;
}
const absolutePath = path.join(entry.parentPath ?? dirPath, entry.name);
const relativeToVault = path.relative(vaultPath, absolutePath).split(path.sep).join("/");
const absolutePath = path.join(vaultPath, entry.relativePath);
const relativeToVault = entry.relativePath.split(path.sep).join("/");
const raw = await fs.readFile(absolutePath, "utf8").catch(() => null);
if (raw === null) {
continue;
extensions/memory-wiki/src/unsafe-local.ts
+15 -16
View File
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
import pMap from "p-map";
import { walkMemoryWikiDirectory } from "./bounded-walk.js";
import type { BridgeMemoryWikiResult } from "./bridge.js";
import type { ResolvedMemoryWikiConfig } from "./config.js";
import { appendMemoryWikiLog } from "./log.js";
@@ -53,22 +54,20 @@ function detectFenceLanguage(filePath: string): string {
}
async function listAllowedFilesRecursive(rootDir: string): Promise<string[]> {
const entries = await fs.readdir(rootDir, { withFileTypes: true });
const files: string[] = [];
for (const entry of entries) {
const fullPath = path.join(rootDir, entry.name);
if (entry.isDirectory()) {
files.push(...(await listAllowedFilesRecursive(fullPath)));
continue;
}
if (
entry.isFile() &&
DIRECTORY_TEXT_EXTENSIONS.has(normalizeLowercaseStringOrEmpty(path.extname(entry.name)))
) {
files.push(fullPath);
}
}
return files.toSorted((left, right) => left.localeCompare(right));
const entries = await walkMemoryWikiDirectory(rootDir, "", {
entryFilter: (entry) =>
entry.kind === "directory" ||
(entry.kind === "file" &&
DIRECTORY_TEXT_EXTENSIONS.has(
normalizeLowercaseStringOrEmpty(path.extname(entry.relativePath)),
))
? "include"
: "skip",
});
return entries
.filter((entry) => entry.kind === "file")
.map((entry) => path.join(rootDir, entry.relativePath))
.toSorted((left, right) => left.localeCompare(right));
}
async function collectUnsafeLocalArtifacts(
extensions/msteams/src/sdk.test.ts
+14 -10
View File
@@ -1,16 +1,19 @@
// Msteams tests cover sdk plugin behavior.
import * as fs from "node:fs";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createMSTeamsTokenProvider, loadMSTeamsSdkWithAuth } from "./sdk.js";
import type { MSTeamsCredentials, MSTeamsFederatedCredentials } from "./token.js";
vi.mock("node:fs", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs")>();
const { readSecretFile } = vi.hoisted(() => ({
readSecretFile: vi
.fn<(filePath: string, label: string) => Promise<string>>()
.mockResolvedValue("-----BEGIN RSA PRIVATE KEY-----\nfake-key\n-----END RSA PRIVATE KEY-----"),
}));
vi.mock("openclaw/plugin-sdk/secret-file", async (importOriginal) => {
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/secret-file")>();
return {
...actual,
readFileSync: vi.fn(
() => "-----BEGIN RSA PRIVATE KEY-----\nfake-key\n-----END RSA PRIVATE KEY-----",
),
readSecretFile,
};
});
@@ -33,6 +36,9 @@ vi.mock("@azure/identity", () => {
afterEach(() => {
vi.restoreAllMocks();
readSecretFile
.mockReset()
.mockResolvedValue("-----BEGIN RSA PRIVATE KEY-----\nfake-key\n-----END RSA PRIVATE KEY-----");
});
async function createMSTeamsApp(...args: Parameters<typeof loadMSTeamsSdkWithAuth>) {
@@ -76,13 +82,11 @@ describe("createMSTeamsApp", () => {
const app = await createMSTeamsApp(creds);
expect(app).toBeDefined();
expect(fs.readFileSync).toHaveBeenCalledWith("/path/to/cert.pem", "utf-8");
expect(readSecretFile).toHaveBeenCalledWith("/path/to/cert.pem", "Microsoft Teams certificate");
});
it("throws when certificate file is missing", async () => {
vi.mocked(fs.readFileSync).mockImplementation(() => {
throw new Error("ENOENT: no such file");
});
readSecretFile.mockRejectedValue(new Error("ENOENT: no such file"));
const creds: MSTeamsFederatedCredentials = {
type: "federated",
extensions/msteams/src/sdk.ts
+5 -5
View File
@@ -1,6 +1,6 @@
// Msteams plugin module implements sdk behavior.
import * as fs from "node:fs";
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
import { readSecretFile } from "openclaw/plugin-sdk/secret-file";
import { normalizeBotFrameworkServiceUrl } from "./bot-framework-service-url.js";
import type { MSTeamsCloudName } from "./cloud.js";
import { MSTEAMS_REQUEST_TIMEOUT_MS } from "./request-timeout.js";
@@ -285,7 +285,7 @@ async function createMSTeamsApp(
};
if (creds.type === "federated") {
return createFederatedApp(creds, App, appOptions);
return await createFederatedApp(creds, App, appOptions);
}
return new App({
clientId: creds.appId,
@@ -295,11 +295,11 @@ async function createMSTeamsApp(
} as ConstructorParameters<typeof App>[0]) as unknown as MSTeamsApp;
}
function createFederatedApp(
async function createFederatedApp(
creds: MSTeamsFederatedCredentials,
App: typeof import("@microsoft/teams.apps").App,
appOptions: Record<string, unknown>,
): MSTeamsApp {
): Promise<MSTeamsApp> {
if (creds.useManagedIdentity) {
// The SDK handles managed identity natively — pass managedIdentityClientId
// and it selects the right credential flow (system MI, user MI, or FIC).
@@ -319,7 +319,7 @@ function createFederatedApp(
let privateKey: string;
try {
privateKey = fs.readFileSync(creds.certificatePath, "utf-8");
privateKey = await readSecretFile(creds.certificatePath, "Microsoft Teams certificate");
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
throw new Error(`Failed to read certificate file at '${creds.certificatePath}': ${msg}`, {
package.json
+14 -4
View File
@@ -409,6 +409,18 @@
"types": "./dist/plugin-sdk/setup-tools.d.ts",
"default": "./dist/plugin-sdk/setup-tools.js"
},
"./plugin-sdk/archive": {
"types": "./dist/plugin-sdk/archive.d.ts",
"default": "./dist/plugin-sdk/archive.js"
},
"./plugin-sdk/root-walk": {
"types": "./dist/plugin-sdk/root-walk.d.ts",
"default": "./dist/plugin-sdk/root-walk.js"
},
"./plugin-sdk/secret-file": {
"types": "./dist/plugin-sdk/secret-file.d.ts",
"default": "./dist/plugin-sdk/secret-file.js"
},
"./plugin-sdk/approval-auth-runtime": {
"types": "./dist/plugin-sdk/approval-auth-runtime.d.ts",
"default": "./dist/plugin-sdk/approval-auth-runtime.js"
@@ -1913,7 +1925,7 @@
"@modelcontextprotocol/sdk": "1.29.0",
"@mozilla/readability": "0.6.0",
"@openclaw/ai": "workspace:*",
"@openclaw/fs-safe": "0.4.7",
"@openclaw/fs-safe": "0.5.0",
"@openclaw/proxyline": "0.3.4",
"@silvia-odwyer/photon-node": "0.3.4",
"acorn": "8.17.0",
@@ -1947,12 +1959,11 @@
"partial-json": "0.1.7",
"playwright-core": "1.61.1",
"pretty-ms": "9.3.0",
"proper-lockfile": "4.1.2",
"qrcode": "1.5.4",
"quickjs-wasi": "3.0.2",
"rastermill": "0.3.1",
"semver": "7.8.5",
"tar": "7.5.20",
"tar": "7.5.21",
"tree-sitter-bash": "0.25.1",
"tslog": "4.11.0",
"typebox": "1.3.6",
@@ -1978,7 +1989,6 @@
"@types/markdown-it": "14.1.2",
"@types/ms": "2.1.0",
"@types/node": "26.1.1",
"@types/proper-lockfile": "4.1.4",
"@types/semver": "7.7.1",
"@types/web-push": "3.6.4",
"@types/ws": "8.18.1",
packages/memory-host-sdk/src/host/fs-utils.ts
+9 -6
View File
@@ -1,6 +1,6 @@
// Memory Host SDK helper module supports fs utils behavior.
import { configureFsSafePython } from "@openclaw/fs-safe/config";
// fs-safe facade with Python validation disabled by default for this package's
import { configureFsSafeNative } from "@openclaw/fs-safe/config";
// fs-safe facade with native acceleration disabled by default for this package's
// host-side memory file operations.
export { root } from "@openclaw/fs-safe/root";
export { isPathInside, isPathInsideWithRealpath } from "@openclaw/fs-safe/path";
@@ -11,11 +11,14 @@ export {
} from "@openclaw/fs-safe/advanced";
export { walkDirectory, type WalkDirectoryEntry } from "@openclaw/fs-safe/walk";
const hasPythonModeOverride =
process.env.FS_SAFE_PYTHON_MODE != null || process.env.OPENCLAW_FS_SAFE_PYTHON_MODE != null;
const hasModeOverride = Object.keys(process.env).some((key) =>
/^(?:OPENCLAW_)?FS_SAFE_(?:NATIVE|PYTHON)_MODE$/u.test(
process.platform === "win32" ? key.toUpperCase() : key,
),
);
if (!hasPythonModeOverride) {
configureFsSafePython({ mode: "off" });
if (!hasModeOverride) {
configureFsSafeNative({ mode: "off" });
}
/** True for missing-file errors emitted by Node or fs-safe. */
pnpm-lock.yaml
Generated
+19 -28
View File
@@ -30,7 +30,7 @@ overrides:
qs: 6.15.3
node-domexception: npm:@nolyfill/domexception@1.0.28
typebox: 1.3.6
tar: 7.5.20
tar: 7.5.21
tough-cookie: 4.1.4
yauzl: 3.4.0
protobufjs: 7.6.5
@@ -85,8 +85,8 @@ importers:
specifier: workspace:*
version: link:packages/ai
'@openclaw/fs-safe':
specifier: 0.4.7
version: 0.4.7
specifier: 0.5.0
version: 0.5.0
'@openclaw/proxyline':
specifier: 0.3.4
version: 0.3.4(undici@8.6.0)
@@ -186,9 +186,6 @@ importers:
pretty-ms:
specifier: 9.3.0
version: 9.3.0
proper-lockfile:
specifier: 4.1.2
version: 4.1.2
qrcode:
specifier: 1.5.4
version: 1.5.4
@@ -202,8 +199,8 @@ importers:
specifier: 7.8.5
version: 7.8.5
tar:
specifier: 7.5.20
version: 7.5.20
specifier: 7.5.21
version: 7.5.21
tree-sitter-bash:
specifier: 0.25.1
version: 0.25.1
@@ -274,9 +271,6 @@ importers:
'@types/node':
specifier: 26.1.1
version: 26.1.1
'@types/proper-lockfile':
specifier: 4.1.4
version: 4.1.4
'@types/semver':
specifier: 7.7.1
version: 7.7.1
@@ -3923,6 +3917,10 @@ packages:
resolution: {integrity: sha512-UO+FCB8vF+VJngOzZuVTdkOapX/gPml0HFEjRj2qIeEmBlCkGLHjRJ4CZWFziRtcq3g4YioWra/6J3iwEH1JaA==}
engines: {node: '>=22'}
'@openclaw/fs-safe@0.5.0':
resolution: {integrity: sha512-TPpFG8PkAKlM/eP7glzHhCeZnDX3sSqyjXgeoc0jeyii0y957/zePB62BIwc75gOFmxaeKLWkd4iF41/Ghlgjg==}
engines: {node: '>=22'}
'@openclaw/libterminal@0.3.2':
resolution: {integrity: sha512-RbxYPG22kuEgvW2Gx8FK2uApYSgDi+yRahfR0lW7pcdMZvAWL9/iLJRVLEMKc1uAA69mAA/IW5AtwQ9LUdHypA==}
engines: {node: ^22.18.0 || >=24.11.0}
@@ -5154,9 +5152,6 @@ packages:
'@types/node@26.1.1':
resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==}
'@types/proper-lockfile@4.1.4':
resolution: {integrity: sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==}
'@types/qs@6.15.1':
resolution: {integrity: sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==}
@@ -5169,9 +5164,6 @@ packages:
'@types/retry@0.12.0':
resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==}
'@types/retry@0.12.5':
resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==}
'@types/sarif@2.1.7':
resolution: {integrity: sha512-kRz0VEkJqWLf1LLVN4pT1cg1Z9wAuvI6L97V3m2f5B76Tg8d413ddvLBPTEHAZJlnn4XSvu0FkZtViCQGVyrXQ==}
@@ -8433,8 +8425,8 @@ packages:
tar-stream@3.2.0:
resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==}
tar@7.5.20:
resolution: {integrity: sha512-9FcyK4PA6+WbzlTM9WhQm6vB5W7cP7dUiPsv1g7YDwEQnQ1CGpK3MGlKk/ITVWMk05kHZuBhmVhiv8LZoy/PFQ==}
tar@7.5.21:
resolution: {integrity: sha512-XdhtCvlMywwxpCW8YEq3lOXBJpUPTR2OHHcwLPO3HwsJqOHa2Ok/oJ7ruGzp+JrKoRPVCzJwAdEjqLW/vNRPHA==}
engines: {node: '>=18'}
teex@1.0.1:
@@ -10913,7 +10905,12 @@ snapshots:
'@openclaw/fs-safe@0.4.7':
optionalDependencies:
jszip: 3.10.1
tar: 7.5.20
tar: 7.5.21
'@openclaw/fs-safe@0.5.0':
optionalDependencies:
jszip: 3.10.1
tar: 7.5.21
'@openclaw/libterminal@0.3.2':
dependencies:
@@ -11995,10 +11992,6 @@ snapshots:
dependencies:
undici-types: 8.3.0
'@types/proper-lockfile@4.1.4':
dependencies:
'@types/retry': 0.12.5
'@types/qs@6.15.1': {}
'@types/range-parser@1.2.7': {}
@@ -12009,8 +12002,6 @@ snapshots:
'@types/retry@0.12.0': {}
'@types/retry@0.12.5': {}
'@types/sarif@2.1.7': {}
'@types/semver@7.7.1': {}
@@ -12650,7 +12641,7 @@ snapshots:
node-api-headers: 1.9.0
rc: 1.2.8
semver: 7.8.5
tar: 7.5.20
tar: 7.5.21
url-join: 4.0.1
which: 6.0.1
yargs: 17.7.3
@@ -15923,7 +15914,7 @@ snapshots:
- bare-buffer
- react-native-b4a
tar@7.5.20:
tar@7.5.21:
dependencies:
'@isaacs/fs-minipass': 4.0.1
chownr: 3.0.0
pnpm-workspace.yaml
+2 -2
View File
@@ -9,7 +9,7 @@ minimumReleaseAge: 2880
minimumReleaseAgeExclude:
- "@openclaw/crabline@0.1.11"
- "@openclaw/fs-safe@0.4.7"
- "@openclaw/fs-safe@0.5.0"
- "@openclaw/libterminal@0.3.2"
- "@openclaw/proxyline@0.3.4"
- "@openclaw/uirouter@0.1.0"
@@ -141,7 +141,7 @@ overrides:
qs: 6.15.3
node-domexception: "npm:@nolyfill/domexception@1.0.28"
typebox: 1.3.6
tar: 7.5.20
tar: 7.5.21
tough-cookie: 4.1.4
yauzl: 3.4.0
protobufjs: 7.6.5
scripts/lib/dependency-ownership.json
-5
View File
@@ -234,11 +234,6 @@
"class": "core-runtime",
"risk": ["formatting"]
},
"proper-lockfile": {
"owner": "core:session-storage",
"class": "core-runtime",
"risk": ["filesystem-locking"]
},
"quickjs-wasi": {
"owner": "core:code-mode",
"class": "core-runtime",
scripts/lib/plugin-sdk-entrypoints.json
+3
View File
@@ -13,6 +13,9 @@
"channel-setup",
"channel-streaming",
"setup-tools",
"archive",
"root-walk",
"secret-file",
"approval-auth-runtime",
"approval-client-runtime",
"approval-delivery-runtime",
scripts/plugin-sdk-surface-report.mjs
+13 -4
View File
@@ -153,7 +153,10 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +1: focused media-local-roots replacement for the legacy agent-media facade.
// +1: account-aware channel DM policy setup descriptors.
// +1: dependency-light CLI argv parsing for machine-output metadata.
143,
// +1: bounded archive extraction and single-entry reads.
// +1: budgeted root-bounded directory walking.
// +1: pinned secret reads and first-writer-wins creation.
146,
env,
),
publicExports: readPluginSdkSurfaceBudgetEnv(
@@ -189,7 +192,10 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +1: logger file-transport flush for graceful shutdown drains.
// +1: process-local sessions.changed plugin notification payload.
// +1: loopback-only host classifier for plugin local-machine boundaries.
4740,
// +7: bounded archive extraction, entry reads, errors, and policy types.
// +3: root-bounded walk iterator, options, and entry contract.
// +5: pinned secret create/read functions and their options contract.
4755,
env,
),
publicFunctionExports: readPluginSdkSurfaceBudgetEnv(
@@ -219,7 +225,10 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +1: authoritative model-picker session-apply operation.
// +1: logger file-transport flush for graceful shutdown drains.
// +1: loopback-only host classifier for plugin local-machine boundaries.
2869,
// +2: bounded archive extraction and single-entry reads.
// +1: root-bounded directory walk iterator.
// +4: pinned secret create and synchronous/asynchronous reads.
2876,
env,
),
publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv(
@@ -234,7 +243,7 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
),
publicWildcardReexports: readPluginSdkSurfaceBudgetEnv(
"OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_WILDCARD_REEXPORTS",
83,
82,
env,
),
};
src/agents/cli-runner/helpers.ts
+1 -1
View File
@@ -407,7 +407,7 @@ export async function writeCliSystemPromptFile(params: {
);
return {
filePath,
cleanup: async () => await workspace.cleanup(),
cleanup: () => workspace.cleanup().then(() => undefined),
};
}
src/agents/embedded-agent-runner/run/attempt.session-lock.ts
+7 -1
View File
@@ -15,7 +15,13 @@ import type {
type SessionLock = Awaited<ReturnType<typeof acquireSessionWriteLock>>;
type AcquireSessionWriteLock = typeof acquireSessionWriteLock;
type LockOptions = Parameters<AcquireSessionWriteLock>[0];
type SessionKeyLockOptions = Extract<
Parameters<AcquireSessionWriteLock>[0],
{ targetKind: "session-key" }
>;
type LockOptions = Omit<SessionKeyLockOptions, "targetKind"> & {
targetKind?: "session-key";
};
const PROMPT_DISPOSE_SETTLE_TIMEOUT_MS = 5_000;
export type EmbeddedAttemptSessionFileOwner = {
src/agents/sandbox-paths.test.ts
+3 -3
View File
@@ -135,7 +135,7 @@ describe("assertSandboxPath", () => {
const escapedRead = `${root}/sub/up/../outside/secret.txt`;
await expect(fs.readFile(escapedRead, "utf8")).resolves.toBe("outside");
await expect(assertSandboxPath({ filePath: escapedRead, cwd: root, root })).rejects.toThrow(
/escapes sandbox root/i,
/(?:resolves outside|escapes) sandbox root/i,
);
await expect(
assertSandboxPath({
@@ -143,10 +143,10 @@ describe("assertSandboxPath", () => {
cwd: root,
root,
}),
).rejects.toThrow(/escapes sandbox root/i);
).rejects.toThrow(/(?:resolves outside|escapes) sandbox root/i);
await expect(
assertSandboxPath({ filePath: `${root}/sub/up/../..`, cwd: root, root }),
).rejects.toThrow(/escapes sandbox root/i);
).rejects.toThrow(/(?:resolves outside|escapes) sandbox root/i);
await fs.mkdir(path.join(root, "a"));
await fs.mkdir(path.join(root, "b"));
src/agents/sandbox/fs-bridge-mutation-helper.test.ts
+1 -1
View File
@@ -1,4 +1,4 @@
// Pinned mutation helper tests cover the Python helper that performs sandbox
// Pinned mutation helper tests cover the native helper that performs sandbox
// filesystem mutations through directory file descriptors.
import { spawnSync } from "node:child_process";
import fs from "node:fs/promises";
src/agents/sandbox/registry.ts
-1
View File
@@ -354,7 +354,6 @@ function normalizeSandboxRegistryEntry(entry: SandboxRegistryEntry): SandboxRegi
async function withRegistryLock<T>(registryPath: string, fn: () => Promise<T>): Promise<T> {
const lock = await acquireSessionWriteLock({
sessionFile: registryPath,
allowReentrant: false,
timeoutMs: 60_000,
});
try {
src/agents/session-write-lock.test.ts
+66 -6
View File
@@ -157,7 +157,6 @@ async function expectActiveInProcessLockIsNotReclaimed(params?: {
acquireSessionWriteLock({
sessionFile,
timeoutMs: 5,
allowReentrant: false,
}),
).rejects.toThrow(/session file locked/);
await lock.release();
@@ -191,15 +190,16 @@ describe("acquireSessionWriteLock", () => {
it("reuses locks across symlinked session paths", async () => {
await withSymlinkedSessionPaths(
async ({ sessionReal, sessionLink, realLockPath, linkLockPath }) => {
const reentrantOwner = "session:symlink-alias-test";
const lockA = await acquireSessionWriteLock({
sessionFile: sessionReal,
timeoutMs: 500,
allowReentrant: true,
reentrantOwner,
});
const lockB = await acquireSessionWriteLock({
sessionFile: sessionLink,
timeoutMs: 500,
allowReentrant: true,
reentrantOwner,
});
await expect(fs.access(realLockPath)).resolves.toBeUndefined();
@@ -220,15 +220,16 @@ describe("acquireSessionWriteLock", () => {
it("keeps the lock file until the last release", async () => {
await withTempSessionLockFile(async ({ sessionFile, lockPath }) => {
const reentrantOwner = "session:final-release-test";
const lockA = await acquireSessionWriteLock({
sessionFile,
timeoutMs: 500,
allowReentrant: true,
reentrantOwner,
});
const lockB = await acquireSessionWriteLock({
sessionFile,
timeoutMs: 500,
allowReentrant: true,
reentrantOwner,
});
await expectLockRemovedOnlyAfterFinalRelease({
@@ -730,7 +731,6 @@ describe("acquireSessionWriteLock", () => {
sessionFile,
timeoutMs: 5,
staleMs: 60_000,
allowReentrant: false,
}),
).rejects.toThrow(/session file locked/);
await expect(fs.access(lockPath)).resolves.toBeUndefined();
@@ -1204,6 +1204,66 @@ describe("acquireSessionWriteLock", () => {
}
});
it("preserves a fresh session lock that replaces the stale sweep candidate", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-lock-replacement-"));
const sessionsDir = path.join(root, "sessions");
await fs.mkdir(sessionsDir, { recursive: true });
const nowMs = Date.now();
const lockPath = path.join(sessionsDir, "replaced.jsonl.lock");
const replacementOwner = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {
stdio: "ignore",
});
if (!replacementOwner.pid) {
throw new Error("missing replacement lock owner pid");
}
const replacement = {
pid: replacementOwner.pid,
createdAt: new Date(nowMs).toISOString(),
};
await fs.writeFile(
lockPath,
JSON.stringify({
pid: process.pid,
createdAt: new Date(nowMs - 120_000).toISOString(),
}),
"utf8",
);
const originalReadFile = fs.readFile.bind(fs);
let lockReads = 0;
const readFileSpy = vi.spyOn(fs, "readFile").mockImplementation((async (filePath, options) => {
const resolvedPath = readFilePathToString(filePath);
const raw = await originalReadFile(filePath, options as never);
if (resolvedPath && path.resolve(resolvedPath) === lockPath) {
lockReads += 1;
if (lockReads === 3) {
fsSync.writeFileSync(lockPath, JSON.stringify(replacement), "utf8");
}
}
return raw;
}) as typeof fs.readFile);
try {
const result = await cleanStaleLockFiles({
sessionsDir,
staleMs: 30_000,
nowMs,
removeStale: true,
readOwnerProcessArgs: (pid) =>
pid === replacementOwner.pid
? ["node", "/opt/openclaw/openclaw.mjs", "agent"]
: ["python", "worker.py"],
});
expect(result.cleaned).toEqual([]);
expect(result.locks).toMatchObject([{ removed: false, stale: true }]);
expect(JSON.parse(await fs.readFile(lockPath, "utf8"))).toEqual(replacement);
} finally {
readFileSpy.mockRestore();
replacementOwner.kill("SIGTERM");
await fs.rm(root, { recursive: true, force: true });
}
});
it("cleans old live .jsonl lock files owned by non-OpenClaw processes", async () => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-lock-"));
const sessionsDir = path.join(root, "sessions");
src/agents/session-write-lock.ts
+79 -18
View File
@@ -507,6 +507,7 @@ type LockInspectionDetails = Pick<
>;
const SESSION_LOCKS = createFileLockManager("openclaw.session-write-lock");
const SESSION_LOCK_SWEEPS = createFileLockManager("openclaw.session-write-lock-sweep");
function isFileLockError(error: unknown, code: string): boolean {
return (error as { code?: unknown } | null)?.code === code;
@@ -828,6 +829,14 @@ function parseLockPayload(raw: string): LockFilePayload | null {
return payload;
}
function parseLockPayloadOrNull(raw: string): LockFilePayload | null {
try {
return parseLockPayload(raw);
} catch {
return null;
}
}
async function readLockPayload(lockPath: string): Promise<LockFilePayload | null> {
try {
const raw = await fs.readFile(lockPath, "utf8");
@@ -1104,16 +1113,46 @@ async function shouldRetryStaleAcquireFailure(params: {
}));
}
async function shouldRemoveLockDuringCleanup(
async function reclaimSessionLockIfUnchanged(
lockPath: string,
details: LockInspectionDetails,
staleMs: number,
nowMs: number,
ownerProcessArgsReader: SessionLockOwnerProcessArgsReader,
): Promise<boolean> {
if (!details.stale) {
return false;
const shouldReclaim = async (payload: LockFilePayload | null) => {
const inspected = inspectLockPayloadForSession({
payload,
staleMs,
nowMs,
heldByThisProcess: false,
reclaimLockWithoutStarttime: false,
readOwnerProcessArgs: ownerProcessArgsReader,
});
return (
inspected.stale && (await shouldRemoveContendedLockFile(lockPath, inspected, staleMs, nowMs))
);
};
try {
const lock = await SESSION_LOCK_SWEEPS.acquire(lockPath, {
lockPath,
staleMs,
timeoutMs: 0,
retry: { retries: 0 },
staleRecovery: "remove-if-unchanged",
payload: () => ({ pid: process.pid, createdAt: new Date().toISOString() }),
parsePayload: parseLockPayloadOrNull,
shouldReclaim: ({ payload }) => shouldReclaim(payload as LockFilePayload | null),
shouldRemoveStaleLock: ({ payload }) => shouldReclaim(payload as LockFilePayload | null),
});
await lock.release();
return true;
} catch (error) {
const code = (error as { code?: unknown }).code;
if (code === "file_lock_timeout" || code === "file_lock_stale") {
return false;
}
throw error;
}
return await shouldRemoveContendedLockFile(lockPath, details, staleMs, nowMs);
}
function sessionLockHeldByThisProcess(normalizedSessionFile: string): boolean {
@@ -1279,7 +1318,8 @@ export async function cleanStaleLockFiles(params: {
reclaimLockWithoutStarttime: false,
readOwnerProcessArgs: ownerProcessArgsReader,
});
const removable = await shouldRemoveLockDuringCleanup(lockPath, inspected, staleMs, nowMs);
const removable =
inspected.stale && (await shouldRemoveContendedLockFile(lockPath, inspected, staleMs, nowMs));
const lockInfo: SessionLockInspection = {
lockPath,
...inspected,
@@ -1288,12 +1328,18 @@ export async function cleanStaleLockFiles(params: {
};
if (removeStale && removable) {
await fs.rm(lockPath, { force: true });
lockInfo.removed = true;
cleaned.push(lockInfo);
params.log?.warn?.(
`removed stale session lock: ${lockPath} (${lockInfo.staleReasons.join(", ") || "unknown"})`,
lockInfo.removed = await reclaimSessionLockIfUnchanged(
lockPath,
staleMs,
nowMs,
ownerProcessArgsReader,
);
if (lockInfo.removed) {
cleaned.push(lockInfo);
params.log?.warn?.(
`removed stale session lock: ${lockPath} (${lockInfo.staleReasons.join(", ") || "unknown"})`,
);
}
}
locks.push(lockInfo);
@@ -1302,15 +1348,31 @@ export async function cleanStaleLockFiles(params: {
return { locks, cleaned };
}
export async function acquireSessionWriteLock(params: {
type AcquireSessionWriteLockBaseParams = {
sessionFile: string;
timeoutMs?: number;
staleMs?: number;
maxHoldMs?: number;
allowReentrant?: boolean;
signal?: AbortSignal;
targetKind?: "file" | "session-key";
}): Promise<{
};
type AcquireSessionWriteLockParams = AcquireSessionWriteLockBaseParams &
(
| {
targetKind: "session-key";
/** Reuse the process-local SQLite lease held for this session key. */
allowReentrant?: boolean;
reentrantOwner?: never;
}
| {
targetKind?: "file";
/** Process-local identity for intentional nesting on retained file-lock targets. */
reentrantOwner?: string;
allowReentrant?: never;
}
);
export async function acquireSessionWriteLock(params: AcquireSessionWriteLockParams): Promise<{
assertOwned?: () => void;
release: () => Promise<void>;
}> {
@@ -1326,7 +1388,6 @@ export async function acquireSessionWriteLock(params: {
throw error;
};
throwIfAborted();
const allowReentrant = params.allowReentrant ?? false;
const defaultOptions = resolveSessionWriteLockOptions();
const timeoutMs = resolvePositiveMs(params.timeoutMs, defaultOptions.timeoutMs, {
allowInfinity: true,
@@ -1339,7 +1400,7 @@ export async function acquireSessionWriteLock(params: {
sessionKey: params.sessionFile,
timeoutMs,
maxHoldMs,
allowReentrant,
allowReentrant: params.allowReentrant ?? false,
...(params.signal ? { signal: params.signal } : {}),
});
}
@@ -1379,7 +1440,7 @@ export async function acquireSessionWriteLock(params: {
timeoutMs: acquireAttemptTimeoutMs,
retry: { minTimeout: 50, maxTimeout: 1000, factor: 1 },
staleRecovery: "remove-if-unchanged",
allowReentrant,
reentrantOwner: params.reentrantOwner,
metadata: { maxHoldMs },
payload: () => {
const createdAt = new Date().toISOString();
src/agents/sessions/storage-lock.ts
+77 -15
View File
@@ -1,21 +1,83 @@
import lockfile from "proper-lockfile";
import fs from "node:fs";
import {
acquireFileLockSync,
type FileLockSyncAcquireOptions,
type FileLockSyncHandle,
} from "../../infra/file-lock-manager.js";
import { isLockOwnerDefinitelyStale } from "../../infra/stale-lock-file.js";
import { getFileLockProcessStartTime } from "../../shared/pid-alive.js";
const MAX_LOCK_ATTEMPTS = 10;
const LOCK_RETRY_DELAY_MS = 20;
const STORAGE_LOCK_STALE_MS = 30_000;
let currentProcessStartTime: number | null | undefined;
export function acquireLockSyncWithRetry(path: string): () => void {
for (let attempt = 1; ; attempt++) {
try {
return lockfile.lockSync(path, { realpath: false });
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ELOCKED" || attempt === MAX_LOCK_ATTEMPTS) {
throw error;
}
}
const start = Date.now();
while (Date.now() - start < LOCK_RETRY_DELAY_MS) {
// proper-lockfile rejects sync retries; preserve the bounded sync storage contract.
}
}
const lock = acquireStorageLockSyncWithRetry(path);
return () => lock.release();
}
function acquireStorageLockSyncWithRetry(path: string): FileLockSyncHandle {
prepareStorageLockPathForFsSafe(path);
return acquireFileLockSync(path, createStorageLockOptions());
}
function createStorageLockPayload(): Record<string, unknown> {
currentProcessStartTime ??= getFileLockProcessStartTime(process.pid);
return {
pid: process.pid,
createdAt: new Date().toISOString(),
...(currentProcessStartTime === null ? {} : { starttime: currentProcessStartTime }),
};
}
function storageLockOwnerIsStale(payload: unknown): boolean {
return isLockOwnerDefinitelyStale({
payload:
payload && typeof payload === "object" && !Array.isArray(payload)
? (payload as Record<string, unknown>)
: null,
});
}
function prepareStorageLockPathForFsSafe(targetPath: string): void {
const lockPath = `${targetPath}.lock`;
let observed: fs.Stats;
try {
observed = fs.lstatSync(lockPath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return;
}
throw error;
}
if (observed.isFile() && !observed.isSymbolicLink()) {
return;
}
if (!observed.isDirectory() || observed.isSymbolicLink()) {
throw new Error(`Storage lock path has an unsupported legacy type: ${lockPath}`);
}
throw Object.assign(
new Error(
`Legacy storage lock requires manual removal after verifying no older OpenClaw process is running: ${lockPath}`,
),
{ code: "file_lock_stale", lockPath },
);
}
function createStorageLockOptions(): FileLockSyncAcquireOptions<Record<string, unknown>> {
return {
staleMs: STORAGE_LOCK_STALE_MS,
retry: {
retries: MAX_LOCK_ATTEMPTS - 1,
factor: 1,
minTimeout: LOCK_RETRY_DELAY_MS,
maxTimeout: LOCK_RETRY_DELAY_MS,
randomize: false,
},
staleRecovery: "remove-if-unchanged",
payload: createStorageLockPayload,
shouldReclaim: ({ payload }) => storageLockOwnerIsStale(payload),
shouldRemoveStaleLock: ({ payload }) => storageLockOwnerIsStale(payload),
};
}
src/config/sessions/transcript-append.test-support.ts
+1 -1
View File
@@ -520,7 +520,7 @@ async function withSessionTranscriptWriteLock<T>(
const lock = await acquireSessionWriteLock({
sessionFile: params.transcriptPath,
...resolveSessionWriteLockOptions(params.config),
allowReentrant: true,
reentrantOwner: `session:${path.resolve(params.transcriptPath)}:append:${randomUUID()}`,
});
try {
return await run();
src/fleet/backup.runtime.test.ts
+29
View File
@@ -374,6 +374,35 @@ describe("fleet restore runtime", () => {
expect(containers.remove).not.toHaveBeenCalled();
});
it("rejects an implicitly deep entry before creating output paths", async () => {
const source = path.join(root, `deep-${crypto.randomUUID()}`);
const deepRelativePath = path.posix.join(
"data",
...Array.from({ length: 6 }, (_, index) => `segment-${index}`),
"state.txt",
);
await fs.mkdir(path.join(source, path.dirname(deepRelativePath)), { recursive: true });
await fs.mkdir(path.join(source, "auth"));
await fs.writeFile(
path.join(source, "manifest.json"),
JSON.stringify({ schemaVersion: 1, kind: "openclaw-fleet-cell-backup", tenant: "acme" }),
);
await fs.writeFile(path.join(source, deepRelativePath), "state");
const archive = path.join(root, `${path.basename(source)}.tgz`);
await tar.c({ gzip: true, file: archive, cwd: source }, [
"manifest.json",
deepRelativePath,
"auth",
]);
const containers = containerMock();
await expect(
restoreFleetCell({ ...restoreParams(containers, archive), maxEntries: 5 }),
).rejects.toThrow(/entry limit/iu);
expect(containers.remove).not.toHaveBeenCalled();
await expect(fs.readdir(path.join(root, "fleet", "restore-tmp"))).resolves.toEqual([]);
});
it("rejects an archive without the auth tree before any destructive step", async () => {
const source = path.join(root, `auth-less-${crypto.randomUUID()}`);
await fs.mkdir(path.join(source, "data"), { recursive: true });
src/fleet/backup.runtime.ts
+77 -56
View File
@@ -5,6 +5,13 @@ import os from "node:os";
import path from "node:path";
import { pipeline } from "node:stream/promises";
import * as tar from "tar";
import {
ARCHIVE_LIMIT_ERROR_CODE,
ArchiveFormatError,
ArchiveLimitError,
ArchiveSecurityError,
extractArchive,
} from "../infra/archive.js";
import { formatErrorMessage as errorMessage } from "../infra/errors.js";
import { root as fsSafeRoot } from "../infra/fs-safe.js";
import {
@@ -40,6 +47,7 @@ const MANIFEST_MAX_BYTES = 4 * 1024 * 1024;
const BACKUP_LEASE_PROBE_INTERVAL_MS = 30_000;
const RESTORE_VERIFY_TIMEOUT_MS = 60_000;
const RESTORE_VERIFY_POLL_MS = 1_000;
const RESTORE_EXTRACT_TIMEOUT_MS = 30 * 60_000;
type BackupLinkCacheKey = `${number}:${number}`;
@@ -414,13 +422,6 @@ function isAllowedRestorePath(rawPath: string): boolean {
);
}
function restoreEntryKind(entry: Stats | tar.ReadEntry): "file" | "directory" | "other" {
if ("isFile" in entry) {
return entry.isFile() ? "file" : entry.isDirectory() ? "directory" : "other";
}
return entry.type === "File" ? "file" : entry.type === "Directory" ? "directory" : "other";
}
// Extraction runs as the invoking user, so a root-invoked restore must repair
// ownership for whichever identity the cell container actually runs as: the
// explicit non-root user mapping when one exists, else the image default
@@ -523,58 +524,78 @@ export async function restoreFleetCell(params: {
let replacementAttemptId = "";
try {
let invalidArchive = false;
let totalBytes = 0;
let totalEntries = 0;
let exceeded = false;
let tooManyEntries = false;
let totalPathComponents = 0;
let tooManyPathComponents = false;
const maxBytes = params.maxBytes ?? DEFAULT_FLEET_BACKUP_MAX_BYTES;
const maxEntries = params.maxEntries ?? FLEET_BACKUP_MAX_ENTRIES;
await tar.x({
file: archivePath,
cwd: tempDir,
preservePaths: false,
preserveOwner: false,
strict: true,
filter: (entryPath, entry) => {
if (exceeded || tooManyEntries) {
return false;
}
// Entry cap complements the byte cap: metadata-only archive bombs stay
// under --max-bytes but can exhaust host inodes during extraction.
// Count path segments, not entries, so deep paths whose intermediate
// directories are created implicitly cannot bypass the budget.
totalEntries += entryPath.split("/").filter(Boolean).length;
if (totalEntries > maxEntries) {
tooManyEntries = true;
return false;
}
const kind = restoreEntryKind(entry);
if (kind === "other" || !isAllowedRestorePath(entryPath)) {
invalidArchive = true;
return false;
}
if (kind === "file") {
totalBytes += entry.size;
if (totalBytes > maxBytes) {
exceeded = true;
return false;
try {
await extractArchive({
archivePath,
destDir: tempDir,
kind: "tar",
tarGzip: true,
timeoutMs: RESTORE_EXTRACT_TIMEOUT_MS,
entryModes: "clamp",
entryFilter: (entry) => {
// Preserve Fleet's aggregate component budget in addition to the
// per-entry preflight enforced by maxEntryPathComponents.
totalPathComponents += entry.path.split("/").filter(Boolean).length;
if (totalPathComponents > maxEntries) {
tooManyPathComponents = true;
return "skip";
}
}
return true;
},
});
if (exceeded) {
throw new Error(
`Fleet restore exceeds the ${maxBytes}-byte limit; raise --max-bytes or use a smaller archive.`,
);
}
if (tooManyEntries) {
throw new Error(
`Fleet restore exceeds the ${maxEntries}-entry limit; the archive is not a usable fleet cell backup.`,
);
}
if (invalidArchive) {
throw new Error("Archive is not a fleet cell backup or was tampered with.");
if (
(entry.kind !== "file" && entry.kind !== "directory") ||
!isAllowedRestorePath(entry.path)
) {
invalidArchive = true;
return "skip";
}
return "extract";
},
onFiltered: "reject-archive",
limits: {
// The caller already pinned this exact archive path and size; setting
// the observed size avoids fs-safe's smaller generic upload default.
maxArchiveBytes: Math.max(1, archiveStat.size),
maxEntries,
maxExtractedBytes: maxBytes,
maxEntryBytes: maxBytes,
maxEntryPathComponents: maxEntries,
},
});
} catch (error) {
if (
tooManyPathComponents ||
(error instanceof ArchiveLimitError &&
(error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT ||
error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT))
) {
throw new Error(
`Fleet restore exceeds the ${maxEntries}-entry limit; the archive is not a usable fleet cell backup.`,
{ cause: error },
);
}
if (
error instanceof ArchiveLimitError &&
(error.code === ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT ||
error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT)
) {
throw new Error(
`Fleet restore exceeds the ${maxBytes}-byte limit; raise --max-bytes or use a smaller archive.`,
{ cause: error },
);
}
if (
invalidArchive ||
error instanceof ArchiveSecurityError ||
error instanceof ArchiveFormatError
) {
throw new Error("Archive is not a fleet cell backup or was tampered with.", {
cause: error,
});
}
throw error;
}
const safeRoot = await fsSafeRoot(tempDir, {
symlinks: "reject",
src/infra/archive-helpers.test.ts
+12 -10
View File
@@ -106,26 +106,28 @@ describe("archive helpers", () => {
});
it("preflights tar entries for blocked link types, path escapes, and size budgets", () => {
const checker = createTarEntryPreflightChecker({
rootDir: "/tmp/dest",
limits: {
maxEntries: 1,
maxEntryBytes: 8,
maxExtractedBytes: 12,
},
});
const createChecker = () =>
createTarEntryPreflightChecker({
rootDir: "/tmp/dest",
limits: {
maxEntries: 1,
maxEntryBytes: 8,
maxExtractedBytes: 12,
},
});
expectTarPreflightError(
checker,
createChecker(),
{ path: "package/link", type: "SymbolicLink", size: 0 },
"tar entry is a link: package/link",
);
expectTarPreflightError(
checker,
createChecker(),
{ path: "../escape.txt", type: "File", size: 1 },
/escapes destination|absolute/i,
);
const checker = createChecker();
checker({ path: "package/ok.txt", type: "File", size: 8 });
expectTarPreflightError(
checker,
src/infra/archive.test.ts
+3 -1
View File
@@ -276,7 +276,9 @@ describe("archive utils", () => {
lstatSpy.mockRestore();
}
await expect(fs.readFile(outsideAlias, "utf8")).resolves.toBe("");
// The raced alias points at attacker-supplied archive bytes. Cleanup unlinks the owned
// destination; truncating the inode would instead mutate a path outside that boundary.
await expect(fs.readFile(outsideAlias, "utf8")).resolves.toBe("owned");
await expectPathMissing(extractedPath);
});
},
src/infra/archive.ts
+6
View File
@@ -4,7 +4,9 @@ import "./fs-safe-defaults.js";
// Archive extraction facade for size limits, staged writes, and traversal checks.
export {
ARCHIVE_LIMIT_ERROR_CODE,
ArchiveFormatError,
ArchiveLimitError,
ArchiveSecurityError,
DEFAULT_MAX_ARCHIVE_BYTES_ZIP,
DEFAULT_MAX_ENTRIES,
DEFAULT_MAX_EXTRACTED_BYTES,
@@ -14,8 +16,12 @@ export {
loadZipArchiveWithPreflight,
mergeExtractedTreeIntoDestination,
prepareArchiveDestinationDir,
readArchiveEntry,
resolveArchiveKind,
resolvePackedRootDir,
withStagedArchiveDestination,
type ArchiveLogger,
type ArchiveEntryKind,
type ArchiveExtractLimits,
type ExtractArchiveOptions,
} from "@openclaw/fs-safe/archive";
src/infra/backup-archive-publication.test.ts
+45 -63
View File
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import path from "node:path";
import { PassThrough } from "node:stream";
import { __setFsSafeTestHooksForTest } from "@openclaw/fs-safe/test-hooks";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import {
@@ -12,29 +13,12 @@ import {
type BackupArchivePublication,
} from "./backup-archive-publication.js";
import { writeArchiveStreamToFile, type PreparedBackupArchive } from "./backup-create-stream.js";
const { durabilityTestState } = vi.hoisted(() => ({
durabilityTestState: {
syncOutcome: undefined as
| { status: "synced" }
| { status: "unsupported"; code?: string }
| undefined,
},
}));
vi.mock("@openclaw/fs-safe/durability", async (importOriginal) => {
const actual = await importOriginal<typeof import("@openclaw/fs-safe/durability")>();
return {
...actual,
syncDirectory: async (...args: Parameters<typeof actual.syncDirectory>) =>
durabilityTestState.syncOutcome ?? (await actual.syncDirectory(...args)),
};
});
import { getPublishFileExclusiveFailureDetails } from "./directory-durability.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
afterEach(() => {
durabilityTestState.syncOutcome = undefined;
__setFsSafeTestHooksForTest(undefined);
});
async function createPublication(
@@ -132,6 +116,21 @@ describe("backup archive publication", () => {
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("racer");
});
it.runIf(process.platform !== "win32")(
"preserves a concurrently published hard link to the prepared archive",
async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-hardlink-race-");
const prepared = await prepareArchive(plan);
await fs.link(prepared.archivePath, outputPath);
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/Refusing to overwrite existing backup archive/iu,
);
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("complete archive");
await expect(fs.lstat(prepared.archivePath)).rejects.toMatchObject({ code: "ENOENT" });
},
);
it("rejects a replaced staging pathname without publishing replacement bytes", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-staging-race-");
const prepared = await prepareArchive(plan);
@@ -140,7 +139,7 @@ describe("backup archive publication", () => {
await fs.writeFile(prepared.archivePath, "replacement", "utf8");
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/staging file changed/iu,
/Backup archive changed during publication/iu,
);
await expect(fs.lstat(outputPath)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.readFile(prepared.archivePath, "utf8")).resolves.toBe("replacement");
@@ -162,12 +161,10 @@ describe("backup archive publication", () => {
await fs.unlink(requestedDir);
await fs.symlink(secondDir, requestedDir);
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/output directory changed/iu,
await expect(publishPreparedBackupArchive({ plan, prepared })).resolves.toBeUndefined();
await expect(fs.readFile(path.join(firstDir, "backup.tar.gz"), "utf8")).resolves.toBe(
"complete archive",
);
await expect(fs.lstat(path.join(firstDir, "backup.tar.gz"))).rejects.toMatchObject({
code: "ENOENT",
});
await expect(fs.lstat(path.join(secondDir, "backup.tar.gz"))).rejects.toMatchObject({
code: "ENOENT",
});
@@ -190,12 +187,12 @@ describe("backup archive publication", () => {
plan,
prepared,
}),
).rejects.toThrow(/output directory changed/iu);
).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.lstat(outputPath)).rejects.toMatchObject({ code: "ENOENT" });
},
);
it.runIf(process.platform !== "win32").each(["EIO", "EINVAL"])(
it.runIf(process.platform !== "win32").each(["EIO", "EINVAL", "ENOTSUP"])(
"preserves the complete final archive when commit directory sync fails with %s",
async (code) => {
const { outputPath, plan } = await createPublication("openclaw-backup-sync-failure-");
@@ -213,49 +210,36 @@ describe("backup archive publication", () => {
return await originalOpen(target, flags, mode);
});
try {
await expect(publishPreparedBackupArchive({ plan, prepared, log })).rejects.toThrow(
/sync failed/iu,
const error = await publishPreparedBackupArchive({ plan, prepared, log }).catch(
(caught: unknown) => caught,
);
expect(error).toBeInstanceOf(Error);
expect(String(error)).toMatch(/sync failed/iu);
expect(getPublishFileExclusiveFailureDetails(error)).toMatchObject({
phase: "directory-sync",
cleanup: "preserved",
directorySync: { status: "failed", code },
});
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("complete archive");
expect(log).toHaveBeenCalledWith(expect.stringContaining("concurrent replacement"));
expect(log).toHaveBeenCalledWith(expect.stringContaining("preserved the final archive"));
} finally {
openSpy.mockRestore();
}
},
);
it.runIf(process.platform !== "win32")(
"fails closed when the commit directory cannot be synchronized",
async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-sync-unsupported-");
const prepared = await prepareArchive(plan);
const log = vi.fn();
durabilityTestState.syncOutcome = { status: "unsupported", code: "ENOTSUP" };
await expect(publishPreparedBackupArchive({ plan, prepared, log })).rejects.toThrow(
/does not support crash-durable directory synchronization \(ENOTSUP\)/iu,
);
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("complete archive");
expect(log).toHaveBeenCalledWith(expect.stringContaining("concurrent replacement"));
},
);
it("preserves a destination that replaces the linked archive before validation", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-linked-race-");
const prepared = await prepareArchive(plan);
const displacedPath = `${outputPath}.displaced`;
const originalLstat = fs.lstat.bind(fs);
let targetLstatCount = 0;
const lstatSpy = vi.spyOn(fs, "lstat").mockImplementation(async (target, options) => {
if (path.resolve(String(target)) === path.resolve(plan.canonicalOutputPath)) {
targetLstatCount += 1;
}
if (targetLstatCount === 2) {
targetLstatCount += 1;
await fs.rename(plan.canonicalOutputPath, displacedPath);
await fs.writeFile(plan.canonicalOutputPath, "racer", "utf8");
}
return await originalLstat(target, options);
__setFsSafeTestHooksForTest({
afterPublishTargetCreated: async (_method, targetPath) => {
if (path.resolve(targetPath) === path.resolve(plan.canonicalOutputPath)) {
__setFsSafeTestHooksForTest(undefined);
await fs.rename(plan.canonicalOutputPath, displacedPath);
await fs.writeFile(plan.canonicalOutputPath, "racer", "utf8");
}
},
});
try {
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
@@ -264,7 +248,7 @@ describe("backup archive publication", () => {
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("racer");
await expect(fs.readFile(displacedPath, "utf8")).resolves.toBe("complete archive");
} finally {
lstatSpy.mockRestore();
__setFsSafeTestHooksForTest(undefined);
}
});
@@ -332,7 +316,7 @@ describe("backup archive publication", () => {
await expect(fs.lstat(plan.stagingDir)).rejects.toMatchObject({ code: "ENOENT" });
});
it("preserves a final-path replacement detected after the commit point", async () => {
it("preserves a final-path replacement after the commit point", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-final-race-");
const prepared = await prepareArchive(plan);
const displacedPath = `${outputPath}.displaced`;
@@ -347,9 +331,7 @@ describe("backup archive publication", () => {
return originalUnlinkSync(target);
});
try {
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/Published backup archive changed/iu,
);
await expect(publishPreparedBackupArchive({ plan, prepared })).resolves.toBeUndefined();
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("racer");
await expect(fs.readFile(displacedPath, "utf8")).resolves.toBe("complete archive");
} finally {
src/infra/backup-archive-publication.ts
+31 -140
View File
@@ -1,15 +1,20 @@
import { randomUUID } from "node:crypto";
import { constants as fsConstants, type Stats } from "node:fs";
import type { Stats } from "node:fs";
import fs from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import path from "node:path";
import { syncDirectory, type DirectoryReceipt } from "@openclaw/fs-safe/durability";
import {
removePreparedBackupArchive,
type BackupArchiveCleanupReceipt,
type PreparedBackupArchive,
} from "./backup-create-stream.js";
import { requireDirectorySync, syncDirectoryIfSupported } from "./directory-durability.js";
import {
getPublishFileExclusiveFailureDetails,
isHardlinkFallbackError,
publishFileExclusive,
requireDirectorySync,
syncDirectoryIfSupported,
type DirectoryReceipt,
} from "./directory-durability.js";
import { sameFileIdentity } from "./fs-safe-advanced.js";
type BackupArchiveLogger = (message: string) => void;
@@ -46,20 +51,6 @@ async function assertTargetAbsent(targetPath: string): Promise<void> {
throw new Error(`Refusing to overwrite existing backup archive: ${targetPath}`);
}
async function assertPublicationParentUnchanged(plan: BackupArchivePublication): Promise<void> {
const currentCanonicalParent = await fs.realpath(plan.requestedParentPath);
const currentParentIdentity = await fs.lstat(plan.canonicalParentPath);
if (
!pathsEqual(currentCanonicalParent, plan.canonicalParentPath) ||
!currentParentIdentity.isDirectory() ||
!sameFileIdentity(plan.parentReceipt.identity, currentParentIdentity)
) {
throw new Error(
`Backup output directory changed during archive creation: ${plan.requestedParentPath}`,
);
}
}
async function removeDirectoryIfOwned(
directoryPath: string,
expectedIdentity: Stats,
@@ -86,80 +77,6 @@ async function removeStagingDirectoryIfOwned(plan: BackupArchivePublication): Pr
return await removeDirectoryIfOwned(plan.stagingDir, plan.stagingIdentity);
}
async function syncPublishedArchiveCommit(
plan: BackupArchivePublication,
preparedHandle: FileHandle,
): Promise<void> {
if (process.platform === "win32") {
// Windows FlushFileBuffers requires a writable file handle and flushes
// buffered file metadata. The prepared handle pins the published inode.
await preparedHandle.sync();
return;
}
// Publication success requires a real directory fsync. Unsupported
// filesystems fail closed instead of weakening crash durability.
const outcome = await syncDirectory(plan.parentReceipt, {
label: "backup publication directory",
});
requireDirectorySync(outcome, "Backup publication directory");
}
function isUnsupportedHardLinkError(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException).code;
return (
code === "EPERM" ||
code === "EXDEV" ||
code === "ENOTSUP" ||
code === "EOPNOTSUPP" ||
code === "ENOSYS"
);
}
async function openPreparedArchive(
plan: BackupArchivePublication,
prepared: PreparedBackupArchive,
): Promise<FileHandle> {
const accessMode = process.platform === "win32" ? fsConstants.O_RDWR : fsConstants.O_RDONLY;
const flags = accessMode | (fsConstants.O_NOFOLLOW ?? 0) | (fsConstants.O_NONBLOCK ?? 0);
const handle = await fs.open(prepared.archivePath, flags);
try {
const openedIdentity = await handle.stat();
const currentIdentity = await fs.lstat(prepared.archivePath);
if (
!pathsEqual(path.dirname(prepared.archivePath), plan.stagingDir) ||
!openedIdentity.isFile() ||
!currentIdentity.isFile() ||
!sameFileIdentity(prepared.identity, openedIdentity) ||
!sameFileIdentity(prepared.identity, currentIdentity)
) {
throw new Error(
`Backup archive staging file changed before publication: ${prepared.archivePath}`,
);
}
return handle;
} catch (error) {
await handle.close().catch(() => undefined);
throw error;
}
}
async function assertPublishedArchiveUnchanged(
plan: BackupArchivePublication,
handle: FileHandle,
expectedIdentity: Stats,
): Promise<void> {
const openedIdentity = await handle.stat();
const currentIdentity = await fs.lstat(plan.canonicalOutputPath);
if (
!openedIdentity.isFile() ||
!currentIdentity.isFile() ||
!sameFileIdentity(expectedIdentity, openedIdentity) ||
!sameFileIdentity(expectedIdentity, currentIdentity)
) {
throw new Error(`Published backup archive changed: ${plan.requestedOutputPath}`);
}
}
export async function createBackupArchivePublication(
outputPath: string,
): Promise<BackupArchivePublication> {
@@ -274,55 +191,44 @@ export async function publishPreparedBackupArchive(params: {
log?: BackupArchiveLogger;
}): Promise<void> {
const { plan, prepared } = params;
let preparedHandle: FileHandle | undefined;
let publishedIdentity: Stats | undefined;
let hardLinkCreated = false;
let publicationPreserved = false;
let committed = false;
try {
await assertPublicationParentUnchanged(plan);
preparedHandle = await openPreparedArchive(plan, prepared);
await assertTargetAbsent(plan.canonicalOutputPath);
// Node has no portable link-by-handle primitive. Under OpenClaw's one-user
// host trust model, post-link identity checks fence cooperative replacement
// races and ensure a changed staging pathname can never produce success.
try {
await fs.link(prepared.archivePath, plan.canonicalOutputPath);
hardLinkCreated = true;
const publication = await publishFileExclusive({
sourcePath: prepared.archivePath,
targetPath: plan.canonicalOutputPath,
expectedSourceIdentity: prepared.identity,
parentReceipt: plan.parentReceipt,
strategy: "link-required",
onSyncFailure: "preserve",
});
publicationPreserved = true;
requireDirectorySync(publication.directorySync, "Backup publication directory");
committed = true;
} catch (error) {
const details = getPublishFileExclusiveFailureDetails(error);
publicationPreserved ||= details?.cleanup === "preserved";
if ((error as NodeJS.ErrnoException).code === "EEXIST") {
throw new Error(
`Refusing to overwrite existing backup archive: ${plan.requestedOutputPath}`,
{ cause: error },
);
}
if (isUnsupportedHardLinkError(error)) {
if (isHardlinkFallbackError(error)) {
throw new Error(
`Atomic backup publication requires hard-link support in ${plan.requestedParentPath}.`,
{ cause: error },
);
}
if ((error as { code?: unknown }).code === "path-mismatch") {
throw new Error(`Backup archive changed during publication: ${plan.requestedOutputPath}`, {
cause: error,
});
}
throw error;
}
await assertPublicationParentUnchanged(plan);
const currentTargetIdentity = await fs.lstat(plan.canonicalOutputPath);
const currentStagingIdentity = await fs.lstat(prepared.archivePath);
if (
!currentTargetIdentity.isFile() ||
!currentStagingIdentity.isFile() ||
!sameFileIdentity(prepared.identity, currentTargetIdentity) ||
!sameFileIdentity(prepared.identity, currentStagingIdentity)
) {
throw new Error(`Backup archive changed during publication: ${plan.requestedOutputPath}`);
}
publishedIdentity = currentTargetIdentity;
await assertPublishedArchiveUnchanged(plan, preparedHandle, publishedIdentity);
// The first parent sync commits the final pathname. After this point,
// cleanup failures must not remove or invalidate the durable archive.
await syncPublishedArchiveCommit(plan, preparedHandle);
committed = true;
if (!removePreparedBackupArchive(prepared)) {
retainArchiveForCleanup(plan, prepared);
params.log?.(`Backup archiver preserved changed staging file ${prepared.archivePath}.`);
@@ -339,24 +245,11 @@ export async function publishPreparedBackupArchive(params: {
}.`,
);
});
await assertPublicationParentUnchanged(plan);
await assertPublishedArchiveUnchanged(plan, preparedHandle, publishedIdentity);
} catch (error) {
if (!committed) {
if (!publishedIdentity && hardLinkCreated) {
const currentTargetIdentity = await fs
.lstat(plan.canonicalOutputPath)
.catch(() => undefined);
if (
currentTargetIdentity?.isFile() &&
sameFileIdentity(currentTargetIdentity, prepared.identity)
) {
publishedIdentity = currentTargetIdentity;
}
}
if (publishedIdentity) {
if (publicationPreserved) {
params.log?.(
`Backup archiver preserved the final archive after publication failed so a concurrent replacement could not be deleted: ${plan.requestedOutputPath}.`,
`Backup archiver preserved the final archive after publication failed: ${plan.requestedOutputPath}.`,
);
}
if (!removePreparedBackupArchive(prepared)) {
@@ -365,7 +258,5 @@ export async function publishPreparedBackupArchive(params: {
await removeStagingDirectoryIfOwned(plan);
}
throw error;
} finally {
await preparedHandle?.close().catch(() => undefined);
}
}
src/infra/directory-durability.test.ts
+50 -1
View File
@@ -1,11 +1,38 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { requireDirectorySync, syncDirectoryIfSupported } from "./directory-durability.js";
import {
getPublishFileExclusiveFailureDetails,
publishFileNoClobber,
requireDirectorySync,
syncDirectoryIfSupported,
} from "./directory-durability.js";
const durabilityTestState = vi.hoisted(() => ({
publishSyncOutcome: undefined as
| { status: "synced" }
| { status: "unsupported"; code?: string }
| undefined,
}));
vi.mock("@openclaw/fs-safe/durability", async (importOriginal) => {
const actual = await importOriginal<typeof import("@openclaw/fs-safe/durability")>();
return {
...actual,
publishFileExclusive: async (...args: Parameters<typeof actual.publishFileExclusive>) => {
const result = await actual.publishFileExclusive(...args);
return durabilityTestState.publishSyncOutcome
? { ...result, directorySync: durabilityTestState.publishSyncOutcome }
: result;
},
};
});
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
afterEach(() => {
durabilityTestState.publishSyncOutcome = undefined;
vi.restoreAllMocks();
});
@@ -31,6 +58,28 @@ describe("directory durability compatibility", () => {
).not.toThrow();
});
it("preserves its target with a receipt when fail-closed durability rejects", async () => {
vi.spyOn(process, "platform", "get").mockReturnValue("linux");
const directoryPath = tempDirs.make("openclaw-publish-cleanup-");
const sourcePath = path.join(directoryPath, "source.txt");
const targetPath = path.join(directoryPath, "target.txt");
await fs.writeFile(sourcePath, "complete publication");
durabilityTestState.publishSyncOutcome = { status: "unsupported", code: "ENOTSUP" };
const error = await publishFileNoClobber(sourcePath, targetPath, {
strategy: "link-or-copy",
durability: "fail-closed",
}).catch((caught: unknown) => caught);
expect(getPublishFileExclusiveFailureDetails(error)).toMatchObject({
phase: "directory-sync",
targetCreated: true,
cleanup: "preserved",
});
await expect(fs.readFile(sourcePath, "utf8")).resolves.toBe("complete publication");
await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("complete publication");
});
it.runIf(process.platform !== "win32")("reports a completed directory sync", async () => {
const directoryPath = tempDirs.make("openclaw-directory-sync-");
src/infra/directory-durability.ts
+109 -1
View File
@@ -1,11 +1,25 @@
import { syncDirectory, type DirectorySyncOutcome } from "@openclaw/fs-safe/durability";
import fs from "node:fs/promises";
import {
publishFileExclusive,
syncDirectory,
type DirectorySyncOutcome,
type PublishFileExclusiveFailureDetails,
type PublishFileExclusiveFailurePhase,
type PublishFileExclusiveResult,
} from "@openclaw/fs-safe/durability";
import { sameFileIdentity } from "./fs-safe-advanced.js";
import { FsSafeError } from "./fs-safe.js";
export {
ensureDurableDirectory,
isHardlinkFallbackError,
pinDirectory,
publishFileExclusive,
sha256File,
syncDirectory,
syncDirectoryBestEffortSync,
type DirectorySyncOutcome,
type DirectoryReceipt,
type DurableDirectoryReceipt,
type PinnedDirectory,
} from "@openclaw/fs-safe/durability";
@@ -31,6 +45,100 @@ export function requireDirectorySync(outcome: DirectoryDurabilityOutcome, label:
throw new Error(`${label} does not support crash-durable directory synchronization${code}.`);
}
export function getPublishFileExclusiveFailureDetails(
error: unknown,
): PublishFileExclusiveFailureDetails | undefined {
if (!(error instanceof FsSafeError)) {
return undefined;
}
const details = error.details;
if (
!details ||
typeof details.targetCreated !== "boolean" ||
(details.cleanup !== "removed" &&
details.cleanup !== "preserved" &&
details.cleanup !== "unknown")
) {
return undefined;
}
return details as PublishFileExclusiveFailureDetails;
}
function postPublicationFailure(params: {
error: unknown;
phase: PublishFileExclusiveFailurePhase;
published: PublishFileExclusiveResult;
}): FsSafeError {
const cause = params.error instanceof Error ? params.error : new Error(String(params.error));
return new FsSafeError(
params.error instanceof FsSafeError ? params.error.code : "helper-failed",
`File publication failed after target creation: ${cause.message}`,
{
cause,
details: {
phase: params.phase,
targetCreated: true,
targetIdentity: {
dev: params.published.identity.dev,
ino: params.published.identity.ino,
},
// After publication succeeds, pathname cleanup cannot atomically prove
// it still owns the target. Callers use this receipt with their pinned guards.
cleanup: "preserved",
} satisfies PublishFileExclusiveFailureDetails,
},
);
}
/** Publish one file without replacement under OpenClaw's durability policy. */
export async function publishFileNoClobber(
sourcePath: string,
targetPath: string,
options: {
strategy: "link-required" | "link-or-copy";
moveSource?: boolean;
durability: "fail-closed" | "degrade";
},
) {
const sourceIdentity = await fs.lstat(sourcePath);
const published = await publishFileExclusive({
sourcePath,
targetPath,
expectedSourceIdentity: sourceIdentity,
strategy: options.strategy,
});
const degraded = published.directorySync.status === "unsupported";
if (options.durability === "fail-closed") {
try {
requireDirectorySync(published.directorySync, "File publication directory");
} catch (error) {
throw postPublicationFailure({
error,
phase: "directory-sync",
published,
});
}
}
if (options.moveSource) {
try {
const currentSource = await fs.lstat(sourcePath);
if (!currentSource.isFile() || !sameFileIdentity(currentSource, sourceIdentity)) {
throw new Error(`File publication source changed before removal: ${sourcePath}`);
}
await fs.unlink(sourcePath);
} catch (error) {
throw postPublicationFailure({
error,
phase: published.method === "hardlink" ? "hardlink-verify" : "copy-verify",
published,
});
}
}
return { ...published, durability: degraded ? "degraded" : "durable" };
}
/** Compatibility adapter for former best-effort call sites. */
export async function syncDirectoryIfSupported(
directoryPath: string,
src/infra/file-lock-manager.ts
+12 -1
View File
@@ -1,6 +1,17 @@
// Exposes lifecycle-owned file lock managers with fs-safe defaults.
import "./fs-safe-defaults.js";
import type { Root as FsSafeRoot } from "@openclaw/fs-safe/root";
// Process-local file lock manager used by code that needs explicit lifecycle
// control instead of a one-shot withFileLock call.
export { createFileLockManager } from "@openclaw/fs-safe/file-lock";
export {
acquireFileLockSync,
createFileLockManager,
type FileLockSyncAcquireOptions,
type FileLockSyncHandle,
} from "@openclaw/fs-safe/file-lock";
/** Recover the full runtime Root type for core-only lockRoot use. */
export function asFsSafeFileLockRoot(root: Omit<FsSafeRoot, "walk">): FsSafeRoot {
return root as FsSafeRoot;
}
src/infra/fs-safe-advanced.ts
+24 -1
View File
@@ -1,5 +1,6 @@
// Provides stricter filesystem helpers for canonical path and symlink-sensitive operations.
import "./fs-safe-defaults.js";
import path from "node:path";
// Advanced fs-safe helpers for symlink, hardlink, and sibling-temp protections.
export {
@@ -7,5 +8,27 @@ export {
assertNoSymlinkParentsSync,
type FileIdentityStat,
sameFileIdentity,
sanitizeUntrustedFileName,
} from "@openclaw/fs-safe/advanced";
export { readSecretFile } from "@openclaw/fs-safe/secret";
/** Preserve the shipped Plugin SDK filename contract while fs-safe owns path basename handling. */
export function sanitizeUntrustedFileName(fileName: string, fallbackName: string): string {
const trimmed = typeof fileName === "string" ? fileName.trim() : "";
if (!trimmed) {
return fallbackName;
}
let base = path.win32.basename(path.posix.basename(trimmed));
let cleaned = "";
for (let index = 0; index < base.length; index += 1) {
const code = base.charCodeAt(index);
if (code < 0x20 || code === 0x7f) {
continue;
}
cleaned += base[index];
}
base = cleaned.trim();
if (!base || base === "." || base === "..") {
return fallbackName;
}
return base.length > 200 ? base.slice(0, 200) : base;
}
src/infra/fs-safe-defaults.test.ts
+43 -11
View File
@@ -1,12 +1,12 @@
// Covers OpenClaw's default fs-safe Python helper configuration.
// Covers OpenClaw's default fs-safe native helper configuration.
import { afterEach, describe, expect, it, vi } from "vitest";
const { configureFsSafePython } = vi.hoisted(() => ({
configureFsSafePython: vi.fn(),
const { configureFsSafeNative } = vi.hoisted(() => ({
configureFsSafeNative: vi.fn(),
}));
vi.mock("@openclaw/fs-safe/config", () => ({
configureFsSafePython,
configureFsSafeNative,
}));
async function importDefaults() {
@@ -16,30 +16,62 @@ async function importDefaults() {
describe("fs-safe defaults", () => {
afterEach(() => {
configureFsSafePython.mockReset();
configureFsSafeNative.mockReset();
delete process.env.FS_SAFE_NATIVE_MODE;
delete process.env.OPENCLAW_FS_SAFE_NATIVE_MODE;
delete process.env.openclaw_fs_safe_native_mode;
delete process.env.FS_SAFE_PYTHON_MODE;
delete process.env.OPENCLAW_FS_SAFE_PYTHON_MODE;
delete process.env.FS_SAFE_PYTHON;
delete process.env.OPENCLAW_FS_SAFE_PYTHON;
delete process.env.OPENCLAW_PINNED_PYTHON;
delete process.env.OPENCLAW_PINNED_WRITE_PYTHON;
});
it("disables the Python helper by default in OpenClaw", async () => {
it("disables the native helper by default in OpenClaw", async () => {
await importDefaults();
expect(configureFsSafePython).toHaveBeenCalledWith({ mode: "off" });
expect(configureFsSafeNative).toHaveBeenCalledWith({ mode: "off" });
});
it("lets fs-safe env mode overrides opt back into the helper", async () => {
process.env.FS_SAFE_PYTHON_MODE = "require";
process.env.FS_SAFE_NATIVE_MODE = "require";
await importDefaults();
expect(configureFsSafePython).not.toHaveBeenCalled();
expect(configureFsSafeNative).not.toHaveBeenCalled();
});
it("honors the OpenClaw-specific env mode override", async () => {
process.env.OPENCLAW_FS_SAFE_PYTHON_MODE = "auto";
process.env.OPENCLAW_FS_SAFE_NATIVE_MODE = "auto";
await importDefaults();
expect(configureFsSafePython).not.toHaveBeenCalled();
expect(configureFsSafeNative).not.toHaveBeenCalled();
});
it("honors case-insensitive mode overrides on Windows", async () => {
vi.spyOn(process, "platform", "get").mockReturnValue("win32");
process.env.openclaw_fs_safe_native_mode = "require";
await importDefaults();
expect(configureFsSafeNative).not.toHaveBeenCalled();
});
it("lets fs-safe migrate legacy require mode without overriding it", async () => {
process.env.OPENCLAW_FS_SAFE_PYTHON_MODE = "require";
await importDefaults();
expect(configureFsSafeNative).not.toHaveBeenCalled();
});
it("does not treat a retired interpreter path as a native mode override", async () => {
process.env.OPENCLAW_FS_SAFE_PYTHON = "/usr/bin/python3";
await importDefaults();
expect(configureFsSafeNative).toHaveBeenCalledWith({ mode: "off" });
});
});
src/infra/fs-safe-defaults.ts
+9 -6
View File
@@ -1,11 +1,14 @@
// Applies OpenClaw's default fs-safe runtime configuration.
import { configureFsSafePython } from "@openclaw/fs-safe/config";
import { configureFsSafeNative } from "@openclaw/fs-safe/config";
// OpenClaw does not rely on Python helpers for normal filesystem safety. Tests
// OpenClaw does not rely on native helpers for normal filesystem safety. Tests
// and operators can still opt in with fs-safe's documented env override.
const hasPythonModeOverride =
process.env.FS_SAFE_PYTHON_MODE != null || process.env.OPENCLAW_FS_SAFE_PYTHON_MODE != null;
const hasModeOverride = Object.keys(process.env).some((key) =>
/^(?:OPENCLAW_)?FS_SAFE_(?:NATIVE|PYTHON)_MODE$/u.test(
process.platform === "win32" ? key.toUpperCase() : key,
),
);
if (!hasPythonModeOverride) {
configureFsSafePython({ mode: "off" });
if (!hasModeOverride) {
configureFsSafeNative({ mode: "off" });
}
src/infra/fs-safe-remove.ts
+21 -2
View File
@@ -17,9 +17,24 @@ function compareDirectoryEntryNames(left: DirectoryEntry, right: DirectoryEntry)
return left.name < right.name ? -1 : 1;
}
function isNotFoundError(error: unknown): error is NodeJS.ErrnoException {
function isNotFoundError(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
return code === "not-found" || code === "ENOENT";
return (
code === "not-found" ||
code === "ENOENT" ||
findPathAliasFilesystemCause(error)?.code === "ENOENT"
);
}
function findPathAliasFilesystemCause(error: unknown): NodeJS.ErrnoException | undefined {
if ((error as NodeJS.ErrnoException | undefined)?.code !== "path-alias") {
return undefined;
}
const cause = (error as Error & { cause?: unknown }).cause;
const causeCode = (cause as NodeJS.ErrnoException | undefined)?.code;
return typeof causeCode === "string" && /^E[A-Z0-9_]+$/u.test(causeCode)
? (cause as NodeJS.ErrnoException)
: undefined;
}
function relativeParentPath(relativePath: string): string {
@@ -59,6 +74,10 @@ async function removeRootRelativePath(
cause: error instanceof Error ? error : undefined,
});
}
const filesystemCause = findPathAliasFilesystemCause(error);
if (filesystemCause) {
throw filesystemCause;
}
throw error;
}
}
src/infra/fs-safe.test.ts
+20 -1
View File
@@ -151,10 +151,29 @@ describe("fs-safe", () => {
},
});
expect(result.path).toBe(path.join(dir, "artifact.txt"));
expect(result).toEqual({ path: path.join(dir, "artifact.txt") });
await expect(fs.readFile(path.join(dir, "artifact.txt"), "utf8")).resolves.toBe("artifact");
});
it.each([
["fallbackFileName", { fallbackFileName: "safe-output.bin" }],
["tempPrefix", { tempPrefix: "safe-output.bin" }],
] as const)("maps %s onto the upstream portable fallback name", async (_label, naming) => {
const dir = await tempDirs.make("openclaw-fs-safe-output-name-");
const result = await writeExternalFileWithinRoot({
rootDir: dir,
path: "\u0001",
...naming,
write: async (tempPath) => {
await fs.writeFile(tempPath, "artifact");
},
});
expect(result).toEqual({ path: path.join(dir, "safe-output.bin") });
await expect(fs.readFile(result.path, "utf8")).resolves.toBe("artifact");
});
it("enforces maxBytes", async () => {
const dir = await tempDirs.make("openclaw-fs-safe-");
const file = path.join(dir, "big.bin");
src/infra/fs-safe.ts
+29 -19
View File
@@ -2,12 +2,14 @@
import "./fs-safe-defaults.js";
import fs from "node:fs/promises";
import path from "node:path";
import { ensureDirectoryWithinRoot, findExistingAncestor } from "@openclaw/fs-safe/advanced";
import { writeExternalFileWithinRoot as writeExternalFileWithinRootBase } from "@openclaw/fs-safe/output";
import {
ensureDirectoryWithinRoot,
findExistingAncestor,
writeViaSiblingTempPath,
} from "@openclaw/fs-safe/advanced";
import { root as fsSafeRoot, type ReadResult } from "@openclaw/fs-safe/root";
root as fsSafeRoot,
type ReadResult,
type Root as FsSafeRoot,
type RootDefaults,
} from "@openclaw/fs-safe/root";
export { FsSafeError, type FsSafeErrorCode } from "@openclaw/fs-safe/errors";
export {
@@ -39,12 +41,10 @@ export {
openLocalFileSafely,
readLocalFileSafely,
resolveOpenedFileRealPathForHandle,
root,
type OpenResult,
type ReadResult,
type Root,
} from "@openclaw/fs-safe/root";
export { sanitizeUntrustedFileName } from "@openclaw/fs-safe/advanced";
export { sanitizeUntrustedFileName } from "./fs-safe-advanced.js";
export {
readSecureFile,
type SecureFileReadOptions,
@@ -59,6 +59,14 @@ export {
} from "@openclaw/fs-safe/walk";
export { withTimeout } from "@openclaw/fs-safe/advanced";
// The broad Plugin SDK infra barrel re-exports this facade. Keep fs-safe 0.5's
// new Root.walk capability core-only until a dedicated plugin contract is approved.
export type Root = Omit<FsSafeRoot, "walk">;
export async function root(rootDir: string, defaults?: RootDefaults): Promise<Root> {
return await fsSafeRoot(rootDir, defaults);
}
export type ExternalFileWriteOptions = {
rootDir: string;
path: string;
@@ -107,15 +115,17 @@ export async function ensureAbsoluteDirectory(
export async function writeExternalFileWithinRoot(
options: ExternalFileWriteOptions,
): Promise<ExternalFileWriteResult> {
const targetPath = path.resolve(options.rootDir, options.path);
await writeViaSiblingTempPath({
const requestedPath = path.resolve(options.rootDir, options.path);
const result = await writeExternalFileWithinRootBase({
rootDir: options.rootDir,
targetPath,
writeTemp: options.write,
fallbackFileName: options.fallbackFileName,
tempPrefix: options.tempPrefix,
path: options.path,
write: options.write,
staging: "sibling",
fallbackFileName: options.fallbackFileName ?? options.tempPrefix,
});
return { path: targetPath };
// Preserve the caller-facing path spelling while carrying forward any
// portable basename selected by fs-safe (for example, /var vs /private/var).
return { path: path.join(path.dirname(requestedPath), path.basename(result.path)) };
}
/** @deprecated Use root(rootDir).read(relativePath, options). */
@@ -127,8 +137,8 @@ export async function readFileWithinRoot(params: {
allowSymlinkTargetWithinRoot?: boolean;
maxBytes?: number;
}): Promise<ReadResult> {
const root = await fsSafeRoot(params.rootDir);
return await root.read(params.relativePath, {
const fsRoot = await fsSafeRoot(params.rootDir);
return await fsRoot.read(params.relativePath, {
hardlinks: params.rejectHardlinks === false ? "allow" : "reject",
maxBytes: params.maxBytes,
nonBlockingRead: params.nonBlockingRead,
@@ -144,8 +154,8 @@ export async function writeFileWithinRoot(params: {
encoding?: BufferEncoding;
mkdir?: boolean;
}): Promise<void> {
const root = await fsSafeRoot(params.rootDir);
await root.write(params.relativePath, params.data, {
const fsRoot = await fsSafeRoot(params.rootDir);
await fsRoot.write(params.relativePath, params.data, {
encoding: params.encoding,
mkdir: params.mkdir,
});
src/infra/gateway-lock.test.ts
+62 -8
View File
@@ -15,6 +15,7 @@ import {
readActiveGatewayLockIdentity,
readActiveGatewayLockPort,
} from "./gateway-lock.js";
import { openNodeSqliteDatabase } from "./node-sqlite.js";
type GatewayLock = NonNullable<Awaited<ReturnType<typeof acquireGatewayLock>>>;
type GatewayLockOptions = NonNullable<Parameters<typeof acquireGatewayLock>[0]>;
@@ -543,6 +544,55 @@ describe("gateway lock", () => {
await nextLock.release();
});
it("continues honoring the legacy lifetime coordinator", async () => {
const env = await makeEnv();
const { stateLockPath } = resolveLockPath(env);
await fs.mkdir(path.dirname(stateLockPath), { recursive: true });
const coordinator = openNodeSqliteDatabase(`${stateLockPath}.sqlite`);
coordinator.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
try {
await expect(acquireForTest(env, { timeoutMs: 15 })).rejects.toBeInstanceOf(GatewayLockError);
} finally {
coordinator.exec("ROLLBACK");
coordinator.close();
}
await expectGatewayLock(await acquireForTest(env)).release();
});
it("preserves a fresh gateway lock that replaces the stale reclaim candidate", async () => {
vi.useRealTimers();
const env = await makeEnv();
const { configPath, stateLockPath } = resolveLockPath(env);
await fs.mkdir(path.dirname(stateLockPath), { recursive: true });
await fs.writeFile(
stateLockPath,
JSON.stringify(createLockPayload({ configPath, startTime: 111 })),
"utf8",
);
const replacement = {
...createLockPayload({ configPath, startTime: 333 }),
ownerId: "replacement-owner",
};
let startTimeReads = 0;
await expect(
acquireForTest(env, {
platform: "linux",
timeoutMs: 25,
readProcessStartTime: () => {
startTimeReads += 1;
if (startTimeReads === 2) {
fsSync.writeFileSync(stateLockPath, JSON.stringify(replacement), "utf8");
}
return startTimeReads >= 3 ? 333 : 222;
},
}),
).rejects.toBeInstanceOf(GatewayLockError);
expect(JSON.parse(await fs.readFile(stateLockPath, "utf8"))).toMatchObject(replacement);
});
it("keeps lock on linux when proc access fails unless stale", async () => {
vi.useRealTimers();
const env = await makeEnv();
@@ -713,7 +763,7 @@ describe("gateway lock", () => {
}
});
it("ages out an old maintenance owner with unreadable process identity", async () => {
it("keeps an old maintenance owner when its live identity is unreadable", async () => {
vi.useRealTimers();
const env = await makeEnv();
const { lockPath, configPath } = resolveLockPath(env);
@@ -732,13 +782,14 @@ describe("gateway lock", () => {
const spy = createEaccesProcStatSpy();
try {
const lock = await acquireForTest(env, {
platform: "linux",
readProcessCmdline: () => null,
staleMs: 0,
timeoutMs: 80,
});
await expectGatewayLock(lock).release();
await expect(
acquireForTest(env, {
platform: "linux",
readProcessCmdline: () => null,
staleMs: 0,
timeoutMs: 80,
}),
).rejects.toBeInstanceOf(GatewayLockError);
} finally {
spy.mockRestore();
}
@@ -794,6 +845,7 @@ describe("gateway lock", () => {
platform: "darwin",
port: 18789,
readProcessCmdline: () => ["/usr/local/bin/openclaw", "gateway", "run"],
readProcessStartTime: () => 111,
});
await expect(pending).rejects.toBeInstanceOf(GatewayLockError);
} finally {
@@ -822,6 +874,7 @@ describe("gateway lock", () => {
},
lockDir: resolveTestLockDir(),
readProcessCmdline: () => ["/usr/local/bin/openclaw", "gateway", "run"],
readProcessStartTime: () => 111,
}),
).rejects.toBeInstanceOf(GatewayLockError);
@@ -1036,6 +1089,7 @@ describe("gateway lock", () => {
platform: "darwin",
port: 18789,
readProcessCmdline: () => ["/usr/local/bin/openclaw", "gateway", "run", "--port", "18789"],
readProcessStartTime: () => 111,
});
await expect(pending).rejects.toBeInstanceOf(GatewayLockError);
src/infra/gateway-lock.ts
+96 -157
View File
@@ -4,7 +4,6 @@ import { randomUUID } from "node:crypto";
import fsSync from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import {
resolvePositiveTimerTimeoutMs,
resolveTimerTimeoutMs,
@@ -15,9 +14,9 @@ import { resolveConfigPath, resolveGatewayLockDir, resolveStateDir } from "../co
import { getFileLockProcessStartTime, isPidAlive } from "../shared/pid-alive.js";
import { safeParseJsonWithSchema } from "../utils/zod-parse.js";
import { sha256HexPrefix } from "./crypto-digest.js";
import { createFileLockManager } from "./file-lock-manager.js";
import { isGatewayArgv, isOpenClawCommandArgv, parseProcCmdline } from "./gateway-process-argv.js";
import { openNodeSqliteDatabase } from "./node-sqlite.js";
import { isSqliteLockError } from "./sqlite-transaction.js";
import { tryAcquireExclusiveSqliteCoordinator } from "./node-sqlite.js";
import {
readWindowsProcessArgsSync,
readWindowsProcessStartTimeSync,
@@ -26,6 +25,7 @@ import {
const DEFAULT_TIMEOUT_MS = 5000;
const DEFAULT_POLL_INTERVAL_MS = 100;
const DEFAULT_STALE_MS = 30_000;
const GATEWAY_LOCKS = createFileLockManager("openclaw.gateway-lock");
type LockPayload = {
pid: number;
@@ -110,44 +110,6 @@ export class GatewayLockError extends Error {
type LockOwnerStatus = "alive" | "dead" | "unknown";
type GatewayLockCoordinator = {
release: () => void;
};
function tryAcquireGatewayLockCoordinator(lockPath: string): GatewayLockCoordinator | null {
const coordinatorDb: DatabaseSync = openNodeSqliteDatabase(`${lockPath}.sqlite`);
try {
coordinatorDb.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
} catch (error) {
try {
coordinatorDb.close();
} catch {}
if (isSqliteLockError(error)) {
return null;
}
throw error;
}
return {
release: () => {
let releaseError: unknown;
try {
coordinatorDb.exec("ROLLBACK");
} catch (error) {
releaseError = error;
}
try {
coordinatorDb.close();
} catch (error) {
releaseError ??= error;
}
if (releaseError) {
throw new GatewayLockError("failed to release gateway lock coordinator", releaseError);
}
},
};
}
function readLinuxCmdline(pid: number): string[] | null {
try {
const raw = fsSync.readFileSync(`/proc/${pid}/cmdline`, "utf8");
@@ -260,12 +222,53 @@ async function resolveGatewayOwnerStatus(
async function readLockPayload(lockPath: string): Promise<LockPayload | null> {
try {
const raw = await fs.readFile(lockPath, "utf8");
return safeParseJsonWithSchema(LockPayloadSchema, raw);
return parseGatewayLockPayload(raw);
} catch {
return null;
}
}
function parseGatewayLockPayload(raw: string): LockPayload | null {
return safeParseJsonWithSchema(LockPayloadSchema, raw);
}
async function shouldReclaimGatewayLock(params: {
lockPath: string;
payload: LockPayload | null;
staleMs: number;
now: () => number;
platform: NodeJS.Platform;
readProcessCmdline?: (pid: number) => string[] | null;
readProcessStartTime?: (pid: number) => number | null;
}): Promise<boolean> {
const ownerPid = params.payload?.pid;
const ownerStatus = ownerPid
? await resolveGatewayOwnerStatus(
ownerPid,
params.payload,
params.platform,
params.readProcessCmdline,
params.readProcessStartTime,
)
: "unknown";
if (ownerPid) {
return ownerStatus === "dead";
}
if (params.payload?.createdAt) {
const createdAt = Date.parse(params.payload.createdAt);
if (Number.isFinite(createdAt) && params.now() - createdAt > params.staleMs) {
return true;
}
}
try {
const stat = await fs.stat(params.lockPath);
return params.now() - stat.mtimeMs > params.staleMs;
} catch {
// An unreadable lock can still belong to a healthy gateway. Fail closed.
return false;
}
}
function canonicalizeStateDir(stateDir: string): string {
const resolved = path.resolve(stateDir);
try {
@@ -439,8 +442,6 @@ async function acquireLockFile(
);
const staleMs = resolveTimerTimeoutMs(opts.staleMs, DEFAULT_STALE_MS, 0);
const platform = opts.platform ?? process.platform;
const port = opts.port;
const role = opts.role;
const now = opts.now ?? Date.now;
const sleep =
opts.sleep ??
@@ -453,124 +454,67 @@ async function acquireLockFile(
const startedAt = now();
let lastPayload: LockPayload | null = null;
const buildPayload = (): LockPayload => {
const startTime = (opts.readProcessStartTime ?? ((pid) => readProcessStartTime(pid, platform)))(
process.pid,
);
return {
pid: process.pid,
ownerId: opts.ownerId,
createdAt: resolveTimestampMsToIsoString(now()),
configPath,
stateDir,
...(typeof opts.port === "number" &&
Number.isInteger(opts.port) &&
opts.port > 0 &&
opts.port <= 65_535
? { port: opts.port }
: {}),
...(opts.role !== "gateway" ? { role: opts.role } : {}),
...(typeof startTime === "number" && Number.isFinite(startTime) ? { startTime } : {}),
};
};
const shouldReclaim = (payload: LockPayload | null) =>
shouldReclaimGatewayLock({
lockPath,
payload,
staleMs,
now,
platform,
readProcessCmdline: opts.readProcessCmdline,
readProcessStartTime: opts.readProcessStartTime,
});
while (now() - startedAt < timeoutMs) {
let coordinator: GatewayLockCoordinator | null;
let coordinator: ReturnType<typeof tryAcquireExclusiveSqliteCoordinator>;
try {
coordinator = tryAcquireGatewayLockCoordinator(lockPath);
coordinator = tryAcquireExclusiveSqliteCoordinator(`${lockPath}.sqlite`);
} catch (error) {
throw new GatewayLockError(`failed to acquire gateway lock at ${lockPath}`, error);
}
if (!coordinator) {
lastPayload = await readLockPayload(lockPath);
} else {
let handle: Awaited<ReturnType<typeof fs.open>> | undefined;
let acquisitionError: unknown;
let waitForOwner = false;
try {
while (!handle && !waitForOwner) {
let candidateHandle: Awaited<ReturnType<typeof fs.open>>;
try {
candidateHandle = await fs.open(lockPath, "wx");
} catch (error) {
const code = (error as { code?: unknown }).code;
if (code !== "EEXIST") {
throw error;
}
lastPayload = await readLockPayload(lockPath);
const ownerPid = lastPayload?.pid;
const ownerStatus = ownerPid
? await resolveGatewayOwnerStatus(
ownerPid,
lastPayload,
platform,
opts.readProcessCmdline,
opts.readProcessStartTime,
)
: "unknown";
if (ownerStatus === "dead" && ownerPid) {
await fs.rm(lockPath, { force: true });
continue;
}
if (ownerStatus !== "alive") {
let stale = false;
if (lastPayload?.createdAt) {
const createdAt = Date.parse(lastPayload.createdAt);
stale = Number.isFinite(createdAt) ? now() - createdAt > staleMs : false;
}
if (!stale) {
try {
const st = await fs.stat(lockPath);
stale = now() - st.mtimeMs > staleMs;
} catch {
// On Windows or locked filesystems we may be unable to stat the
// lock file even though the existing gateway is still healthy.
// Treat the lock as non-stale so we keep waiting instead of
// forcefully removing another gateway's lock.
stale = false;
}
}
if (stale) {
await fs.rm(lockPath, { force: true });
continue;
}
}
waitForOwner = true;
continue;
}
try {
const startTime = (
opts.readProcessStartTime ?? ((pid) => readProcessStartTime(pid, platform))
)(process.pid);
const payload: LockPayload = {
pid: process.pid,
ownerId: opts.ownerId,
createdAt: resolveTimestampMsToIsoString(now()),
configPath,
stateDir,
};
if (typeof port === "number" && Number.isInteger(port) && port > 0 && port <= 65_535) {
payload.port = port;
}
if (role !== "gateway") {
payload.role = role;
}
if (typeof startTime === "number" && Number.isFinite(startTime)) {
payload.startTime = startTime;
}
await candidateHandle.writeFile(JSON.stringify(payload), "utf8");
handle = candidateHandle;
} catch (error) {
// Acquisition owns both resources until the release callback exists.
// Unwind them if payload preparation fails before ownership transfers.
await candidateHandle.close().catch(() => undefined);
await fs.rm(lockPath, { force: true }).catch(() => undefined);
throw error;
}
}
} catch (error) {
acquisitionError = error;
}
if (handle) {
const lock = await GATEWAY_LOCKS.acquire(lockPath, {
lockPath,
staleMs,
timeoutMs: 0,
retry: { retries: 0 },
staleRecovery: "remove-if-unchanged",
payload: buildPayload,
parsePayload: parseGatewayLockPayload,
shouldReclaim: ({ payload }) => shouldReclaim(payload as LockPayload | null),
shouldRemoveStaleLock: ({ payload }) => shouldReclaim(payload as LockPayload | null),
});
return {
lockPath,
configPath,
release: async () => {
let releaseError: unknown;
try {
await handle.close();
} catch (error) {
await lock.release().catch((error: unknown) => {
releaseError = error;
}
try {
await fs.rm(lockPath, { force: true });
} catch (error) {
releaseError ??= error;
}
});
try {
coordinator.release();
} catch (error) {
@@ -584,18 +528,13 @@ async function acquireLockFile(
}
},
};
}
try {
coordinator.release();
} catch (error) {
acquisitionError ??= error;
}
if (acquisitionError) {
throw new GatewayLockError(
`failed to acquire gateway lock at ${lockPath}`,
acquisitionError,
);
coordinator.release();
const code = (error as { code?: unknown }).code;
if (code !== "file_lock_timeout" && code !== "file_lock_stale") {
throw new GatewayLockError(`failed to acquire gateway lock at ${lockPath}`, error);
}
lastPayload = await readLockPayload(lockPath);
}
}
src/infra/install-package-dir.test.ts
+7 -13
View File
@@ -57,10 +57,6 @@ function normalizeComparablePath(filePath: string): string {
return path.join(comparableParent, basename);
}
function createFsError(code: string, message = code): NodeJS.ErrnoException {
return Object.assign(new Error(message), { code });
}
async function expectMissingPath(filePath: string): Promise<void> {
try {
await fs.stat(filePath);
@@ -319,7 +315,7 @@ describe("installPackageDir", () => {
await expect(fs.readdir(backupRoot)).resolves.toHaveLength(0);
});
it("publishes the staged install through the copy fallback when rename crosses devices", async () => {
it("publishes through the staged-copy path when source hardlinks are rejected", async () => {
await fixtureRootTracker.setup();
const fixtureRoot = await fixtureRootTracker.make("case");
const sourceDir = path.join(fixtureRoot, "source");
@@ -329,17 +325,15 @@ describe("installPackageDir", () => {
await fs.writeFile(path.join(sourceDir, "marker.txt"), "new");
const realRename = fs.rename.bind(fs);
let exdevMoves = 0;
let directMoves = 0;
vi.spyOn(fs, "rename").mockImplementation(async (...args: Parameters<typeof fs.rename>) => {
const [from, to] = args;
const fromPath = String(from);
if (
exdevMoves === 0 &&
path.basename(fromPath).startsWith(".openclaw-install-stage-") &&
normalizeComparablePath(String(to)) === normalizeComparablePath(targetDir)
) {
exdevMoves += 1;
throw createFsError("EXDEV", "cross-device link not permitted");
directMoves += 1;
}
return await realRename(...args);
});
@@ -355,7 +349,7 @@ describe("installPackageDir", () => {
});
expect(result).toEqual({ ok: true });
expect(exdevMoves).toBe(1);
expect(directMoves).toBe(0);
await expect(fs.readFile(path.join(targetDir, "marker.txt"), "utf8")).resolves.toBe("new");
await expect(
listMatchingDirs(installBaseDir, ".openclaw-install-stage-"),
@@ -443,7 +437,7 @@ describe("installPackageDir", () => {
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error).toContain("Hardlinked source file is not allowed");
expect(result.error).toContain("Refusing to move hardlinked file");
}
await expect(fs.readFile(path.join(targetDir, "marker.txt"), "utf8")).resolves.toBe("old");
},
@@ -472,7 +466,7 @@ describe("installPackageDir", () => {
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error).toContain("Hardlinked source file is not allowed");
expect(result.error).toContain("Refusing to move hardlinked file");
}
await expect(fs.readFile(path.join(targetDir, "marker.txt"), "utf8")).resolves.toBe("old");
},
@@ -506,7 +500,7 @@ describe("installPackageDir", () => {
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.error).toContain("Hardlinked source file is not allowed");
expect(result.error).toContain("Refusing to move hardlinked file");
}
await expectMissingPath(path.join(targetDir, "marker.txt"));
},
src/infra/node-sqlite.ts
+27
View File
@@ -3,6 +3,7 @@ import { createRequire } from "node:module";
import path from "node:path";
import { formatErrorMessage } from "./errors.js";
import { isSqliteWalResetSafeVersion } from "./sqlite-runtime-version.js";
import { isSqliteLockError } from "./sqlite-transaction.js";
import { installProcessWarningFilter } from "./warning-filter.js";
const require = createRequire(import.meta.url);
@@ -96,3 +97,29 @@ export function openNodeSqliteDatabase(
? new sqlite.DatabaseSync(resolvedLocation)
: new sqlite.DatabaseSync(resolvedLocation, options);
}
/** Hold a raw exclusive transaction until release for cross-process coordination. */
export function tryAcquireExclusiveSqliteCoordinator(
location: string,
): { release: () => void } | null {
const database = openNodeSqliteDatabase(location);
try {
// Kysely transaction callbacks cannot own a lock beyond their synchronous commit section.
database.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
} catch (error) {
database.close();
if (isSqliteLockError(error)) {
return null;
}
throw error;
}
return {
release: () => {
try {
database.exec("ROLLBACK");
} finally {
database.close();
}
},
};
}
src/infra/push-apns-auth.ts
+2 -2
View File
@@ -1,8 +1,8 @@
// Resolves APNs provider credentials and owns provider-token signing/cache state.
import { createHash, createPrivateKey, sign as signJwt } from "node:crypto";
import fs from "node:fs/promises";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { formatErrorMessage } from "./errors.js";
import { readSecretFile } from "./fs-safe-advanced.js";
/** Direct APNs provider authentication used to mint ES256 bearer tokens. */
export type ApnsAuthConfig = {
@@ -104,7 +104,7 @@ export async function resolveApnsAuthConfigFromEnv(
};
}
try {
const privateKey = normalizePrivateKey(await fs.readFile(keyPath, "utf8"));
const privateKey = normalizePrivateKey(await readSecretFile(keyPath, "APNs private key"));
return {
ok: true,
value: {
src/infra/replace-file.test.ts
+1 -1
View File
@@ -24,7 +24,7 @@ describe("movePathWithCopyFallback", () => {
sourceHardlinks: "reject",
to: targetDir,
}),
).rejects.toThrow("Hardlinked source file is not allowed");
).rejects.toMatchObject({ code: "hardlink" });
await expect(fs.readFile(sourceFile, "utf8")).resolves.toBe("hello");
let statError: NodeJS.ErrnoException | undefined;
src/infra/replace-file.ts
+6 -54
View File
@@ -1,60 +1,12 @@
// Wraps fs-safe atomic replacement and move helpers for OpenClaw install flows.
import "./fs-safe-defaults.js";
import fs from "node:fs/promises";
import path from "node:path";
import {
movePathWithCopyFallback as movePathWithCopyFallbackBase,
replaceFileAtomic as replaceFileAtomicBase,
type MovePathWithCopyFallbackOptions as BaseMovePathWithCopyFallbackOptions,
} from "@openclaw/fs-safe/atomic";
import { replaceFileAtomic as replaceFileAtomicBase } from "@openclaw/fs-safe/atomic";
export { replaceDirectoryAtomic, replaceFileAtomicSync } from "@openclaw/fs-safe/atomic";
export {
movePathWithCopyFallback,
replaceDirectoryAtomic,
replaceFileAtomicSync,
} from "@openclaw/fs-safe/atomic";
/** Atomic file replacement primitive re-exported through the fs-safe defaults shim. */
export const replaceFileAtomic = replaceFileAtomicBase;
/** Options for moving paths while optionally rejecting hardlinked source files. */
type MovePathWithCopyFallbackOptions = BaseMovePathWithCopyFallbackOptions & {
sourceHardlinks?: "allow" | "reject";
};
/**
* Moves a path using fs-safe's copy fallback, with an OpenClaw hardlink guard
* for install/update flows that must not preserve package-manager links.
*/
export async function movePathWithCopyFallback(
options: MovePathWithCopyFallbackOptions,
): Promise<void> {
if (options.sourceHardlinks === "reject") {
await assertNoHardlinkedSourceFiles(options.from);
}
await movePathWithCopyFallbackBase({ from: options.from, to: options.to });
}
async function assertNoHardlinkedSourceFiles(sourcePath: string): Promise<void> {
const sourceStat = await fs.lstat(sourcePath);
if (sourceStat.isFile() && sourceStat.nlink > 1) {
throw new Error(`Hardlinked source file is not allowed: ${sourcePath}`);
}
if (!sourceStat.isDirectory()) {
return;
}
const entries = await fs.readdir(sourcePath, { withFileTypes: true });
await Promise.all(
entries.map(async (entry) => {
const entryPath = path.join(sourcePath, entry.name);
if (entry.isDirectory()) {
await assertNoHardlinkedSourceFiles(entryPath);
return;
}
if (!entry.isFile()) {
return;
}
const entryStat = await fs.lstat(entryPath);
if (entryStat.nlink > 1) {
throw new Error(`Hardlinked source file is not allowed: ${entryPath}`);
}
}),
);
}
src/infra/root-walk.ts
+14
View File
@@ -0,0 +1,14 @@
// Root-bounded directory walking facade for explicit plugin SDK consumers.
import "./fs-safe-defaults.js";
import { root, type RootWalkEntry, type RootWalkOptions } from "@openclaw/fs-safe/root";
export type { RootWalkEntry, RootWalkOptions } from "@openclaw/fs-safe/root";
export async function* walkRootDirectory(
rootDir: string,
relativePath: string,
options: RootWalkOptions,
): AsyncGenerator<RootWalkEntry> {
const capability = await root(rootDir);
yield* capability.walk(relativePath, options);
}
src/infra/secret-file.ts
+2
View File
@@ -9,9 +9,11 @@ import {
import { resolveUserPath } from "../utils.js";
export {
createSecretFileAtomic,
DEFAULT_SECRET_FILE_MAX_BYTES,
PRIVATE_SECRET_DIR_MODE,
PRIVATE_SECRET_FILE_MODE,
readSecretFile,
readSecretFileSync,
type SecretFileReadOptions,
} from "@openclaw/fs-safe/secret";
src/infra/sqlite-snapshot.test.ts
+2 -2
View File
@@ -304,11 +304,11 @@ describe("createVerifiedSqliteSnapshot", () => {
const sourcePath = path.join(tempDir, "source.sqlite");
const targetPath = path.join(tempDir, "snapshot.sqlite");
createHotRollbackJournal(sourcePath);
const journalPath = `${sourcePath}-journal`;
const canonicalJournalPath = `${fsSync.realpathSync.native(sourcePath)}-journal`;
const lstatSync = fsSync.lstatSync.bind(fsSync);
let hidJournal = false;
vi.spyOn(fsSync, "lstatSync").mockImplementation(((pathname, options) => {
if (!hidJournal && path.resolve(String(pathname)) === path.resolve(journalPath)) {
if (!hidJournal && path.resolve(String(pathname)) === canonicalJournalPath) {
hidJournal = true;
const error = new Error("missing");
(error as NodeJS.ErrnoException).code = "ENOENT";
src/infra/sqlite-snapshot.ts
+90 -93
View File
@@ -6,9 +6,16 @@ import type { FileHandle } from "node:fs/promises";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js";
import { pinDirectory, requireDirectorySync, syncDirectory } from "./directory-durability.js";
import {
getPublishFileExclusiveFailureDetails,
isHardlinkFallbackError,
pinDirectory,
publishFileNoClobber,
requireDirectorySync,
syncDirectory,
} from "./directory-durability.js";
import { formatErrorMessage } from "./errors.js";
import { sameFileIdentity } from "./fs-safe-advanced.js";
import { sameFileIdentity, type FileIdentityStat } from "./fs-safe-advanced.js";
import {
openNodeSqliteDatabase,
requireNodeSqlite,
@@ -341,9 +348,12 @@ function assertExpectedContent(
}
}
type CleanupIdentity = FileIdentityStat &
Partial<Pick<Stats, "birthtimeMs" | "ctimeMs" | "mtimeMs" | "size">>;
function removePublishedTargetIfOwned(
filePath: string,
expectedIdentity: Stats,
expectedIdentity: CleanupIdentity,
requireFingerprint = false,
): boolean {
let currentIdentity: Stats;
@@ -354,7 +364,11 @@ function removePublishedTargetIfOwned(
}
const fingerprintMatches =
!requireFingerprint ||
(expectedIdentity.size === currentIdentity.size &&
(typeof expectedIdentity.size === "number" &&
typeof expectedIdentity.mtimeMs === "number" &&
typeof expectedIdentity.ctimeMs === "number" &&
typeof expectedIdentity.birthtimeMs === "number" &&
expectedIdentity.size === currentIdentity.size &&
expectedIdentity.mtimeMs === currentIdentity.mtimeMs &&
expectedIdentity.ctimeMs === currentIdentity.ctimeMs &&
expectedIdentity.birthtimeMs === currentIdentity.birthtimeMs);
@@ -371,6 +385,17 @@ function removePublishedTargetIfOwned(
}
}
function sameFileStatFingerprint(left: Stats, right: Stats): boolean {
// Creating the publication hard link changes source ctime, so compare the
// mutation fields that remain stable for the same bytes and pathname owner.
return (
sameFileIdentity(left, right) &&
left.size === right.size &&
left.mtimeMs === right.mtimeMs &&
left.birthtimeMs === right.birthtimeMs
);
}
function assertSynchronousCallbackResult(result: unknown, label: string): void {
if (
result &&
@@ -382,17 +407,6 @@ function assertSynchronousCallbackResult(result: unknown, label: string): void {
}
}
function isLinkFallbackError(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException).code;
return (
code === "EPERM" ||
code === "EXDEV" ||
code === "ENOTSUP" ||
code === "EOPNOTSUPP" ||
code === "ENOSYS"
);
}
/**
* Publish the exact bytes of one already-verified SQLite file without reopening
* its pathname during the copy. The target is always created exclusively.
@@ -421,18 +435,15 @@ export async function publishVerifiedSqliteFile(
let source: FileHandle | undefined;
let target: FileHandle | undefined;
let targetPinFileDescriptor: number | undefined;
let verifiedStagedIdentity: Stats | undefined;
let linkedCandidateIdentity: Stats | undefined;
let failedPublicationIdentity: FileIdentityStat | undefined;
let publishedIdentity: Stats | undefined;
let ownershipPinned = false;
let hardLinkCreated = false;
try {
stagingIdentity = await fs.lstat(stagingDir);
await fs.chmod(stagingDir, 0o700);
source = await fs.open(options.sourcePath, "r");
await assertOpenFileIdentity(source, options.sourcePath, options.sourceIdentity);
const staged = await copyFileExclusive(source, stagedPath);
verifiedStagedIdentity = staged.identity;
const expectedContent = options.expectedContent;
assertExpectedContent(staged.content, expectedContent, options.targetPath);
await source.close();
@@ -443,76 +454,89 @@ export async function publishVerifiedSqliteFile(
assertExpectedContent(validatedContent, expectedContent, options.targetPath);
await options.beforePublish?.();
await assertTargetAbsent(options.targetPath);
let usedHardLink = false;
const currentStagedIdentity = await fs.lstat(stagedPath);
if (!sameFileStatFingerprint(staged.identity, currentStagedIdentity)) {
throw new Error(`SQLite snapshot staging file changed during publication: ${stagedPath}`);
}
try {
await fs.link(stagedPath, options.targetPath);
usedHardLink = true;
hardLinkCreated = true;
const publication = await publishFileNoClobber(stagedPath, options.targetPath, {
strategy: options.requireAtomicPublication ? "link-required" : "link-or-copy",
durability: "fail-closed",
});
publishedIdentity = publication.identity;
} catch (error) {
if (!isLinkFallbackError(error)) {
throw error;
const details = getPublishFileExclusiveFailureDetails(error);
const stagedAfterFailure = details?.targetCreated
? await fs.lstat(stagedPath).catch(() => undefined)
: undefined;
const targetAfterFailure = details?.targetCreated
? await fs.lstat(options.targetPath).catch(() => undefined)
: undefined;
const stagedPathChanged =
!stagedAfterFailure || !sameFileStatFingerprint(staged.identity, stagedAfterFailure);
if (
details?.targetCreated &&
details.cleanup !== "removed" &&
stagedAfterFailure &&
targetAfterFailure &&
sameFileIdentity(stagedAfterFailure, targetAfterFailure)
) {
// fs-safe proves this call created the path; the SQLite layer can also
// prove it linked from the staged name, so cleanup remains owned.
failedPublicationIdentity = targetAfterFailure;
} else if (
details?.targetCreated &&
details.cleanup !== "removed" &&
details.targetIdentity &&
targetAfterFailure &&
sameFileIdentity(details.targetIdentity, targetAfterFailure)
) {
// Copy fallback has a distinct inode; the receipt ties its created
// identity to the current path before fingerprint-guarded cleanup.
failedPublicationIdentity = targetAfterFailure;
}
if (options.requireAtomicPublication) {
if (options.requireAtomicPublication && isHardlinkFallbackError(error)) {
throw new Error(
`Atomic SQLite publication requires hard-link support in ${targetDirectory}.`,
{ cause: error },
);
}
const stagedSource = await fs.open(stagedPath, "r");
try {
const copied = await copyFileExclusive(stagedSource, options.targetPath);
publishedIdentity = copied.identity;
assertExpectedContent(copied.content, expectedContent, options.targetPath);
} finally {
await stagedSource.close();
}
}
if (usedHardLink) {
target = await fs.open(options.targetPath, "r");
const linkedIdentity = await target.stat();
linkedCandidateIdentity = linkedIdentity;
const currentTargetIdentity = await fs.lstat(options.targetPath);
const currentStagedIdentity = await fs.lstat(stagedPath);
if (!sameFileIdentity(linkedIdentity, currentTargetIdentity)) {
throw new Error(`SQLite snapshot target changed during publication: ${options.targetPath}`);
}
const matchesVerifiedStaging = sameFileIdentity(staged.identity, linkedIdentity);
const matchesCurrentStaging = sameFileIdentity(currentStagedIdentity, linkedIdentity);
if (matchesVerifiedStaging || matchesCurrentStaging) {
// The target handle pins exactly what link() published for ownership-safe cleanup.
publishedIdentity = linkedIdentity;
ownershipPinned = true;
}
if (!matchesCurrentStaging) {
throw new Error(`SQLite snapshot staging path changed after publication: ${stagedPath}`);
}
if (!matchesVerifiedStaging) {
if (details?.targetCreated) {
if (stagedPathChanged) {
throw new Error(
`SQLite snapshot staging file changed during publication: ${options.targetPath}`,
{ cause: error },
);
}
throw new Error(
`SQLite snapshot staging file changed during publication: ${options.targetPath}`,
`SQLite snapshot target changed during publication: ${options.targetPath}`,
{ cause: error },
);
}
throw error;
}
if (!publishedIdentity) {
throw new Error(`SQLite snapshot target was not published: ${options.targetPath}`);
}
const initialPublishedIdentity = publishedIdentity;
target ??= await fs.open(options.targetPath, "r");
target = await fs.open(options.targetPath, "r");
await assertOpenFileIdentity(target, options.targetPath, initialPublishedIdentity);
ownershipPinned = true;
requireDirectorySync(
await syncDirectory(targetDirectoryReceipt),
"SQLite publication directory",
);
// Retire the writable staging hard link before the final byte verification.
await fs.unlink(stagedPath);
const expectedIdentity = await target.stat();
publishedIdentity = expectedIdentity;
const publishedContent = await hashOpenPublishedFile(
target,
options.targetPath,
expectedIdentity,
);
assertExpectedContent(publishedContent, expectedContent, options.targetPath);
await fs.rmdir(stagingDir);
requireDirectorySync(
await syncDirectory(targetDirectoryReceipt),
"SQLite publication directory",
);
const linkedContent = await hashOpenPublishedFile(target, options.targetPath, expectedIdentity);
assertExpectedContent(linkedContent, expectedContent, options.targetPath);
await target.close();
target = undefined;
ownershipPinned = false;
@@ -545,34 +569,6 @@ export async function publishVerifiedSqliteFile(
targetPinFileDescriptor = undefined;
ownershipPinned = false;
} catch (error) {
if (!publishedIdentity && hardLinkCreated && verifiedStagedIdentity) {
const currentTargetIdentity = await fs.lstat(options.targetPath).catch(() => undefined);
const currentStagedIdentity = await fs.lstat(stagedPath).catch(() => undefined);
const targetMatchesStaging =
currentTargetIdentity &&
currentStagedIdentity &&
sameFileIdentity(currentTargetIdentity, currentStagedIdentity);
const targetMatchesVerified =
currentTargetIdentity && sameFileIdentity(currentTargetIdentity, verifiedStagedIdentity);
if (targetMatchesStaging || targetMatchesVerified) {
publishedIdentity = currentTargetIdentity;
ownershipPinned = Boolean(targetMatchesStaging);
}
}
if (!publishedIdentity && target && linkedCandidateIdentity && verifiedStagedIdentity) {
const currentTargetIdentity = await fs.lstat(options.targetPath).catch(() => undefined);
const currentStagedIdentity = await fs.lstat(stagedPath).catch(() => undefined);
const targetStillMatches =
currentTargetIdentity && sameFileIdentity(currentTargetIdentity, linkedCandidateIdentity);
const targetCameFromStaging =
(currentStagedIdentity &&
sameFileIdentity(currentStagedIdentity, linkedCandidateIdentity)) ||
sameFileIdentity(verifiedStagedIdentity, linkedCandidateIdentity);
if (targetStillMatches && targetCameFromStaging) {
publishedIdentity = linkedCandidateIdentity;
ownershipPinned = true;
}
}
if (target && publishedIdentity) {
const openedIdentity = await target.stat().catch(() => undefined);
if (openedIdentity && sameFileIdentity(openedIdentity, publishedIdentity)) {
@@ -580,10 +576,11 @@ export async function publishVerifiedSqliteFile(
ownershipPinned = true;
}
}
if (publishedIdentity) {
const cleanupIdentity = publishedIdentity ?? failedPublicationIdentity;
if (cleanupIdentity) {
const removed = removePublishedTargetIfOwned(
options.targetPath,
publishedIdentity,
cleanupIdentity,
!ownershipPinned,
);
if (removed) {
src/infra/state-migrations.audit-logs.test.ts
+3 -1
View File
@@ -854,7 +854,9 @@ describe("legacy core audit log migration", () => {
mode: number,
) {
chmodCalls += 1;
if (chmodCalls === 3) {
// fs-safe 0.5 applies the requested mode while creating the sanitized file before the
// migration's explicit hardening checks. Fail the later raw-archive hardening call.
if (chmodCalls === 4) {
return Promise.reject(new Error("simulated chmod failure"));
}
return originalChmod.call(this, mode);
src/infra/state-migrations.audit-recovery.test.ts
+2 -1
View File
@@ -381,7 +381,8 @@ describe("legacy audit recovery byte handling", () => {
mode: number,
) {
chmodCalls += 1;
if (chmodCalls === 2) {
// fs-safe 0.5 applies the write mode before the migration's explicit hardening check.
if (chmodCalls === 3) {
return Promise.reject(new Error("simulated recovery chmod failure"));
}
return originalChmod.call(this, mode);
src/infra/state-migrations.mcp-oauth-lock.ts
+35 -90
View File
@@ -1,22 +1,16 @@
import { randomUUID } from "node:crypto";
import type { Root } from "@openclaw/fs-safe";
import path from "node:path";
import { getFileLockProcessStartTime } from "../shared/pid-alive.js";
import { asFsSafeFileLockRoot, createFileLockManager } from "./file-lock-manager.js";
import type { Root } from "./fs-safe.js";
import { isDefinitelyStaleLegacyMcpOAuthLock } from "./state-migrations.mcp-oauth-lock-stale.js";
const LOCK_RETRIES = 20;
const LOCK_RETRY_FACTOR = 1.3;
const LOCK_RETRY_MIN_MS = 25;
const LOCK_RETRY_MAX_MS = 500;
const MAX_LEGACY_LOCK_BYTES = 64 * 1024;
function retryDelayMs(attempt: number): number {
return Math.min(
LOCK_RETRY_MAX_MS,
Math.max(LOCK_RETRY_MIN_MS, LOCK_RETRY_MIN_MS * LOCK_RETRY_FACTOR ** attempt),
);
}
function createLockPayload(): string {
const MCP_OAUTH_LOCKS = createFileLockManager("openclaw.mcp-oauth-legacy-migration");
function createLockPayload(): Record<string, unknown> {
const payload: Record<string, unknown> = {
pid: process.pid,
createdAt: new Date().toISOString(),
@@ -26,81 +20,16 @@ function createLockPayload(): string {
if (starttime !== null) {
payload.starttime = starttime;
}
return `${JSON.stringify(payload, null, 2)}\n`;
return payload;
}
function isAlreadyExists(error: unknown): boolean {
return (error as { code?: unknown }).code === "already-exists";
}
function isNotFound(error: unknown): boolean {
const code = (error as { code?: unknown }).code;
return code === "ENOENT" || code === "not-found";
}
async function hasDefinitelyStaleLock(params: {
stateRoot: Root;
lockRelativePath: string;
}): Promise<boolean> {
try {
const observed = await params.stateRoot.read(params.lockRelativePath, {
maxBytes: MAX_LEGACY_LOCK_BYTES,
});
return isDefinitelyStaleLegacyMcpOAuthLock({ raw: observed.buffer.toString("utf8") });
} catch (error) {
if (isNotFound(error)) {
return false;
}
throw error;
}
}
async function acquireRootBoundedLegacyLock(params: {
stateRoot: Root;
targetRelativePath: string;
}): Promise<() => Promise<void>> {
const lockRelativePath = `${params.targetRelativePath}.lock`;
const raw = createLockPayload();
for (let attempt = 0; ; attempt += 1) {
try {
// Root.create pins the parent directory and uses no-clobber semantics, so
// a directory swap cannot redirect the retired runtime's sidecar outside stateDir.
await params.stateRoot.create(lockRelativePath, raw, { mode: 0o600 });
break;
} catch (error) {
if (!isAlreadyExists(error) || attempt >= LOCK_RETRIES) {
if (isAlreadyExists(error)) {
throw new Error(`file lock timeout for ${params.targetRelativePath}`, {
cause: error,
});
}
throw error;
}
if (
await hasDefinitelyStaleLock({
stateRoot: params.stateRoot,
lockRelativePath,
})
) {
// POSIX path removal cannot be fenced against a replacement owner.
// Security-sensitive migration therefore reports stale proof but never unlinks it.
throw Object.assign(new Error(`file lock stale for ${params.targetRelativePath}`), {
code: "file_lock_stale",
});
}
await new Promise<void>((resolve) => {
setTimeout(resolve, retryDelayMs(attempt));
});
}
}
return async () => {
const current = await params.stateRoot.readText(lockRelativePath);
if (current !== raw) {
throw new Error(`legacy file lock ownership changed for ${params.targetRelativePath}`);
}
await params.stateRoot.remove(lockRelativePath);
};
function readLegacyRawPayload(payload: unknown): string {
return payload &&
typeof payload === "object" &&
"raw" in payload &&
typeof payload.raw === "string"
? payload.raw
: "";
}
/** Share the retired runtime's sidecar protocol without leaving the pinned state root. */
@@ -108,10 +37,26 @@ export async function withRootBoundedLegacyFileLock<T>(
params: { stateRoot: Root; targetRelativePath: string },
run: () => Promise<T>,
): Promise<T> {
const release = await acquireRootBoundedLegacyLock(params);
try {
return await run();
} finally {
await release();
}
const targetPath = path.resolve(params.stateRoot.rootReal, params.targetRelativePath);
return await MCP_OAUTH_LOCKS.withLock(
targetPath,
{
lockPath: `${targetPath}.lock`,
lockRoot: asFsSafeFileLockRoot(params.stateRoot),
retry: {
retries: LOCK_RETRIES,
factor: LOCK_RETRY_FACTOR,
minTimeout: LOCK_RETRY_MIN_MS,
maxTimeout: LOCK_RETRY_MAX_MS,
},
staleRecovery: "fail-closed",
payload: createLockPayload,
parsePayload: (raw) => ({ raw }),
shouldReclaim: ({ payload }) =>
isDefinitelyStaleLegacyMcpOAuthLock({
raw: readLegacyRawPayload(payload),
}),
},
run,
);
}
src/infra/tls/gateway.test.ts
+7
View File
@@ -22,6 +22,13 @@ vi.mock("@openclaw/fs-safe/durability", async (importOriginal) => {
const actual = await importOriginal<typeof import("@openclaw/fs-safe/durability")>();
return {
...actual,
publishFileExclusive: async (...args: Parameters<typeof actual.publishFileExclusive>) => {
const result = await actual.publishFileExclusive(...args);
return {
...result,
directorySync: durabilityTestState.syncOutcome ?? result.directorySync,
};
},
syncDirectory: async (...args: Parameters<typeof actual.syncDirectory>) =>
durabilityTestState.syncOutcome ?? (await actual.syncDirectory(...args)),
};
src/infra/tls/gateway.ts
+24 -49
View File
@@ -5,14 +5,10 @@ import type { Stats } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import tls from "node:tls";
import {
ensureDurableDirectory,
syncDirectory,
type DirectoryReceipt,
} from "@openclaw/fs-safe/durability";
import type { GatewayTlsConfig } from "../../config/types.gateway.js";
import { runExec } from "../../process/exec.js";
import { CONFIG_DIR, resolveUserPath, shortenHomeInString } from "../../utils.js";
import { ensureDurableDirectory, publishFileNoClobber } from "../directory-durability.js";
import { sameFileIdentity } from "../fs-safe-advanced.js";
import { canonicalPathFromExistingAncestor, pathExists } from "../fs-safe.js";
import { resolveSystemBin } from "../resolve-system-bin.js";
@@ -43,11 +39,6 @@ function gatewayTlsDegradation(reason: GatewayTlsDegradation["reason"]): Gateway
};
}
function isHardLinkUnsupportedError(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
return code === "ENOTSUP" || code === "EOPNOTSUPP" || code === "EPERM";
}
type PublishedGeneratedTlsOutput = {
degradationReasons: GatewayTlsDegradation["reason"][];
identity: Stats;
@@ -56,8 +47,6 @@ type PublishedGeneratedTlsOutput = {
async function publishGeneratedTlsOutput(
stagedPath: string,
finalPath: string,
contents: string,
parentReceipt: DirectoryReceipt,
): Promise<PublishedGeneratedTlsOutput> {
const degradationReasons: GatewayTlsDegradation["reason"][] = [];
const stagedHandle = await fs.open(stagedPath, "r+");
@@ -68,42 +57,32 @@ async function publishGeneratedTlsOutput(
} finally {
await stagedHandle.close();
}
let publishedIdentity: Stats | undefined;
// Publication failures deliberately preserve any final path already created. Node has no
// cross-platform unlink-if-inode-matches primitive, so rollback could delete a replacement.
try {
await fs.link(stagedPath, finalPath);
const linkedIdentity = await fs.lstat(finalPath);
if (!linkedIdentity.isFile() || !sameFileIdentity(stagedIdentity, linkedIdentity)) {
throw new Error(`Generated TLS output changed during publication: ${finalPath}`);
}
publishedIdentity = linkedIdentity;
} catch (error) {
if (!isHardLinkUnsupportedError(error)) {
throw error;
}
degradationReasons.push("atomic hard-link publication unavailable");
// Some supported filesystems cannot publish with hard links. An exclusive handle keeps
// no-overwrite semantics without pathname cleanup that could delete concurrent output.
const handle = await fs.open(finalPath, "wx", 0o600);
try {
publishedIdentity = await handle.stat();
await handle.writeFile(contents, "utf8");
await handle.sync();
} finally {
await handle.close();
}
}
if (!publishedIdentity) {
throw new Error(`Generated TLS output was not published: ${finalPath}`);
}
const directorySync = await syncDirectory(parentReceipt, {
label: "gateway TLS publication directory",
const publication = await publishFileNoClobber(stagedPath, finalPath, {
strategy: "link-or-copy",
durability: "degrade",
});
if (directorySync.status === "unsupported") {
if (publication.method === "exclusive-copy") {
degradationReasons.push("atomic hard-link publication unavailable");
}
if (publication.durability === "degraded") {
degradationReasons.push("directory durability unavailable");
}
return { degradationReasons, identity: publishedIdentity };
const [currentStagedIdentity, currentPublishedIdentity] = await Promise.all([
fs.lstat(stagedPath),
fs.lstat(finalPath),
]);
const hardlinkChanged =
publication.method === "hardlink" && !sameFileIdentity(stagedIdentity, publication.identity);
if (
!currentStagedIdentity.isFile() ||
!currentPublishedIdentity.isFile() ||
!sameFileIdentity(stagedIdentity, currentStagedIdentity) ||
!sameFileIdentity(publication.identity, currentPublishedIdentity) ||
hardlinkChanged
) {
throw new Error(`Generated TLS output changed during publication: ${finalPath}`);
}
return { degradationReasons, identity: publication.identity };
}
// Gateway TLS runtime carries loaded cert material plus the normalized SHA-256
@@ -183,16 +162,12 @@ async function generateSelfSignedCert(params: {
const certPublication = await publishGeneratedTlsOutput(
stagedCertPath,
path.join(certDirectory.path, path.basename(params.certPath)),
cert,
certDirectory,
);
certPublication.degradationReasons.forEach((reason) => degradationReasons.add(reason));
// Preserve the published certificate on key failure: conditional pathname removal is not atomic.
const keyPublication = await publishGeneratedTlsOutput(
stagedKeyPath,
path.join(keyDirectory.path, path.basename(params.keyPath)),
key,
keyDirectory,
);
keyPublication.degradationReasons.forEach((reason) => degradationReasons.add(reason));
for (const reason of degradationReasons) {
src/plugin-sdk/archive.ts
+10
View File
@@ -0,0 +1,10 @@
/** Public SDK subpath for bounded archive extraction and single-entry reads. */
export {
ARCHIVE_LIMIT_ERROR_CODE,
ArchiveLimitError,
extractArchive,
readArchiveEntry,
type ArchiveEntryKind,
type ArchiveExtractLimits,
type ExtractArchiveOptions,
} from "../infra/archive.js";
src/plugin-sdk/file-lock.test.ts
+53
View File
@@ -86,6 +86,59 @@ describe("acquireFileLock", () => {
await lock.release();
});
it.runIf(process.platform !== "win32")(
"shares canonical aliases for one logical owner until the final release",
async () => {
const realDir = path.join(tempDir, "real");
const linkDir = path.join(tempDir, "link");
await fs.mkdir(realDir);
await fs.symlink(realDir, linkDir, "dir");
const realPath = path.join(realDir, "state.json");
const linkPath = path.join(linkDir, "state.json");
const options = {
retries: { retries: 0, factor: 1, minTimeout: 1, maxTimeout: 1 },
stale: 60_000,
reentrantOwner: "test-operation:alias",
} as const;
const first = await acquireFileLock(realPath, options);
const second = await acquireFileLock(linkPath, options);
try {
expect(second.lockPath).toBe(first.lockPath);
await first.release();
await expect(fs.access(first.lockPath)).resolves.toBeUndefined();
await second.release();
await expect(fs.access(first.lockPath)).rejects.toMatchObject({ code: "ENOENT" });
} finally {
await first.release();
await second.release();
}
},
);
it.each([
["different", "test-operation:second"],
["absent", undefined],
] as const)("makes a %s logical owner contend normally", async (_label, reentrantOwner) => {
const filePath = path.join(tempDir, "owner-isolation.json");
const first = await acquireFileLock(filePath, {
retries: { retries: 0, factor: 1, minTimeout: 1, maxTimeout: 1 },
stale: 60_000,
reentrantOwner: "test-operation:first",
});
try {
await expect(
acquireFileLock(filePath, {
retries: { retries: 0, factor: 1, minTimeout: 1, maxTimeout: 1 },
stale: 60_000,
reentrantOwner,
}),
).rejects.toMatchObject({ code: FILE_LOCK_TIMEOUT_ERROR_CODE });
} finally {
await first.release();
}
});
it("fails closed for a security-sensitive stale lock", async () => {
const filePath = path.join(tempDir, "exec-approvals.json");
const lockPath = `${filePath}.lock`;
src/plugin-sdk/file-lock.ts
+21 -9
View File
@@ -14,7 +14,7 @@ import { getFileLockProcessStartTime } from "../shared/pid-alive.js";
/** Retry and stale-recovery policy for acquiring a filesystem lock. */
export type FileLockOptions = {
/** Retry policy used while waiting for another process or re-entrant holder to release. */
/** Retry policy used while waiting for another process or logical holder to release. */
retries: {
retries: number;
factor: number;
@@ -26,6 +26,11 @@ export type FileLockOptions = {
stale: number;
/** Fail closed for security-sensitive state; generic locks retain shipped stale recovery. */
staleRecovery?: "fail-closed" | "remove-if-unchanged";
/**
* Logical operation identity for intentional nested acquisition.
* Reuse one key only within that call chain; omit it for ordinary contention.
*/
reentrantOwner?: string;
};
/** Live file-lock handle returned after successful acquisition. */
@@ -80,6 +85,12 @@ function createCurrentProcessLockPayload(): Record<string, unknown> {
return payload;
}
function asLockPayload(payload: unknown): Record<string, unknown> | null {
return payload && typeof payload === "object" && !Array.isArray(payload)
? (payload as Record<string, unknown>)
: null;
}
function sameStatValue(left: number | bigint, right: number | bigint): boolean {
return typeof left === typeof right ? left === right : BigInt(left) === BigInt(right);
}
@@ -139,7 +150,7 @@ export async function drainFileLockStateForTest(): Promise<void> {
await drainFileLockManagerForTest(FILE_LOCK_MANAGER_KEY, FILE_LOCK_MANAGER_KEY);
}
/** Acquire a re-entrant process-local file lock backed by a `.lock` sidecar file. */
/** Acquire an owner-scoped process-local file lock backed by a `.lock` sidecar file. */
export async function acquireFileLock(
filePath: string,
options: FileLockOptions,
@@ -151,21 +162,21 @@ export async function acquireFileLock(
staleMs: options.stale,
retry: options.retries,
staleRecovery,
allowReentrant: true,
reentrantOwner: options.reentrantOwner,
payload: createCurrentProcessLockPayload,
shouldReclaim: (params) =>
staleRecovery === "fail-closed"
? isLockOwnerDefinitelyStale({ payload: params.payload })
? isLockOwnerDefinitelyStale({ payload: asLockPayload(params.payload) })
: shouldRemoveDeadOwnerOrExpiredLock({
payload: params.payload,
payload: asLockPayload(params.payload),
staleMs: params.staleMs,
nowMs: params.nowMs,
}),
...(staleRecovery === "remove-if-unchanged"
? {
shouldRemoveStaleLock: (snapshot: { payload: Record<string, unknown> | null }) =>
shouldRemoveStaleLock: (snapshot: { payload: unknown }) =>
shouldRemoveDeadOwnerOrExpiredLock({
payload: snapshot.payload,
payload: asLockPayload(snapshot.payload),
staleMs: options.stale,
}),
}
@@ -199,8 +210,9 @@ export async function reclaimDefinitelyStaleFileLock(
// Pin approval to the regular-file identity first observed. fs-safe then
// rechecks that identity and raw payload immediately before path removal.
const ownerIsDefinitelyStale = async (payload: Record<string, unknown> | null) =>
(await isSameRegularFile(lockPath, observed)) && isLockOwnerDefinitelyStale({ payload });
const ownerIsDefinitelyStale = async (payload: unknown) =>
(await isSameRegularFile(lockPath, observed)) &&
isLockOwnerDefinitelyStale({ payload: asLockPayload(payload) });
const targetPath = lockPath.endsWith(".lock") ? lockPath.slice(0, -".lock".length) : lockPath;
try {
const reclaimed = await acquireFsSafeFileLock(targetPath, {
src/plugin-sdk/infra-runtime.ts
+11 -1
View File
@@ -245,7 +245,17 @@ export * from "../infra/outbound/send-deps.js";
export * from "../infra/retry.js";
export * from "../infra/retry-policy.js";
export * from "../infra/scp-host.ts";
export * from "../infra/secret-file.js";
export {
DEFAULT_SECRET_FILE_MAX_BYTES,
loadSecretFileSync,
PRIVATE_SECRET_DIR_MODE,
PRIVATE_SECRET_FILE_MODE,
readSecretFileSync,
tryReadSecretFileSync,
writePrivateSecretFileAtomic,
type SecretFileReadOptions,
type SecretFileReadResult,
} from "../infra/secret-file.js";
export * from "../infra/secure-random.js";
export * from "../infra/system-events.js";
export * from "../infra/system-message.ts";
src/plugin-sdk/root-walk.ts
+2
View File
@@ -0,0 +1,2 @@
/** Public SDK subpath for budgeted, root-bounded directory walking. */
export { walkRootDirectory, type RootWalkEntry, type RootWalkOptions } from "../infra/root-walk.js";
src/plugin-sdk/secret-file.ts
+8
View File
@@ -0,0 +1,8 @@
/** Public SDK subpath for pinned secret reads and first-writer-wins creation. */
export {
createSecretFileAtomic,
readSecretFile,
readSecretFileSync,
tryReadSecretFileSync,
type SecretFileReadOptions,
} from "../infra/secret-file.js";
src/snapshot/local-repository.test.ts
+36
View File
@@ -1544,6 +1544,42 @@ describe("local SQLite snapshot repository", () => {
}
});
it("cleans an entry linked from a replaced staging pathname", async () => {
const tempDir = await createTempDir();
const sourcePath = path.join(tempDir, "source.sqlite");
const repositoryPath = path.join(tempDir, "snapshots");
createGenericDatabase(sourcePath);
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
const originalLink = fs.link.bind(fs);
let raced = false;
const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => {
if (
!raced &&
path.basename(String(target)) === SNAPSHOT_SQLITE_FILENAME &&
!path.basename(path.dirname(String(target))).startsWith(".tmp-")
) {
await fs.unlink(source);
await fs.writeFile(source, "raced staging bytes");
raced = true;
}
await originalLink(source, target);
});
try {
await expect(
provider.create({
path: sourcePath,
identity: { role: "generic", id: "replaced-entry-staging" },
}),
).rejects.toThrow(/publication|staging|target/u);
expect(raced).toBe(true);
await expect(provider.list()).resolves.toEqual([]);
await expect(fs.readdir(repositoryPath)).resolves.toEqual([]);
} finally {
linkSpy.mockRestore();
}
});
it("never overwrites a file raced into the final snapshot directory", async () => {
const tempDir = await createTempDir();
const sourcePath = path.join(tempDir, "source.sqlite");
src/snapshot/local-repository.ts
+27 -87
View File
@@ -1,7 +1,6 @@
import { randomUUID } from "node:crypto";
import fsSync, { type Stats } from "node:fs";
import fs from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
@@ -9,7 +8,9 @@ import { z } from "zod";
import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js";
import {
ensureDurableDirectory,
getPublishFileExclusiveFailureDetails,
pinDirectory,
publishFileNoClobber,
requireDirectorySync,
syncDirectory,
syncDirectoryIfSupported,
@@ -902,52 +903,44 @@ function assertDirectoryIdentitySync(directoryPath: string, expectedIdentity: St
}
}
function isSnapshotEntryLinkFallbackError(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException).code;
return (
code === "EPERM" ||
code === "EXDEV" ||
code === "ENOTSUP" ||
code === "EOPNOTSUPP" ||
code === "ENOSYS"
);
}
async function publishSnapshotEntryNoOverwrite(
sourcePath: string,
targetPath: string,
entryName: string,
publishedEntries: Map<string, Stats>,
): Promise<void> {
let linked = false;
let linkedSourceIdentity: Stats | undefined;
let publication: Awaited<ReturnType<typeof publishFileNoClobber>>;
try {
linkedSourceIdentity = await fs.lstat(sourcePath);
await fs.link(sourcePath, targetPath);
publishedEntries.set(entryName, linkedSourceIdentity);
linked = true;
publication = await publishFileNoClobber(sourcePath, targetPath, {
strategy: "link-or-copy",
moveSource: true,
durability: "fail-closed",
});
} catch (error) {
if (!isSnapshotEntryLinkFallbackError(error)) {
throw error;
const details = getPublishFileExclusiveFailureDetails(error);
if (details?.targetCreated && details.cleanup !== "removed") {
const [currentSource, currentTarget] = await Promise.all([
fs.lstat(sourcePath).catch(() => undefined),
fs.lstat(targetPath).catch(() => undefined),
]);
const matchesReceipt =
details.targetIdentity &&
currentTarget &&
sameFileIdentity(details.targetIdentity, currentTarget);
const matchesSource =
currentSource && currentTarget && sameFileIdentity(currentSource, currentTarget);
if (currentTarget && (matchesReceipt || matchesSource)) {
publishedEntries.set(entryName, currentTarget);
}
}
const copiedIdentity = await copySnapshotEntryExclusive(sourcePath, targetPath);
publishedEntries.set(entryName, copiedIdentity);
throw error;
}
const expectedTargetIdentity = publishedEntries.get(entryName);
const expectedTargetIdentity = publication.identity;
publishedEntries.set(entryName, expectedTargetIdentity);
const initialTargetIdentity = await fs.lstat(targetPath);
if (!expectedTargetIdentity || !sameFileIdentity(expectedTargetIdentity, initialTargetIdentity)) {
if (!sameFileIdentity(expectedTargetIdentity, initialTargetIdentity)) {
throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`);
}
if (linked) {
if (!linkedSourceIdentity || !sameFileIdentity(linkedSourceIdentity, initialTargetIdentity)) {
throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`);
}
const sourceIdentity = await fs.lstat(sourcePath);
if (!sameFileIdentity(sourceIdentity, initialTargetIdentity)) {
throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`);
}
}
await fs.unlink(sourcePath);
const finalTargetIdentity = await fs.lstat(targetPath);
if (!sameFileIdentity(initialTargetIdentity, finalTargetIdentity)) {
throw new Error(`SQLite snapshot entry changed after publication: ${targetPath}`);
@@ -955,59 +948,6 @@ async function publishSnapshotEntryNoOverwrite(
publishedEntries.set(entryName, finalTargetIdentity);
}
async function copySnapshotEntryExclusive(sourcePath: string, targetPath: string): Promise<Stats> {
const source = await fs.open(sourcePath, "r");
let target: FileHandle | undefined;
let targetIdentity: Stats | undefined;
try {
target = await fs.open(targetPath, "wx+", SNAPSHOT_FILE_MODE);
targetIdentity = await target.stat();
const buffer = Buffer.allocUnsafe(1024 * 1024);
let offset = 0;
while (true) {
const { bytesRead } = await source.read(buffer, 0, buffer.length, offset);
if (bytesRead === 0) {
break;
}
let bytesWritten = 0;
while (bytesWritten < bytesRead) {
const result = await target.write(
buffer,
bytesWritten,
bytesRead - bytesWritten,
offset + bytesWritten,
);
if (result.bytesWritten === 0) {
throw new Error(`SQLite snapshot entry copy made no progress: ${targetPath}`);
}
bytesWritten += result.bytesWritten;
}
offset += bytesRead;
}
await target.sync();
const finalIdentity = await target.stat();
const currentIdentity = await fs.lstat(targetPath);
if (
!sameFileIdentity(targetIdentity, finalIdentity) ||
!sameFileIdentity(targetIdentity, currentIdentity)
) {
throw new Error(`SQLite snapshot entry changed during copy: ${targetPath}`);
}
return finalIdentity;
} catch (error) {
if (targetIdentity) {
const currentIdentity = await fs.lstat(targetPath).catch(() => undefined);
if (currentIdentity && sameFileIdentity(currentIdentity, targetIdentity)) {
await fs.unlink(targetPath).catch(() => undefined);
}
}
throw error;
} finally {
await target?.close().catch(() => undefined);
await source.close().catch(() => undefined);
}
}
async function assertExactSnapshotContents(snapshotDir: string): Promise<void> {
await assertSnapshotContents(
snapshotDir,
src/snapshot/manifest.ts
+13 -14
View File
@@ -1,7 +1,7 @@
import { createHash } from "node:crypto";
import type { BigIntStats, Stats } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { sha256File } from "../infra/directory-durability.js";
import { sameFileIdentity } from "../infra/fs-safe-advanced.js";
import { root } from "../infra/fs-safe.js";
import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js";
@@ -95,17 +95,15 @@ async function hashFileHandle(
target?: OpenFileHandle,
): Promise<Omit<SnapshotArtifactDigest, "stat">> {
const initialStat = await source.stat({ bigint: true });
const hash = createHash("sha256");
const buffer = Buffer.allocUnsafe(1024 * 1024);
let sizeBytes = 0;
while (true) {
const { bytesRead } = await source.read(buffer, 0, buffer.length, sizeBytes);
if (bytesRead === 0) {
break;
}
hash.update(buffer.subarray(0, bytesRead));
let bytesWritten = 0;
if (target) {
if (target) {
const buffer = Buffer.allocUnsafe(1024 * 1024);
while (true) {
const { bytesRead } = await source.read(buffer, 0, buffer.length, sizeBytes);
if (bytesRead === 0) {
break;
}
let bytesWritten = 0;
while (bytesWritten < bytesRead) {
const result = await target.write(
buffer,
@@ -118,14 +116,15 @@ async function hashFileHandle(
}
bytesWritten += result.bytesWritten;
}
sizeBytes += bytesRead;
}
sizeBytes += bytesRead;
}
const hashed = await sha256File(target ?? source);
const finalStat = await source.stat({ bigint: true });
if (!sameMutationFingerprint(initialStat, finalStat)) {
if (!sameMutationFingerprint(initialStat, finalStat) || (target && sizeBytes !== hashed.bytes)) {
throw new Error("Snapshot artifact changed while being read.");
}
return { sha256: hash.digest("hex"), sizeBytes };
return { sha256: hashed.digest, sizeBytes: hashed.bytes };
}
function sameMutationFingerprint(left: BigIntStats, right: BigIntStats): boolean {