diff --git a/README.md b/README.md index b208bdc63e2a..7cfdb4baec22 100644 --- a/README.md +++ b/README.md @@ -273,7 +273,7 @@ Details: [Release channels](https://docs.openclaw.ai/install/development-channel ## Agent workspace + skills - Workspace root: `~/.openclaw/workspace` (configurable via `agents.defaults.workspace`). -- Injected prompt files: `AGENTS.md`, `SOUL.md`, `TOOLS.md`. +- Injected prompt files: `AGENTS.md`, `SOUL.md`, and other workspace context files. - Skills: `~/.openclaw/workspace/skills//SKILL.md`. ## Configuration diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 312c32b6ce74..3f8693deb2de 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -1662,5 +1662,9 @@ { "source": "Cron CLI", "target": "Cron CLI" + }, + { + "source": "TOOLS.md retired", + "target": "TOOLS.md 已弃用" } ] diff --git a/docs/automation/hooks.md b/docs/automation/hooks.md index f31846e023c4..273ad5647baa 100644 --- a/docs/automation/hooks.md +++ b/docs/automation/hooks.md @@ -251,7 +251,7 @@ Extracts the last user/assistant messages (default 15, configurable with `hooks. "entries": { "bootstrap-extra-files": { "enabled": true, - "paths": ["packages/*/AGENTS.md", "packages/*/TOOLS.md"] + "paths": ["packages/*/AGENTS.md"] } } } @@ -259,7 +259,9 @@ Extracts the last user/assistant messages (default 15, configurable with `hooks. } ``` -`patterns` and `files` are accepted as aliases of `paths`. Paths resolve relative to the workspace and must stay inside it. Only recognized bootstrap basenames are loaded (`AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md`, `MEMORY.md`). +`patterns` and `files` are accepted as aliases of `paths`. Paths resolve relative to the workspace and must stay inside it. Only recognized bootstrap basenames are loaded (`AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md`, `MEMORY.md`). + +`TOOLS.md` is no longer a recognized bootstrap basename and is not loaded into runtime context. `openclaw doctor --fix` migrates the workspace-root `TOOLS.md` into the `## Tools` section of `AGENTS.md`; patterns that name other `TOOLS.md` files are not migrated and should be repointed at `AGENTS.md`. diff --git a/docs/automation/standing-orders.md b/docs/automation/standing-orders.md index ce17c4da5e5b..9b04c1bd7ace 100644 --- a/docs/automation/standing-orders.md +++ b/docs/automation/standing-orders.md @@ -31,7 +31,7 @@ Each program specifies: The agent loads these instructions every session via the workspace bootstrap files (see [Agent Workspace](/concepts/agent-workspace) for the full list of auto-injected files) and executes against them, combined with [cron jobs](/automation/cron-jobs) for time-based enforcement. -Put standing orders in `AGENTS.md` to guarantee they're loaded every session. The workspace bootstrap automatically injects `AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md`, and `MEMORY.md` - but not arbitrary files in subdirectories. +Put standing orders in `AGENTS.md` to guarantee they're loaded every session. The workspace bootstrap automatically injects `AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md`, and `MEMORY.md` - but not arbitrary files in subdirectories. ## Anatomy of a standing order diff --git a/docs/cli/hooks.md b/docs/cli/hooks.md index b337f2210223..25d54698d89b 100644 --- a/docs/cli/hooks.md +++ b/docs/cli/hooks.md @@ -96,13 +96,13 @@ Hook packs install through the unified plugins installer/updater; `openclaw hook ## Bundled hooks -| Hook | Events | What it does | -| --------------------- | ------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| boot-md | `gateway:startup` | Runs `BOOT.md` at gateway startup for each configured agent scope | -| bootstrap-extra-files | `agent:bootstrap` | Injects extra bootstrap files (for example monorepo `AGENTS.md`/`TOOLS.md`) during agent bootstrap | -| command-logger | `command` | Logs command events to `~/.openclaw/logs/commands.log` | -| compaction-notifier | `session:compact:before`, `session:compact:after` | Sends visible chat notices when session compaction starts and finishes | -| session-memory | `command:new`, `command:reset` | Saves session context to memory on `/new` or `/reset` | +| Hook | Events | What it does | +| --------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------------- | +| boot-md | `gateway:startup` | Runs `BOOT.md` at gateway startup for each configured agent scope | +| bootstrap-extra-files | `agent:bootstrap` | Injects extra bootstrap files (for example monorepo `AGENTS.md`) during agent bootstrap | +| command-logger | `command` | Logs command events to `~/.openclaw/logs/commands.log` | +| compaction-notifier | `session:compact:before`, `session:compact:after` | Sends visible chat notices when session compaction starts and finishes | +| session-memory | `command:new`, `command:reset` | Saves session context to memory on `/new` or `/reset` | Enable any bundled hook with `openclaw hooks enable `. Full details, config keys, and defaults: [Bundled hooks](/automation/hooks#bundled-hooks). diff --git a/docs/cli/onboard.md b/docs/cli/onboard.md index b43de7ec6d18..537e5a4f9843 100644 --- a/docs/cli/onboard.md +++ b/docs/cli/onboard.md @@ -291,7 +291,7 @@ openclaw onboard --non-interactive \ - Unless you pass `--skip-health`, onboarding waits for a reachable local gateway before exiting successfully. - `--install-daemon` starts the managed gateway install path first. Without it, a local gateway must already be running (for example `openclaw gateway run`). - `--skip-health` skips the wait if you only want config/workspace/bootstrap writes in automation. -- `--skip-bootstrap` sets `agents.defaults.skipBootstrap: true` and skips creating `AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, `USER.md`, and `BOOTSTRAP.md`. +- `--skip-bootstrap` sets `agents.defaults.skipBootstrap: true` and skips creating `AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`, and `BOOTSTRAP.md`. - On native Windows, `--install-daemon` tries Scheduled Tasks first and falls back to a per-user Startup-folder login item if task creation is denied. ### Interactive ref mode diff --git a/docs/cli/policy.md b/docs/cli/policy.md index b567808a234d..20774579258d 100644 --- a/docs/cli/policy.md +++ b/docs/cli/policy.md @@ -20,7 +20,7 @@ Policy checks configured channels, MCP servers, model providers, network SSRF posture, ingress/channel access, Gateway exposure and node command posture, authored message-routing probes, agent workspace access, sandbox posture, data-handling posture, secret -provider/auth profile posture, and governed tool metadata (`TOOLS.md`). Use it +provider/auth profile posture, and governed tool metadata (the `## Tools` section of `AGENTS.md`). Use it when a workspace needs a durable, checkable statement such as "Telegram must not be enabled" or "governed tools must declare risk and owner metadata." If you only need local behavior with no attestation or drift detection, plain @@ -500,9 +500,9 @@ only reviewed exec approval posture for selected agents. #### Tool metadata -| Policy field | Observed state | Use when | -| ----------------------- | -------------------------------- | ------------------------------------------------------------------------------------------ | -| `tools.requireMetadata` | Governed `TOOLS.md` declarations | Require governed tools to declare metadata keys such as `risk`, `sensitivity`, or `owner`. | +| Policy field | Observed state | Use when | +| ----------------------- | -------------------------------------- | ------------------------------------------------------------------------------------------ | +| `tools.requireMetadata` | Governed `AGENTS.md` tool declarations | Require governed tools to declare metadata keys such as `risk`, `sensitivity`, or `owner`. | #### Tool posture @@ -741,7 +741,7 @@ Example JSON output: "tools": [ { "id": "deploy", - "source": "oc://TOOLS.md/tools/deploy", + "source": "oc://AGENTS.md/tools/deploy", "line": 12, "risk": "critical", "sensitivity": "restricted", @@ -897,12 +897,12 @@ Example findings: { "checkId": "policy/tools-missing-risk-level", "severity": "error", - "message": "TOOLS.md tool 'deploy' has no explicit risk classification.", + "message": "AGENTS.md tool 'deploy' has no explicit risk classification.", "source": "policy", - "path": "TOOLS.md", + "path": "AGENTS.md", "line": 12, - "ocPath": "oc://TOOLS.md/tools/deploy", - "target": "oc://TOOLS.md/tools/deploy", + "ocPath": "oc://AGENTS.md/tools/deploy", + "target": "oc://AGENTS.md/tools/deploy", "requirement": "oc://policy.jsonc/tools/requireMetadata" } ``` diff --git a/docs/concepts/agent-workspace.md b/docs/concepts/agent-workspace.md index 258ca5094869..5243b5c2527e 100644 --- a/docs/concepts/agent-workspace.md +++ b/docs/concepts/agent-workspace.md @@ -76,8 +76,8 @@ Standard files OpenClaw expects inside the workspace: The agent's name, vibe, and emoji. Created/updated during the bootstrap ritual. - - Notes about your local tools and conventions. Does not control tool availability; it is only guidance. + + The `## Tools` section holds local environment notes and conventions. It does not control tool availability; it is only guidance. Optional startup checklist run automatically on gateway restart (when [internal hooks](/automation/hooks) are enabled). Keep it short; use the message tool for outbound sends. @@ -137,7 +137,7 @@ Run these steps on the machine where the Gateway runs (that is where the workspa ```bash cd ~/.openclaw/workspace git init - git add AGENTS.md SOUL.md TOOLS.md IDENTITY.md USER.md memory/ + git add AGENTS.md SOUL.md IDENTITY.md USER.md memory/ git commit -m "Add agent workspace" ``` diff --git a/docs/concepts/agent.md b/docs/concepts/agent.md index d0415c465274..6644fee73500 100644 --- a/docs/concepts/agent.md +++ b/docs/concepts/agent.md @@ -34,7 +34,6 @@ Inside the workspace, OpenClaw expects these user-editable files: | -------------- | ---------------------------------------------------- | | `AGENTS.md` | Operating instructions + "memory" | | `SOUL.md` | Persona, boundaries, tone | -| `TOOLS.md` | User-maintained tool notes and conventions | | `IDENTITY.md` | Agent name/vibe/emoji | | `USER.md` | User profile + preferred address | | `BOOTSTRAP.md` | One-time first-run ritual (deleted after completion) | @@ -67,8 +66,7 @@ To disable bootstrap file creation entirely (for pre-seeded workspaces), set: Core tools (read/exec/edit/write and related system tools) are always available, subject to tool policy. `apply_patch` is on by default for OpenAI models and gated by -`tools.exec.applyPatch` (`enabled`, `workspaceOnly`, `allowModels`). `TOOLS.md` does **not** control which tools exist; it's -guidance for how _you_ want them used. +`tools.exec.applyPatch` (`enabled`, `workspaceOnly`, `allowModels`). The `## Tools` section of `AGENTS.md` does **not** control which tools exist; it is guidance for how _you_ want them used. ## Skills diff --git a/docs/concepts/context.md b/docs/concepts/context.md index 9ff58f407a08..068422a7237f 100644 --- a/docs/concepts/context.md +++ b/docs/concepts/context.md @@ -44,7 +44,6 @@ System prompt (run): 38,412 chars (~9,603 tok) (Project Context 23,901 chars (~5 Injected workspace files: - AGENTS.md: OK | raw 1,742 chars (~436 tok) | injected 1,742 chars (~436 tok) - SOUL.md: OK | raw 912 chars (~228 tok) | injected 912 chars (~228 tok) -- TOOLS.md: TRUNCATED | raw 54,210 chars (~13,553 tok) | injected 20,962 chars (~5,241 tok) - IDENTITY.md: OK | raw 211 chars (~53 tok) | injected 211 chars (~53 tok) - USER.md: OK | raw 388 chars (~97 tok) | injected 388 chars (~97 tok) - BOOTSTRAP.md: OK | raw 0 chars (~0 tok) | injected 0 chars (~0 tok) @@ -118,7 +117,6 @@ By default, OpenClaw injects a fixed set of workspace files (if present): - `AGENTS.md` - `SOUL.md` -- `TOOLS.md` - `IDENTITY.md` - `USER.md` - `BOOTSTRAP.md` (first-run only) diff --git a/docs/concepts/system-prompt.md b/docs/concepts/system-prompt.md index 8c8fb1cb4216..5fbb806fd228 100644 --- a/docs/concepts/system-prompt.md +++ b/docs/concepts/system-prompt.md @@ -94,13 +94,12 @@ Bootstrap files are resolved from the active workspace and routed to the prompt - `AGENTS.md` - `SOUL.md` -- `TOOLS.md` - `IDENTITY.md` - `USER.md` - `BOOTSTRAP.md` (only on brand-new workspaces) - `MEMORY.md` when present -On the native Codex harness, OpenClaw avoids repeating stable workspace files in every user turn. Codex loads `AGENTS.md` through its own project-doc discovery. `TOOLS.md` is forwarded as inherited Codex developer instructions. `SOUL.md`, `IDENTITY.md`, and `USER.md` are forwarded as turn-scoped collaboration developer instructions so native Codex sub-agents do not inherit them. `MEMORY.md` content is not pasted into every native Codex turn either: when memory tools are available for the workspace, Codex turns get a small workspace-memory note directing the model to `memory_search` or `memory_get`. If tools are disabled, memory search is unavailable, or the active workspace differs from the agent memory workspace, `MEMORY.md` falls back to the normal bounded turn-context path. `BOOTSTRAP.md` keeps the normal turn-context role. +On the native Codex harness, OpenClaw avoids repeating stable workspace files in every user turn. Codex loads `AGENTS.md`, including its `## Tools` section, through native project-doc discovery. `SOUL.md`, `IDENTITY.md`, and `USER.md` are forwarded as turn-scoped collaboration developer instructions so native Codex sub-agents do not inherit them. `MEMORY.md` content is not pasted into every native Codex turn either: when memory tools are available for the workspace, Codex turns get a small workspace-memory note directing the model to `memory_search` or `memory_get`. If tools are disabled, memory search is unavailable, or the active workspace differs from the agent memory workspace, `MEMORY.md` falls back to the normal bounded turn-context path. `BOOTSTRAP.md` keeps the normal turn-context role. Heartbeat monitor scratch is not a bootstrap file. The heartbeat runner appends it only to heartbeat turns; normal turns do not receive it. The default agent's system prompt automatically includes heartbeat guidance while its cadence is enabled, with no independent heartbeat setting to hide that section. @@ -122,7 +121,7 @@ Missing files inject a short missing-file marker. Detailed raw/injected counts s For memory files, truncation is not data loss: the file stays intact on disk. On native Codex, `MEMORY.md` is read on demand through memory tools when available, with bounded prompt fallback otherwise. On other harnesses, the model only sees the shortened injected copy until it reads or searches memory directly. If `MEMORY.md` is repeatedly truncated, distill it into a shorter durable summary, move detailed history into `memory/*.md`, or intentionally raise the bootstrap limits. -Sub-agent sessions only inject `AGENTS.md` and `TOOLS.md` (other bootstrap files are filtered out to keep sub-agent context small). +Sub-agent sessions only inject `AGENTS.md` (other bootstrap files are filtered out to keep sub-agent context small). Internal hooks can intercept this step via the `agent:bootstrap` event to mutate or replace the injected bootstrap files (for example swapping `SOUL.md` for an alternate persona). diff --git a/docs/docs.json b/docs/docs.json index f2c89278e2f9..90ee7a01fbac 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1946,7 +1946,6 @@ "reference/templates/HEARTBEAT", "reference/templates/IDENTITY", "reference/templates/SOUL", - "reference/templates/TOOLS", "reference/templates/USER" ] }, diff --git a/docs/docs_map.md b/docs/docs_map.md index ac37a2dbf9eb..30b3834d37a9 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -8787,7 +8787,8 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Soul (required) - H2: Shared spaces (recommended) - H2: Memory system (recommended) - - H2: Tools and skills + - H2: Tools + - H3: Local notes - H2: Backup tip (recommended) - H2: What OpenClaw does - H2: Core skills (enable in Settings → Skills) @@ -9063,6 +9064,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Existing solutions preflight - H2: Daily memory (recommended) - H2: Heartbeats (optional) + - H2: Tools - H2: Customize - H2: C-3PO Origin Memory - H3: Birth Day: 2026-01-09 @@ -9137,24 +9139,11 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Continuity - H2: Related -## reference/templates/TOOLS.dev.md - -- Route: /reference/templates/TOOLS.dev -- Headings: - - H1: TOOLS.md - User Tool Notes (editable) - - H2: Examples - - H3: imsg - - H3: sag - - H2: Related - ## reference/templates/TOOLS.md - Route: /reference/templates/TOOLS - Headings: - - H1: TOOLS.md - Local Notes - - H2: Examples - - H2: Why Separate? - - H2: Related + - H1: TOOLS.md is retired ## reference/templates/USER.dev.md diff --git a/docs/gateway/config-agents.md b/docs/gateway/config-agents.md index 2d4fb398d704..a17da0ce0103 100644 --- a/docs/gateway/config-agents.md +++ b/docs/gateway/config-agents.md @@ -63,7 +63,7 @@ Optional default skill allowlist for agents that do not set ### `agents.defaults.skipBootstrap` -Disables automatic creation of workspace bootstrap files (`AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md`). +Disables automatic creation of workspace bootstrap files (`AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md`). ```json5 { @@ -73,7 +73,7 @@ Disables automatic creation of workspace bootstrap files (`AGENTS.md`, `SOUL.md` ### `agents.defaults.skipOptionalBootstrapFiles` -Skips creation of selected optional workspace files while still writing required bootstrap files (`AGENTS.md`, `TOOLS.md`, `BOOTSTRAP.md`). Valid values: `SOUL.md`, `USER.md`, and `IDENTITY.md` (`HEARTBEAT.md` is accepted but a no-op since heartbeat context moved to cron monitor scratch). +Skips creation of selected optional workspace files while still writing required bootstrap files (`AGENTS.md`, `BOOTSTRAP.md`). Valid values: `SOUL.md`, `USER.md`, and `IDENTITY.md` (`HEARTBEAT.md` is accepted but a no-op since heartbeat context moved to cron monitor scratch). ```json5 { diff --git a/docs/gateway/doctor.md b/docs/gateway/doctor.md index fb1bdf074049..38793cd7db30 100644 --- a/docs/gateway/doctor.md +++ b/docs/gateway/doctor.md @@ -163,6 +163,7 @@ Flags: - Legacy on-disk state migration (sessions/agent dir/WhatsApp auth). - Legacy plugin manifest contract key migration (`speechProviders`, `realtimeTranscriptionProviders`, `realtimeVoiceProviders`, `mediaUnderstandingProviders`, `imageGenerationProviders`, `videoGenerationProviders`, `webFetchProviders`, `webSearchProviders` → `contracts`). - Legacy cron store migration (`jobId`, `schedule.cron`, top-level delivery/payload fields, payload `provider`, `notify: true` webhook fallback jobs). + - Legacy workspace `TOOLS.md` migration into the `## Tools` section of `AGENTS.md`, with the original archived under the state directory before removal. - Codex CLI runtime pin repair (`agentRuntime.id: "codex-cli"` → `"codex"`) across `agents.defaults`, `agents.entries.*`, and `models.providers.*` (including per-model entries). - Stale plugin config cleanup when plugins are enabled; when `plugins.enabled=false`, stale plugin references are preserved as inert containment config. diff --git a/docs/help/debugging.md b/docs/help/debugging.md index d3a25c54c3d0..be34405cd2ec 100644 --- a/docs/help/debugging.md +++ b/docs/help/debugging.md @@ -181,7 +181,7 @@ What this does: 2. **Dev bootstrap** (`gateway --dev`) - Writes a minimal config if missing (`gateway.mode=local`, bind loopback). - Sets `agents.defaults.workspace` to the dev workspace and `agents.defaults.skipBootstrap=true`. - - Seeds the workspace files if missing: `AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, `USER.md`. + - Seeds the workspace files if missing: `AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`. - Default identity: **C3-PO** (protocol droid). - `pnpm gateway:dev` also sets `OPENCLAW_SKIP_CHANNELS=1` to skip channel providers. diff --git a/docs/plugins/codex-harness-reference.md b/docs/plugins/codex-harness-reference.md index a6642e66ab1b..b427ee804117 100644 --- a/docs/plugins/codex-harness-reference.md +++ b/docs/plugins/codex-harness-reference.md @@ -673,11 +673,8 @@ OpenClaw does not write synthetic Codex project-doc files or depend on Codex fallback filenames for persona files, because Codex fallbacks only apply when `AGENTS.md` is missing. -For OpenClaw workspace parity, the Codex harness forwards the other -bootstrap files as developer instructions, but not identically: +For OpenClaw workspace parity, local tool notes live in the `## Tools` section of `AGENTS.md` and ride Codex's native project-doc discovery. The Codex harness forwards the other bootstrap files as developer instructions: -- `TOOLS.md` is forwarded as **inherited** Codex developer instructions, so - native Codex subagents spawned during the turn also see it. - `SOUL.md`, `IDENTITY.md`, and `USER.md` are forwarded as **turn-scoped** collaboration instructions. Native Codex subagents do not inherit them, which keeps subagent turns from picking up the parent agent's persona and diff --git a/docs/reference/AGENTS.default.md b/docs/reference/AGENTS.default.md index 054721efbb0f..cfdc0c8367d5 100644 --- a/docs/reference/AGENTS.default.md +++ b/docs/reference/AGENTS.default.md @@ -21,7 +21,6 @@ mkdir -p ~/.openclaw/workspace ```bash cp docs/reference/templates/AGENTS.md ~/.openclaw/workspace/AGENTS.md cp docs/reference/templates/SOUL.md ~/.openclaw/workspace/SOUL.md -cp docs/reference/templates/TOOLS.md ~/.openclaw/workspace/TOOLS.md ``` 3. Optional: use this file's personal-assistant skill roster instead of the generic template: @@ -75,10 +74,12 @@ Before proposing or building a custom system, feature, workflow, tool, integrati - Capture: decisions, preferences, constraints, open loops. - Avoid secrets unless explicitly requested. -## Tools and skills +## Tools + +### Local notes - Tools live in skills; follow each skill's `SKILL.md` when you need it. -- Keep environment-specific notes in `TOOLS.md` (notes for skills). +- Keep environment-specific notes in this file's `## Tools` section. ## Backup tip (recommended) diff --git a/docs/reference/templates/AGENTS.dev.md b/docs/reference/templates/AGENTS.dev.md index a56192116921..5f53e9a9dc04 100644 --- a/docs/reference/templates/AGENTS.dev.md +++ b/docs/reference/templates/AGENTS.dev.md @@ -53,6 +53,17 @@ Before proposing or building a custom system, feature, workflow, tool, integrati - The heartbeat monitor's cron scratch can hold a tiny checklist for heartbeat runs; keep it small. +## Tools + +Skills define how tools work. Keep environment-specific details here so shared skills can update independently without exposing your local setup. + +Example placeholders (replace or remove them): + +```markdown +- SSH: dev-server -> 192.168.1.100, user admin +- TTS: preferred voice "Nova"; default speaker Office +``` + ## Customize - Add your preferred style, rules, and "memory" here. diff --git a/docs/reference/templates/AGENTS.md b/docs/reference/templates/AGENTS.md index 115220cbc0b6..7017bab9e2ff 100644 --- a/docs/reference/templates/AGENTS.md +++ b/docs/reference/templates/AGENTS.md @@ -44,7 +44,7 @@ Capture what matters: decisions, context, things to remember. Skip secrets unles Memory is limited. "Mental notes" don't survive session restarts; files do. Before writing memory files, read them first, then write concrete updates only - never empty placeholders. - Someone says "remember this" -> update `memory/YYYY-MM-DD.md` or the relevant file. -- You learn a lesson -> update `AGENTS.md`, `TOOLS.md`, or the relevant skill. +- You learn a lesson -> update `AGENTS.md` or the relevant skill. - You make a mistake -> document it so future-you doesn't repeat it. ## Red Lines @@ -85,15 +85,25 @@ On platforms that support reactions (Discord, Slack), use emoji reactions natura ## Tools -Skills provide your tools. When you need one, check its `SKILL.md`. Keep local notes (camera names, SSH details, voice preferences) in `TOOLS.md`. +Skills define how tools work. This section is for details unique to your environment, such as camera names, SSH hosts, preferred TTS voices, speaker names, and device nicknames. Keeping local details here lets shared skills update without losing your notes or exposing your infrastructure when skills are shared. + +### Local notes + +Example placeholders (replace or remove them): + +```markdown +- Cameras: living-room -> main area; front-door -> entrance +- SSH: home-server -> 192.168.1.100, user admin +- TTS: preferred voice "Nova"; default speaker Kitchen HomePod +``` **Voice storytelling:** if you have `sag` (ElevenLabs TTS), use voice for stories, movie summaries, and storytime moments - more engaging than walls of text. **Platform formatting:** -- Discord/WhatsApp: no markdown tables - use bullet lists instead. -- Discord links: wrap multiple links in `<>` to suppress embeds (``). -- WhatsApp: no headers - use **bold** or CAPS for emphasis. +- On Discord and WhatsApp, use bullet lists instead of markdown tables. +- On Discord, wrap multiple links in `<>` to suppress embeds (``). +- On WhatsApp, use **bold** or CAPS instead of headers. ## Heartbeats - Be Proactive diff --git a/docs/reference/templates/BOOTSTRAP.md b/docs/reference/templates/BOOTSTRAP.md index e31cb45b0135..96c6e4461a0c 100644 --- a/docs/reference/templates/BOOTSTRAP.md +++ b/docs/reference/templates/BOOTSTRAP.md @@ -9,7 +9,7 @@ read_when: _You just woke up. Keep this first conversation short and make it yours._ -OpenClaw only seeds this file into a brand-new workspace, alongside `AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, and `USER.md`. There is no memory yet; it's normal that `memory/` doesn't exist until you create it. +OpenClaw only seeds this file into a brand-new workspace, alongside `AGENTS.md`, `SOUL.md`, `IDENTITY.md`, and `USER.md`. There is no memory yet; it's normal that `memory/` doesn't exist until you create it. **The user's request always comes first.** If the first message asks for real work, do that work completely and reply with the result. Do not open with diff --git a/docs/reference/templates/TOOLS.dev.md b/docs/reference/templates/TOOLS.dev.md deleted file mode 100644 index 28fb5afcbc86..000000000000 --- a/docs/reference/templates/TOOLS.dev.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -summary: "Dev agent tools notes (C-3PO)" -title: "TOOLS.dev template" -read_when: - - Using the dev gateway templates - - Updating the default dev agent identity ---- - -# TOOLS.md - User Tool Notes (editable) - -This file is for _your_ notes about external tools and conventions. It does not define which tools exist; OpenClaw provides built-in tools internally, and skills add the rest. - -## Examples - -### imsg - -- Send an iMessage/SMS: describe who/what, confirm before sending. -- Prefer short messages; avoid sending secrets. - -### sag - -- Text-to-speech: specify voice, target speaker/room, and whether to stream. - -Add whatever else you want the assistant to know about your local toolchain. - -## Related - -- [TOOLS.md template](/reference/templates/TOOLS) diff --git a/docs/reference/templates/TOOLS.md b/docs/reference/templates/TOOLS.md index bcdd2f5947d3..0a186717997f 100644 --- a/docs/reference/templates/TOOLS.md +++ b/docs/reference/templates/TOOLS.md @@ -1,40 +1,14 @@ --- -summary: "Workspace template for TOOLS.md" -title: "TOOLS.md template" +summary: "Retired TOOLS.md workspace template" +title: "TOOLS.md retired" read_when: - Bootstrapping a workspace manually --- -# TOOLS.md - Local Notes +# TOOLS.md is retired -Skills define _how_ tools work. This file is for _your_ specifics — the stuff that's unique to your setup: camera names and locations, SSH hosts and aliases, preferred TTS voices, speaker/room names, device nicknames, anything environment-specific. +Local tool and environment notes now live in the `## Tools` section of `AGENTS.md`. Skills continue to define how tools work. -## Examples +Run `openclaw doctor --fix` to archive an existing workspace `TOOLS.md`, merge customized content into `AGENTS.md`, and remove the retired file. -```markdown -### Cameras - -- living-room → Main area, 180° wide angle -- front-door → Entrance, motion-triggered - -### SSH - -- home-server → 192.168.1.100, user: admin - -### TTS - -- Preferred voice: "Nova" (warm, slightly British) -- Default speaker: Kitchen HomePod -``` - -## Why Separate? - -Skills are shared. Your setup is yours. Keeping them apart means you can update skills without losing your notes, and share skills without leaking your infrastructure. - ---- - -Add whatever helps you do your job. This is your cheat sheet. - -## Related - -- [Agent workspace](/concepts/agent-workspace) +See the [AGENTS.md template](/reference/templates/AGENTS) and [agent workspace guide](/concepts/agent-workspace). diff --git a/docs/reference/token-use.md b/docs/reference/token-use.md index 8838fa64ddb9..fa5b9db4c310 100644 --- a/docs/reference/token-use.md +++ b/docs/reference/token-use.md @@ -20,7 +20,7 @@ OpenClaw assembles its own system prompt on every run. It includes: Bounded by `skills.limits.maxSkillsPromptChars`, with optional per-agent override at `agents.entries.*.skillsLimits.maxSkillsPromptChars`. - Self-update instructions -- Workspace + bootstrap files (`AGENTS.md`, `SOUL.md`, `TOOLS.md`, +- Workspace + bootstrap files (`AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md` when new, plus `MEMORY.md` when present). Large injected files are truncated by `agents.defaults.bootstrapMaxChars` (default: `20000`); total bootstrap diff --git a/docs/start/bootstrapping.md b/docs/start/bootstrapping.md index d7c081d316f2..1bc3e4f5c6c9 100644 --- a/docs/start/bootstrapping.md +++ b/docs/start/bootstrapping.md @@ -17,7 +17,7 @@ onboarding, on the agent's first real turn. On the first run against a brand-new workspace (default `~/.openclaw/workspace`), OpenClaw: -- Seeds `AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, `USER.md`, and `BOOTSTRAP.md`. +- Seeds `AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`, and `BOOTSTRAP.md`. Environment-specific tool notes belong in the `## Tools` section of `AGENTS.md`. - Has the agent follow a capped three-beat birth sequence: it asks what you want to call it, shares one short soul/vibe line, and asks whether you want the minimal recommended plugin set or maximum convenience. diff --git a/docs/start/hubs.md b/docs/start/hubs.md index 6db79bc55aa3..4dba76be72b4 100644 --- a/docs/start/hubs.md +++ b/docs/start/hubs.md @@ -183,7 +183,7 @@ Use these hubs to discover every page, including deep dives and reference docs t - [Retired HEARTBEAT.md workspace file](/reference/templates/HEARTBEAT) - [Templates: IDENTITY](/reference/templates/IDENTITY) - [Templates: SOUL](/reference/templates/SOUL) -- [Templates: TOOLS](/reference/templates/TOOLS) +- [Templates: AGENTS](/reference/templates/AGENTS) - [Templates: USER](/reference/templates/USER) ## Project diff --git a/docs/start/openclaw.md b/docs/start/openclaw.md index a6234f2ac38d..4e8b0d67ea85 100644 --- a/docs/start/openclaw.md +++ b/docs/start/openclaw.md @@ -64,7 +64,7 @@ When onboarding finishes, OpenClaw auto-opens the dashboard and prints a clean ( OpenClaw reads operating instructions and "memory" from its workspace directory. -By default, OpenClaw uses `~/.openclaw/workspace` as the agent workspace, and creates it (plus starter `AGENTS.md`, `SOUL.md`, `TOOLS.md`, `IDENTITY.md`, `USER.md`) automatically on onboarding or first agent run. `BOOTSTRAP.md` is only created for a brand-new workspace and should not come back after you delete it. `MEMORY.md` is optional and never auto-created; when present, it loads for normal sessions. Subagent sessions only inject `AGENTS.md` and `TOOLS.md`. +By default, OpenClaw uses `~/.openclaw/workspace` as the agent workspace, and creates it (plus starter `AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`) automatically on onboarding or first agent run. Put environment-specific tool notes in the `## Tools` section of `AGENTS.md`. `BOOTSTRAP.md` is only created for a brand-new workspace and should not come back after you delete it. `MEMORY.md` is optional and never auto-created; when present, it loads for normal sessions. Subagent sessions only inject `AGENTS.md`. Treat this folder like OpenClaw's memory and make it a git repo (ideally private) so your `AGENTS.md` and memory files are backed up. If git is installed, brand-new workspaces are auto-initialized with `git init`. diff --git a/docs/tools/subagents.md b/docs/tools/subagents.md index bf091f0e059b..a62eb8f9bafb 100644 --- a/docs/tools/subagents.md +++ b/docs/tools/subagents.md @@ -674,7 +674,7 @@ still need normal device approval for scope upgrades. - Sub-agent announce is **best-effort**. If the gateway restarts, pending "announce back" work is lost. - Sub-agents still share the same gateway process resources; treat `maxConcurrent` as a safety valve. - `sessions_spawn` is always non-blocking: it returns `{ status: "accepted", runId, childSessionKey }` immediately. -- Sub-agent context only injects `AGENTS.md` and `TOOLS.md` (no `SOUL.md`, `IDENTITY.md`, `USER.md`, `MEMORY.md`, or `BOOTSTRAP.md`). Codex-native subagents follow the same boundary: `TOOLS.md` stays in inherited Codex thread instructions, while parent-only persona, identity, and user files are injected as turn-scoped collaboration instructions so children do not clone them. +- Sub-agent context only injects `AGENTS.md` (no `SOUL.md`, `IDENTITY.md`, `USER.md`, `MEMORY.md`, or `BOOTSTRAP.md`). Its `## Tools` section carries environment-specific notes. Codex-native subagents follow the same boundary through native `AGENTS.md` discovery, while parent-only persona, identity, and user files are injected as turn-scoped collaboration instructions so children do not clone them. - Maximum nesting depth is 5 (`maxSpawnDepth` range: 1-5). Depth 2 is recommended for most use cases. - `maxChildrenPerAgent` caps active children per session (default `5`, range `1-20`). diff --git a/extensions/codex/src/app-server/attempt-context.test.ts b/extensions/codex/src/app-server/attempt-context.test.ts index ee2cf8bd5308..c11f6178aafc 100644 --- a/extensions/codex/src/app-server/attempt-context.test.ts +++ b/extensions/codex/src/app-server/attempt-context.test.ts @@ -92,7 +92,6 @@ describe("Codex app-server attempt context", () => { bootstrapFiles: [], contextFiles: [], promptContextFiles: [], - developerInstructionFiles: [], }, skillsPrompt: "", tools, diff --git a/extensions/codex/src/app-server/attempt-context.ts b/extensions/codex/src/app-server/attempt-context.ts index bdc61bd1d55d..27e744cacca8 100644 --- a/extensions/codex/src/app-server/attempt-context.ts +++ b/extensions/codex/src/app-server/attempt-context.ts @@ -33,23 +33,20 @@ import { } from "./thread-lifecycle.js"; const CODEX_NATIVE_PROJECT_DOC_BASENAMES = new Set(["agents.md"]); -const CODEX_INHERITED_WORKSPACE_DEVELOPER_CONTEXT_BASENAMES = new Set(["tools.md"]); const CODEX_TURN_SCOPED_WORKSPACE_DEVELOPER_CONTEXT_BASENAMES = new Set([ "identity.md", "soul.md", "user.md", ]); -const CODEX_WORKSPACE_DEVELOPER_CONTEXT_BASENAMES = new Set([ - ...CODEX_INHERITED_WORKSPACE_DEVELOPER_CONTEXT_BASENAMES, - ...CODEX_TURN_SCOPED_WORKSPACE_DEVELOPER_CONTEXT_BASENAMES, -]); +const CODEX_WORKSPACE_DEVELOPER_CONTEXT_BASENAMES = new Set( + CODEX_TURN_SCOPED_WORKSPACE_DEVELOPER_CONTEXT_BASENAMES, +); const CODEX_MEMORY_CONTEXT_BASENAME = "memory.md"; const CODEX_MEMORY_TOOL_NAMES = new Set(["memory_search", "memory_get"]); const CODEX_BOOTSTRAP_CONTEXT_ORDER = new Map([ ["soul.md", 10], ["identity.md", 20], ["user.md", 30], - ["tools.md", 40], ["bootstrap.md", 50], ["memory.md", 60], ]); @@ -64,14 +61,12 @@ export type CodexSystemPromptReport = NonNullable section?.trim()) - .join("\n\n"); + const threadDeveloperInstructions = testing.buildDeveloperInstructions(params, { dynamicTools }); const openClawPromptContext = buildCodexOpenClawPromptContext({ params, workspacePromptContext: workspaceBootstrapContext.promptContext, @@ -2881,19 +2876,17 @@ describe("runCodexAppServerAttempt", () => { expect(secondInputText).toContain("continue from there"); }); - it("passes stable workspace files as Codex developer instructions and routes MEMORY.md through tools", async () => { + it("routes AGENTS.md natively and MEMORY.md through tools", async () => { const { sessionFile, workspaceDir } = createRunPaths(); const agentsGuidance = "Follow AGENTS guidance."; const soulGuidance = "Soul voice goes here."; const identityGuidance = "Identity guidance goes here."; - const toolGuidance = "Tool guidance goes here."; const userProfile = "User profile goes here."; const memorySummary = "Memory summary goes here."; await fs.mkdir(workspaceDir, { recursive: true }); await fs.writeFile(path.join(workspaceDir, "AGENTS.md"), agentsGuidance); await fs.writeFile(path.join(workspaceDir, "SOUL.md"), soulGuidance); await fs.writeFile(path.join(workspaceDir, "IDENTITY.md"), identityGuidance); - await fs.writeFile(path.join(workspaceDir, "TOOLS.md"), toolGuidance); await fs.writeFile(path.join(workspaceDir, "USER.md"), userProfile); await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), memorySummary); registerMemoryPromptForTest(); @@ -2912,10 +2905,8 @@ describe("runCodexAppServerAttempt", () => { systemPromptReport, threadDeveloperInstructions, } = await buildCodexTurnContextForTest(params, workspaceDir); - expect(threadDeveloperInstructions).toContain("OpenClaw Workspace Instructions"); expect(threadDeveloperInstructions).not.toContain(soulGuidance); expect(threadDeveloperInstructions).not.toContain(identityGuidance); - expect(threadDeveloperInstructions).toContain(toolGuidance); expect(threadDeveloperInstructions).not.toContain(userProfile); expect(threadDeveloperInstructions).not.toContain(memorySummary); expect(threadDeveloperInstructions).not.toContain("Codex loads AGENTS.md natively"); @@ -2927,7 +2918,6 @@ describe("runCodexAppServerAttempt", () => { expect(collaborationInstructions).toContain(""); expect(collaborationInstructions).toContain(soulGuidance); expect(collaborationInstructions).toContain(identityGuidance); - expect(collaborationInstructions).not.toContain(toolGuidance); expect(collaborationInstructions).toContain(userProfile); expect(collaborationInstructions).toContain("## Memory Recall"); expect(collaborationInstructions).toContain("MEMORY.md + memory/*.md"); @@ -2949,7 +2939,6 @@ describe("runCodexAppServerAttempt", () => { expect(inputText).not.toContain("not developer policy"); expect(inputText).not.toContain(soulGuidance); expect(inputText).not.toContain(identityGuidance); - expect(inputText).not.toContain(toolGuidance); expect(inputText).not.toContain(userProfile); expect(inputText).not.toContain(memorySummary); expect(inputText).not.toContain("OpenClaw Workspace Memory"); @@ -2975,11 +2964,6 @@ describe("runCodexAppServerAttempt", () => { injectedChars: identityGuidance.length, truncated: false, }); - expect(fileStats.get("TOOLS.md")).toMatchObject({ - rawChars: toolGuidance.length, - injectedChars: toolGuidance.length, - truncated: false, - }); expect(fileStats.get("USER.md")).toMatchObject({ rawChars: userProfile.length, injectedChars: userProfile.length, @@ -3050,18 +3034,16 @@ describe("runCodexAppServerAttempt", () => { expect(inputText).toBe("hello"); expect(inputText).not.toContain(memorySummary); }); - it("sends workspace bootstrap instructions through Codex app-server payloads", async () => { + it("sends turn-scoped workspace instructions through Codex app-server payloads", async () => { const { sessionFile, workspaceDir } = createRunPaths(); const agentsGuidance = "Follow AGENTS guidance."; const soulGuidance = "Soul voice goes here."; const identityGuidance = "Identity guidance goes here."; - const toolGuidance = "Tool guidance goes here."; const userProfile = "User profile goes here."; await fs.mkdir(workspaceDir, { recursive: true }); await fs.writeFile(path.join(workspaceDir, "AGENTS.md"), agentsGuidance); await fs.writeFile(path.join(workspaceDir, "SOUL.md"), soulGuidance); await fs.writeFile(path.join(workspaceDir, "IDENTITY.md"), identityGuidance); - await fs.writeFile(path.join(workspaceDir, "TOOLS.md"), toolGuidance); await fs.writeFile(path.join(workspaceDir, "USER.md"), userProfile); const harness = createStartedThreadHarness(); const params = createParams(sessionFile, workspaceDir); @@ -3079,8 +3061,6 @@ describe("runCodexAppServerAttempt", () => { developerInstructions?: string; }; expect(threadStartParams.config?.instructions).toBeUndefined(); - expect(threadStartParams.developerInstructions).toContain("OpenClaw Workspace Instructions"); - expect(threadStartParams.developerInstructions).toContain(toolGuidance); expect(threadStartParams.developerInstructions).not.toContain(agentsGuidance); expect(threadStartParams.developerInstructions).not.toContain(soulGuidance); expect(threadStartParams.developerInstructions).not.toContain(identityGuidance); @@ -3102,7 +3082,6 @@ describe("runCodexAppServerAttempt", () => { expect(collaborationInstructions).toContain(soulGuidance); expect(collaborationInstructions).toContain(identityGuidance); expect(collaborationInstructions).toContain(userProfile); - expect(collaborationInstructions).not.toContain(toolGuidance); const inputText = turnStartParams.input?.[0]?.text ?? ""; expect(inputText).toBe("hello"); expect(inputText).not.toContain(agentsGuidance); diff --git a/extensions/copilot/src/workspace-bootstrap.ts b/extensions/copilot/src/workspace-bootstrap.ts index eb2923d7f22e..1c9f347ac5ef 100644 --- a/extensions/copilot/src/workspace-bootstrap.ts +++ b/extensions/copilot/src/workspace-bootstrap.ts @@ -42,7 +42,7 @@ type CopilotWorkspaceBootstrapResult = { /** * Loads OpenClaw workspace bootstrap files (IDENTITY.md, SOUL.md, - * HEARTBEAT.md, USER.md, TOOLS.md, BOOTSTRAP.md, MEMORY.md, ...) using + * HEARTBEAT.md, USER.md, BOOTSTRAP.md, MEMORY.md, ...) using * the shared core helper PI and codex both use, then renders them as a * single string suitable for `SessionConfig.systemMessage.content` on * the Copilot SDK. diff --git a/extensions/memory-core/src/flush-plan.ts b/extensions/memory-core/src/flush-plan.ts index 8de6cb754a81..f51571f8616a 100644 --- a/extensions/memory-core/src/flush-plan.ts +++ b/extensions/memory-core/src/flush-plan.ts @@ -17,7 +17,7 @@ const MEMORY_FLUSH_TARGET_HINT = const MEMORY_FLUSH_APPEND_ONLY_HINT = "If memory/YYYY-MM-DD.md already exists, APPEND new content only and do not overwrite existing entries."; const MEMORY_FLUSH_READ_ONLY_HINT = - "Treat workspace bootstrap/reference files such as MEMORY.md, DREAMS.md, SOUL.md, TOOLS.md, and AGENTS.md as read-only during this flush; never overwrite, replace, or edit them."; + "Treat workspace bootstrap/reference files such as MEMORY.md, DREAMS.md, SOUL.md, and AGENTS.md as read-only during this flush; never overwrite, replace, or edit them."; const MEMORY_FLUSH_REQUIRED_HINTS = [ MEMORY_FLUSH_TARGET_HINT, MEMORY_FLUSH_APPEND_ONLY_HINT, diff --git a/extensions/policy/src/doctor/check-ids.ts b/extensions/policy/src/doctor/check-ids.ts index 1a46d7c2d2c6..8de3e23888d7 100644 --- a/extensions/policy/src/doctor/check-ids.ts +++ b/extensions/policy/src/doctor/check-ids.ts @@ -5,6 +5,7 @@ export const CHECK_IDS = { policyHashMismatch: "policy/policy-hash-mismatch", policyInvalidFile: "policy/policy-jsonc-invalid", policyMissingFile: "policy/policy-jsonc-missing", + policyUnmigratedToolsFile: "policy/tools-md-migration-required", policyDeniedMcpServer: "policy/mcp-denied-server", policyUnapprovedMcpServer: "policy/mcp-unapproved-server", policyDeniedModelProvider: "policy/models-denied-provider", @@ -74,6 +75,7 @@ export const CHECK_IDS = { export const POLICY_CHECK_IDS = [ CHECK_IDS.policyMissingFile, CHECK_IDS.policyInvalidFile, + CHECK_IDS.policyUnmigratedToolsFile, CHECK_IDS.policyHashMismatch, CHECK_IDS.policyAttestationMismatch, CHECK_IDS.policyDeniedChannelProvider, diff --git a/extensions/policy/src/doctor/evaluation.ts b/extensions/policy/src/doctor/evaluation.ts index 081a33eaf0be..fce2616a9733 100644 --- a/extensions/policy/src/doctor/evaluation.ts +++ b/extensions/policy/src/doctor/evaluation.ts @@ -187,8 +187,26 @@ async function evaluatePolicyUncached(ctx: HealthCheckContext): Promise 0) { - const toolsFile = await readWorkspaceFile(ctx, "TOOLS.md"); + const [toolsFile, legacyToolsFile] = await Promise.all([ + readWorkspaceFile(ctx, "AGENTS.md"), + readWorkspaceFile(ctx, "TOOLS.md"), + ]); + if (legacyToolsFile !== null) { + unmigratedToolsFinding = { + checkId: CHECK_IDS.policyUnmigratedToolsFile, + severity: "error", + message: + "TOOLS.md contains unmigrated governed tool declarations; run `openclaw doctor --fix` to migrate them into the AGENTS.md `## Tools` section before policy evaluation can pass.", + source: "policy", + path: "TOOLS.md", + target: "oc://TOOLS.md/tools", + requirement: `oc://${policyFile.ocDocName}/tools/requireMetadata`, + fixHint: + "Run `openclaw doctor --fix` to migrate TOOLS.md into the AGENTS.md `## Tools` section.", + }; + } evidence = await collectPolicyEvidence(ctx.cfg as Record, { toolsRaw: toolsFile?.raw ?? "", includeIngress, @@ -242,6 +260,7 @@ async function evaluatePolicyUncached(ctx: HealthCheckContext): Promise finding.checkId === CHECK_IDS.policyInvalidFile); + return findings.some( + (finding) => + finding.checkId === CHECK_IDS.policyInvalidFile || + finding.checkId === CHECK_IDS.policyUnmigratedToolsFile, + ); } function channelFindings( diff --git a/extensions/policy/src/doctor/fix-metadata.ts b/extensions/policy/src/doctor/fix-metadata.ts index 98c05c91ee5d..da2e9653eccd 100644 --- a/extensions/policy/src/doctor/fix-metadata.ts +++ b/extensions/policy/src/doctor/fix-metadata.ts @@ -27,6 +27,12 @@ const m = ( const POLICY_FIX_METADATA = [ m(CHECK_IDS.policyMissingFile, "manual", "Restore or author the approved policy artifact."), m(CHECK_IDS.policyInvalidFile, "manual", "Repair the policy JSONC syntax or schema."), + m( + CHECK_IDS.policyUnmigratedToolsFile, + "manual", + "Run openclaw doctor --fix to migrate governed tool declarations into AGENTS.md.", + { policyPath: ["tools", "requireMetadata"] }, + ), m( CHECK_IDS.policyHashMismatch, "manual", diff --git a/extensions/policy/src/doctor/metadata.test.ts b/extensions/policy/src/doctor/metadata.test.ts index 6741d912a9d5..bafe237272a1 100644 --- a/extensions/policy/src/doctor/metadata.test.ts +++ b/extensions/policy/src/doctor/metadata.test.ts @@ -238,6 +238,7 @@ describe("policy doctor metadata", () => { "policy/policy-jsonc-missing", "policy/sandbox-browser-cdp-source-range-missing", "policy/secrets-unmanaged-provider", + "policy/tools-md-migration-required", "policy/tools-missing-owner", "policy/tools-missing-risk-level", "policy/tools-missing-sensitivity-token", diff --git a/extensions/policy/src/doctor/metadata.ts b/extensions/policy/src/doctor/metadata.ts index cf25d401666f..317a1e873cf5 100644 --- a/extensions/policy/src/doctor/metadata.ts +++ b/extensions/policy/src/doctor/metadata.ts @@ -314,6 +314,7 @@ export const POLICY_RULE_METADATA = [ strictness: "denylist-superset", valueType: "string-list", checkIds: [ + CHECK_IDS.policyUnmigratedToolsFile, CHECK_IDS.policyMissingToolRisk, CHECK_IDS.policyMissingToolSensitivity, CHECK_IDS.policyMissingToolOwner, diff --git a/extensions/policy/src/doctor/register.base.test-utils.ts b/extensions/policy/src/doctor/register.base.test-utils.ts index 57f2d9cb8bc9..8bad37dff1ea 100644 --- a/extensions/policy/src/doctor/register.base.test-utils.ts +++ b/extensions/policy/src/doctor/register.base.test-utils.ts @@ -316,6 +316,7 @@ describe("registerPolicyDoctorChecks", () => { "policy/exec-approvals-auto-allow-skills-enabled", "policy/exec-approvals-allowlist-missing", "policy/exec-approvals-allowlist-unexpected", + "policy/tools-md-migration-required", "policy/tools-missing-risk-level", "policy/tools-unknown-risk-level", "policy/tools-missing-sensitivity-token", @@ -1100,7 +1101,7 @@ describe("registerPolicyDoctorChecks", () => { expect(result.findings).toEqual([]); }); - it("does not include unrelated TOOLS.md evidence in channel-only attestations", async () => { + it("does not include unrelated AGENTS.md tool evidence in channel-only attestations", async () => { const configPath = join(workspaceDir, "openclaw.jsonc"); const policy = { channels: { denyRules: [] } }; const policyHash = policyDocumentHash(policy); @@ -1126,7 +1127,7 @@ describe("registerPolicyDoctorChecks", () => { }).attestationHash; await fs.writeFile(configPath, "{}", "utf-8"); await fs.writeFile(join(workspaceDir, "policy.jsonc"), JSON.stringify(policy), "utf-8"); - await fs.writeFile(join(workspaceDir, "TOOLS.md"), "## Tools\n\n### deploy\n", "utf-8"); + await fs.writeFile(join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy\n", "utf-8"); const result = await runPolicyChecks( ctx(configPath, cfgWithPolicy({ expectedAttestationHash: acceptedAttestationHash })), diff --git a/extensions/policy/src/doctor/register.gateway-data-and-approvals.test-utils.ts b/extensions/policy/src/doctor/register.gateway-data-and-approvals.test-utils.ts index 33c866bf35b6..86f159e1fcc3 100644 --- a/extensions/policy/src/doctor/register.gateway-data-and-approvals.test-utils.ts +++ b/extensions/policy/src/doctor/register.gateway-data-and-approvals.test-utils.ts @@ -1436,7 +1436,7 @@ describe("registerPolicyDoctorChecks", () => { "utf-8", ); await fs.writeFile( - join(workspaceDir, "TOOLS.md"), + join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy risk:critical sensitivity:secret\n", "utf-8", ); @@ -1449,8 +1449,8 @@ describe("registerPolicyDoctorChecks", () => { expect.objectContaining({ checkId: "policy/tools-unknown-sensitivity-token", severity: "error", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), ]); }); diff --git a/extensions/policy/src/doctor/register.models-and-mcp.test-utils.ts b/extensions/policy/src/doctor/register.models-and-mcp.test-utils.ts index 761007de6176..452217bb3515 100644 --- a/extensions/policy/src/doctor/register.models-and-mcp.test-utils.ts +++ b/extensions/policy/src/doctor/register.models-and-mcp.test-utils.ts @@ -171,7 +171,7 @@ describe("registerPolicyDoctorChecks", () => { JSON.stringify({ channels: {}, mcp: {}, models: {}, network: {}, tools: {} }), "utf-8", ); - await fs.writeFile(join(workspaceDir, "TOOLS.md"), "## Tools\n\n### deploy\n", "utf-8"); + await fs.writeFile(join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy\n", "utf-8"); const result = await runPolicyChecks(ctx(configPath, cfg)); @@ -186,7 +186,7 @@ describe("registerPolicyDoctorChecks", () => { JSON.stringify({ tools: { requireMetadata: ["risk", "unsupported"] } }), "utf-8", ); - await fs.writeFile(join(workspaceDir, "TOOLS.md"), "## Tools\n\n### deploy\n", "utf-8"); + await fs.writeFile(join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy\n", "utf-8"); const result = await runDoctorLintChecks(ctx(configPath, cfgWithPolicy()), { checks: registerChecks(), @@ -258,7 +258,7 @@ describe("registerPolicyDoctorChecks", () => { JSON.stringify({ tools: { requireMetadata: ["risk", "sensitivity", "owner"] } }), "utf-8", ); - await fs.writeFile(join(workspaceDir, "TOOLS.md"), "## Tools\n\n### deploy\n", "utf-8"); + await fs.writeFile(join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy\n", "utf-8"); const result = await runDoctorLintChecks(ctx(configPath, cfgWithPolicy()), { checks: registerChecks(), @@ -270,25 +270,119 @@ describe("registerPolicyDoctorChecks", () => { expect.objectContaining({ checkId: "policy/tools-missing-risk-level", severity: "error", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), expect.objectContaining({ checkId: "policy/tools-missing-sensitivity-token", severity: "error", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), expect.objectContaining({ checkId: "policy/tools-missing-owner", severity: "error", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), ]), ); }); + it("blocks governed tool evaluation until TOOLS.md is migrated into AGENTS.md", async () => { + const configPath = join(workspaceDir, "openclaw.jsonc"); + await fs.writeFile(configPath, "{}", "utf-8"); + await fs.writeFile( + join(workspaceDir, "policy.jsonc"), + JSON.stringify({ tools: { requireMetadata: ["risk"] } }), + "utf-8", + ); + await fs.writeFile(join(workspaceDir, "TOOLS.md"), "### deploy\n", "utf-8"); + + const beforeMigration = await runDoctorLintChecks(ctx(configPath, cfgWithPolicy()), { + checks: registerChecks(), + }); + + expect(beforeMigration.findings).toEqual([ + expect.objectContaining({ + checkId: "policy/tools-md-migration-required", + severity: "error", + message: + "TOOLS.md contains unmigrated governed tool declarations; run `openclaw doctor --fix` to migrate them into the AGENTS.md `## Tools` section before policy evaluation can pass.", + path: "TOOLS.md", + }), + ]); + + await fs.writeFile(join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy\n", "utf-8"); + await fs.rm(join(workspaceDir, "TOOLS.md")); + + const afterMigration = await runDoctorLintChecks(ctx(configPath, cfgWithPolicy()), { + checks: registerChecks(), + }); + + expect(afterMigration.findings).toEqual([ + expect.objectContaining({ + checkId: "policy/tools-missing-risk-level", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", + }), + ]); + }); + + it("runs required metadata checks for a tool literally named tools", async () => { + const configPath = join(workspaceDir, "openclaw.jsonc"); + await fs.writeFile(configPath, "{}", "utf-8"); + await fs.writeFile( + join(workspaceDir, "policy.jsonc"), + JSON.stringify({ tools: { requireMetadata: ["risk", "sensitivity", "owner"] } }), + "utf-8", + ); + await fs.writeFile( + join(workspaceDir, "AGENTS.md"), + "# Tools\n\n## Local notes\n\n- SSH: prod-host\n\n## tools risk: high\n", + "utf-8", + ); + + const result = await runDoctorLintChecks(ctx(configPath, cfgWithPolicy()), { + checks: registerChecks(), + }); + + expect(result.findings).toHaveLength(2); + expect(result.findings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + checkId: "policy/tools-missing-sensitivity-token", + ocPath: "oc://AGENTS.md/tools/tools", + }), + expect.objectContaining({ + checkId: "policy/tools-missing-owner", + ocPath: "oc://AGENTS.md/tools/tools", + }), + ]), + ); + }); + + it("does not capture an unrelated nested Tools section as governed evidence", async () => { + const configPath = join(workspaceDir, "openclaw.jsonc"); + await fs.writeFile(configPath, "{}", "utf-8"); + await fs.writeFile( + join(workspaceDir, "policy.jsonc"), + JSON.stringify({ tools: { requireMetadata: ["risk", "sensitivity", "owner"] } }), + "utf-8", + ); + await fs.writeFile( + join(workspaceDir, "AGENTS.md"), + "## Build\n\n### Tools\n\n- npm: install dependencies\n", + "utf-8", + ); + + const result = await runDoctorLintChecks(ctx(configPath, cfgWithPolicy()), { + checks: registerChecks(), + }); + + expect(result.findings).toEqual([]); + }); + it("reports governed bullet tools missing required metadata", async () => { const configPath = join(workspaceDir, "openclaw.jsonc"); await fs.writeFile(configPath, "{}", "utf-8"); @@ -297,7 +391,7 @@ describe("registerPolicyDoctorChecks", () => { JSON.stringify({ tools: { requireMetadata: ["risk", "sensitivity", "owner"] } }), "utf-8", ); - await fs.writeFile(join(workspaceDir, "TOOLS.md"), "## Tools\n\n- deploy: deploys\n", "utf-8"); + await fs.writeFile(join(workspaceDir, "AGENTS.md"), "## Tools\n\n- deploy: deploys\n", "utf-8"); const result = await runDoctorLintChecks(ctx(configPath, cfgWithPolicy()), { checks: registerChecks(), @@ -308,18 +402,18 @@ describe("registerPolicyDoctorChecks", () => { expect.arrayContaining([ expect.objectContaining({ checkId: "policy/tools-missing-risk-level", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), expect.objectContaining({ checkId: "policy/tools-missing-sensitivity-token", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), expect.objectContaining({ checkId: "policy/tools-missing-owner", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), ]), ); @@ -334,7 +428,7 @@ describe("registerPolicyDoctorChecks", () => { "utf-8", ); await fs.writeFile( - join(workspaceDir, "TOOLS.md"), + join(workspaceDir, "AGENTS.md"), [ "## Tools", "", @@ -359,7 +453,7 @@ describe("registerPolicyDoctorChecks", () => { const evidence = await collectPolicyEvidence( {}, { - toolsRaw: await fs.readFile(join(workspaceDir, "TOOLS.md"), "utf-8"), + toolsRaw: await fs.readFile(join(workspaceDir, "AGENTS.md"), "utf-8"), }, ); @@ -367,7 +461,7 @@ describe("registerPolicyDoctorChecks", () => { expect(evidence.tools).toEqual([ { id: "deploy", - source: "oc://TOOLS.md/tools/deploy", + source: "oc://AGENTS.md/tools/deploy", line: 3, risk: "critical", sensitivity: "restricted", @@ -376,7 +470,7 @@ describe("registerPolicyDoctorChecks", () => { }, { id: "inspect", - source: "oc://TOOLS.md/tools/inspect", + source: "oc://AGENTS.md/tools/inspect", line: 9, risk: "low", sensitivity: "public", @@ -394,7 +488,7 @@ describe("registerPolicyDoctorChecks", () => { "utf-8", ); await fs.writeFile( - join(workspaceDir, "TOOLS.md"), + join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy risk:critcal\n", "utf-8", ); @@ -407,8 +501,8 @@ describe("registerPolicyDoctorChecks", () => { expect.objectContaining({ checkId: "policy/tools-unknown-risk-level", severity: "error", - path: "TOOLS.md", - ocPath: "oc://TOOLS.md/tools/deploy", + path: "AGENTS.md", + ocPath: "oc://AGENTS.md/tools/deploy", }), ]); }); @@ -762,7 +856,7 @@ describe("registerPolicyDoctorChecks", () => { }), "utf-8", ); - await fs.writeFile(join(workspaceDir, "TOOLS.md"), "## Tools\n\n### deploy\n", "utf-8"); + await fs.writeFile(join(workspaceDir, "AGENTS.md"), "## Tools\n\n### deploy\n", "utf-8"); const result = await runPolicyDoctorLint( ctx(configPath, cfgWithPolicy({ enabled: undefined })), diff --git a/extensions/policy/src/doctor/scopes/tools.ts b/extensions/policy/src/doctor/scopes/tools.ts index fc9a14740643..087d75f36d12 100644 --- a/extensions/policy/src/doctor/scopes/tools.ts +++ b/extensions/policy/src/doctor/scopes/tools.ts @@ -145,10 +145,19 @@ export function createPolicyToolMetadataChecks( ): readonly HealthCheck[] { const { evaluatePolicy, findingsForCheck } = deps; + const policyUnmigratedToolsFileCheck: HealthCheck = { + id: CHECK_IDS.policyUnmigratedToolsFile, + kind: "plugin", + description: "Governed tool declarations have been migrated from TOOLS.md into AGENTS.md.", + source: "policy", + async detect(ctx) { + return findingsForCheck(await evaluatePolicy(ctx), CHECK_IDS.policyUnmigratedToolsFile); + }, + }; const policyToolsMissingRiskCheck: HealthCheck = { id: CHECK_IDS.policyMissingToolRisk, kind: "plugin", - description: "TOOLS.md policy entries declare explicit risk levels.", + description: "AGENTS.md tool policy entries declare explicit risk levels.", source: "policy", async detect(ctx) { return findingsForCheck(await evaluatePolicy(ctx), CHECK_IDS.policyMissingToolRisk); @@ -157,7 +166,7 @@ export function createPolicyToolMetadataChecks( const policyToolsUnknownRiskCheck: HealthCheck = { id: CHECK_IDS.policyUnknownToolRisk, kind: "plugin", - description: "TOOLS.md policy entries use known risk levels.", + description: "AGENTS.md tool policy entries use known risk levels.", source: "policy", async detect(ctx) { return findingsForCheck(await evaluatePolicy(ctx), CHECK_IDS.policyUnknownToolRisk); @@ -166,7 +175,7 @@ export function createPolicyToolMetadataChecks( const policyToolsMissingSensitivityCheck: HealthCheck = { id: CHECK_IDS.policyMissingToolSensitivity, kind: "plugin", - description: "TOOLS.md policy entries declare default artifact sensitivity.", + description: "AGENTS.md tool policy entries declare default artifact sensitivity.", source: "policy", async detect(ctx) { return findingsForCheck(await evaluatePolicy(ctx), CHECK_IDS.policyMissingToolSensitivity); @@ -175,7 +184,7 @@ export function createPolicyToolMetadataChecks( const policyToolsUnknownSensitivityCheck: HealthCheck = { id: CHECK_IDS.policyUnknownToolSensitivity, kind: "plugin", - description: "TOOLS.md policy entries use known sensitivity levels.", + description: "AGENTS.md tool policy entries use known sensitivity levels.", source: "policy", async detect(ctx) { return findingsForCheck(await evaluatePolicy(ctx), CHECK_IDS.policyUnknownToolSensitivity); @@ -184,7 +193,7 @@ export function createPolicyToolMetadataChecks( const policyToolsMissingOwnerCheck: HealthCheck = { id: CHECK_IDS.policyMissingToolOwner, kind: "plugin", - description: "TOOLS.md policy entries declare an accountable owner.", + description: "AGENTS.md tool policy entries declare an accountable owner.", source: "policy", async detect(ctx) { return findingsForCheck(await evaluatePolicy(ctx), CHECK_IDS.policyMissingToolOwner); @@ -192,6 +201,7 @@ export function createPolicyToolMetadataChecks( }; return [ + policyUnmigratedToolsFileCheck, policyToolsMissingRiskCheck, policyToolsUnknownRiskCheck, policyToolsMissingSensitivityCheck, diff --git a/extensions/policy/src/doctor/tool-findings.ts b/extensions/policy/src/doctor/tool-findings.ts index 403f22c4e6d8..bbd834280f38 100644 --- a/extensions/policy/src/doctor/tool-findings.ts +++ b/extensions/policy/src/doctor/tool-findings.ts @@ -362,9 +362,9 @@ export function toolRiskFindings( return { checkId: CHECK_IDS.policyMissingToolRisk, severity: "error", - message: `TOOLS.md tool '${tool.id}' has no explicit risk classification.`, + message: `AGENTS.md tool '${tool.id}' has no explicit risk classification.`, source: "policy", - path: "TOOLS.md", + path: "AGENTS.md", line: tool.line, ocPath: tool.source, target: tool.source, @@ -389,9 +389,9 @@ export function toolUnknownRiskFindings( return { checkId: CHECK_IDS.policyUnknownToolRisk, severity: "error", - message: `TOOLS.md tool '${tool.id}' declares unknown risk '${tool.risk}'.`, + message: `AGENTS.md tool '${tool.id}' declares unknown risk '${tool.risk}'.`, source: "policy", - path: "TOOLS.md", + path: "AGENTS.md", line: tool.line, ocPath: tool.source, target: tool.source, @@ -411,9 +411,9 @@ export function toolSensitivityFindings( { checkId: CHECK_IDS.policyMissingToolSensitivity, severity: "error", - message: `TOOLS.md tool '${tool.id}' has no declared artifact sensitivity.`, + message: `AGENTS.md tool '${tool.id}' has no declared artifact sensitivity.`, source: "policy", - path: "TOOLS.md", + path: "AGENTS.md", line: tool.line, ocPath: tool.source, target: tool.source, @@ -433,9 +433,9 @@ export function toolSensitivityFindings( { checkId: CHECK_IDS.policyUnknownToolSensitivity, severity: "error", - message: `TOOLS.md tool '${tool.id}' declares unknown sensitivity '${tool.sensitivity}'.`, + message: `AGENTS.md tool '${tool.id}' declares unknown sensitivity '${tool.sensitivity}'.`, source: "policy", - path: "TOOLS.md", + path: "AGENTS.md", line: tool.line, ocPath: tool.source, target: tool.source, @@ -456,9 +456,9 @@ export function toolOwnerFindings( return { checkId: CHECK_IDS.policyMissingToolOwner, severity: "error", - message: `TOOLS.md tool '${tool.id}' has no declared owner.`, + message: `AGENTS.md tool '${tool.id}' has no declared owner.`, source: "policy", - path: "TOOLS.md", + path: "AGENTS.md", line: tool.line, ocPath: tool.source, target: tool.source, diff --git a/extensions/policy/src/policy-state-tools.ts b/extensions/policy/src/policy-state-tools.ts index 3d34a5d95cc2..442f9544046d 100644 --- a/extensions/policy/src/policy-state-tools.ts +++ b/extensions/policy/src/policy-state-tools.ts @@ -1,4 +1,4 @@ -// Policy plugin TOOLS.md evidence. +// Policy plugin AGENTS.md Tools-section evidence. import { COLLAPSE_HYPHENS, NON_SLUG_CHARS, TRIM_HYPHENS } from "./policy-state-types.js"; import type { PolicyToolEvidence } from "./policy-state-types.js"; @@ -12,9 +12,47 @@ function scanPolicyToolHeaders(raw: string): readonly PolicyToolEvidence[] { return []; } const tools: PolicyToolEvidence[] = []; + let localNotesMode: "plain" | "migrated" | undefined; for (let index = 0; index < section.length; index += 1) { - const line = section[index]?.text ?? ""; - const heading = /^###\s+([^\s#]+)(.*)$/.exec(line); + const sectionLine = section[index]; + const line = sectionLine?.text ?? ""; + const sectionHeading = /^(#{1,6})\s+(.+?)\s*#*\s*$/u.exec(line); + const isChildHeading = sectionHeading?.[1]?.length === (sectionLine?.sectionDepth ?? 0) + 1; + if ( + line.includes("Skills provide your tools.") && + line.includes("Keep local notes") && + line.includes("TOOLS.md") + ) { + localNotesMode = "plain"; + continue; + } + if (isChildHeading && /^Local notes\s*$/iu.test(sectionHeading?.[2] ?? "")) { + localNotesMode = "plain"; + continue; + } + if ( + isChildHeading && + /^Local notes \(migrated from TOOLS\.md\)\s*$/iu.test(sectionHeading?.[2] ?? "") + ) { + localNotesMode = "migrated"; + continue; + } + if ( + localNotesMode && + sectionHeading && + sectionHeading[1]!.length <= (section[index]?.sectionDepth ?? 0) && + slugify(sectionHeading[2] ?? "") === "tools" + ) { + localNotesMode = undefined; + continue; + } + if (localNotesMode === "plain" && isChildHeading) { + localNotesMode = undefined; + } + if (localNotesMode) { + continue; + } + const heading = isChildHeading ? /^([^\s#]+)(.*)$/u.exec(sectionHeading?.[2] ?? "") : null; const bullet = /^[-*+]\s+([^:\s][^:]*?)\s*:(.*)$/.exec(line); const match = heading ?? bullet; const toolName = match?.[1]; @@ -35,13 +73,19 @@ function scanPolicyToolHeaders(raw: string): readonly PolicyToolEvidence[] { capabilities?: readonly string[]; } = { id, - source: `oc://TOOLS.md/tools/${id}`, + source: `oc://AGENTS.md/tools/${id}`, line: section[index]?.line ?? index + 1, }; const metaLines = [match[2] ?? ""]; for (let metaIndex = index + 1; metaIndex < section.length; metaIndex += 1) { - const metaLine = section[metaIndex]?.text ?? ""; - if (/^###\s+\S+/.test(metaLine.trim()) || /^[-*+]\s+[^:\s][^:]*?\s*:/.test(metaLine)) { + const metaSectionLine = section[metaIndex]; + const metaLine = metaSectionLine?.text ?? ""; + const metaHeading = /^(#{1,6})\s+(.+?)\s*#*\s*$/u.exec(metaLine.trim()); + if ( + (metaHeading !== null && + (metaHeading[1]?.length ?? 0) <= (metaSectionLine?.sectionDepth ?? 0) + 1) || + /^[-*+]\s+[^:\s][^:]*?\s*:/.test(metaLine) + ) { break; } metaLines.push(metaLine); @@ -71,30 +115,59 @@ function scanPolicyToolHeaders(raw: string): readonly PolicyToolEvidence[] { function markdownSectionLines( raw: string, sectionSlug: string, -): readonly { readonly line: number; readonly text: string }[] { +): readonly { + readonly line: number; + readonly text: string; + readonly sectionDepth: number; +}[] { const lines = raw.split(/\r?\n/); let sectionDepth: number | undefined; - const section: { line: number; text: string }[] = []; + let foundSection = false; + let fence: { marker: "`" | "~"; length: number } | undefined; + const section: { line: number; text: string; sectionDepth: number }[] = []; for (let index = 0; index < lines.length; index += 1) { const line = lines[index] ?? ""; + const fenceRun = /^\s*(`{3,}|~{3,})/.exec(line)?.[1]; + const closingFenceRun = /^\s*(`{3,}|~{3,})\s*$/.exec(line)?.[1]; + const marker = fenceRun?.[0] as "`" | "~" | undefined; + if (marker && !fence) { + fence = { marker, length: fenceRun!.length }; + continue; + } + if ( + closingFenceRun && + fence && + closingFenceRun[0] === fence.marker && + closingFenceRun.length >= fence.length + ) { + fence = undefined; + continue; + } + if (fence) { + continue; + } const heading = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line); if (heading !== null) { const depth = heading[1]?.length ?? 0; const slug = slugify(heading[2] ?? ""); if (sectionDepth !== undefined && depth <= sectionDepth) { - break; + sectionDepth = undefined; } if (sectionDepth !== undefined) { - section.push({ line: index + 1, text: line }); + section.push({ line: index + 1, text: line, sectionDepth }); continue; } - if (sectionDepth === undefined && slug === sectionSlug) { + if (depth <= 2 && slug === sectionSlug) { + if (foundSection) { + section.push({ line: index + 1, text: line, sectionDepth: depth }); + } + foundSection = true; sectionDepth = depth; } continue; } if (sectionDepth !== undefined) { - section.push({ line: index + 1, text: line }); + section.push({ line: index + 1, text: line, sectionDepth }); } } return section; diff --git a/extensions/policy/src/policy-state.test.ts b/extensions/policy/src/policy-state.test.ts index c47c13600f35..2f157c0f5028 100644 --- a/extensions/policy/src/policy-state.test.ts +++ b/extensions/policy/src/policy-state.test.ts @@ -70,7 +70,7 @@ describe("scanPolicyTools", () => { ).resolves.toEqual([ { id: "deploy-tool", - source: "oc://TOOLS.md/tools/deploy-tool", + source: "oc://AGENTS.md/tools/deploy-tool", line: 2, risk: "critical", sensitivity: "restricted", @@ -79,7 +79,7 @@ describe("scanPolicyTools", () => { }, { id: "inspect", - source: "oc://TOOLS.md/tools/inspect", + source: "oc://AGENTS.md/tools/inspect", line: 3, risk: "low", sensitivity: "public", @@ -94,13 +94,106 @@ describe("scanPolicyTools", () => { ).resolves.toEqual([ { id: "deploy", - source: "oc://TOOLS.md/tools/deploy", + source: "oc://AGENTS.md/tools/deploy", line: 2, risk: "critical", owner: "ops", }, ]); }); + + it("ignores local-note examples inside fenced blocks", async () => { + await expect( + scanPolicyTools( + [ + "## Tools", + "```markdown", + "- SSH: home-server -> 192.168.1.100", + "### Cameras", + "```", + ].join("\n"), + ), + ).resolves.toEqual([]); + }); + + it("ignores the complete local-notes subsection", async () => { + await expect( + scanPolicyTools( + ["## Tools", "### Local notes", "- SSH: prod-host", "### deploy risk: high"].join("\n"), + ), + ).resolves.toEqual([expect.objectContaining({ id: "deploy", risk: "high" })]); + }); + + it("parses a tool literally named tools after local notes", async () => { + await expect( + scanPolicyTools( + [ + "## Tools", + "### Local notes", + "- SSH: prod-host", + "### tools risk: high sensitivity: restricted owner: ops", + ].join("\n"), + ), + ).resolves.toEqual([ + expect.objectContaining({ + id: "tools", + risk: "high", + sensitivity: "restricted", + owner: "ops", + }), + ]); + }); + + it("ignores deeper Tools sections outside the governed H1/H2 contract", async () => { + await expect( + scanPolicyTools( + [ + "## Build", + "### Tools", + "- npm: risk: high owner: ops", + "## Tools", + "### deploy risk: low owner: release", + ].join("\n"), + ), + ).resolves.toEqual([expect.objectContaining({ id: "deploy", risk: "low", owner: "release" })]); + }); + + it("does not carry metadata across repeated Tools section boundaries", async () => { + const evidence = await scanPolicyTools( + [ + "## Tools", + "### deploy risk: high", + "## Tools", + "owner: ops", + "### inspect risk: low owner: support", + ].join("\n"), + ); + expect(evidence).toEqual([ + expect.objectContaining({ id: "deploy", risk: "high" }), + expect.objectContaining({ id: "inspect", owner: "support" }), + ]); + expect(evidence[0]).not.toHaveProperty("owner"); + }); + + it("keeps longer fences open across shorter delimiter runs", async () => { + await expect( + scanPolicyTools(["## Tools", "````markdown", "```", "- SSH: home-server", "````"].join("\n")), + ).resolves.toEqual([]); + }); + + it("scans a migrated legacy Tools section after its document heading", async () => { + await expect( + scanPolicyTools( + [ + "## Tools", + "### Local notes (migrated from TOOLS.md)", + "# TOOLS.md", + "## Tools", + "### deploy risk: high sensitivity: restricted owner: ops", + ].join("\n"), + ), + ).resolves.toEqual([expect.objectContaining({ id: "deploy", risk: "high", owner: "ops" })]); + }); }); describe("scanPolicyExecApprovals", () => { diff --git a/scripts/docker/install-sh-e2e/run.sh b/scripts/docker/install-sh-e2e/run.sh index 19a2e472b30c..eacbe118fe61 100755 --- a/scripts/docker/install-sh-e2e/run.sh +++ b/scripts/docker/install-sh-e2e/run.sh @@ -855,7 +855,6 @@ run_profile() { test -f "$workspace/IDENTITY.md" test -f "$workspace/USER.md" test -f "$workspace/SOUL.md" - test -f "$workspace/TOOLS.md" # The remaining checks are deterministic tool smokes, not the interactive # first-run identity ritual. Drop BOOTSTRAP.md so provider prompts stay focused # on the fixture task and do not spend turns following onboarding copy. diff --git a/scripts/e2e/lib/onboard/scenario.sh b/scripts/e2e/lib/onboard/scenario.sh index 8859c42e4406..4f2e6b5ce910 100644 --- a/scripts/e2e/lib/onboard/scenario.sh +++ b/scripts/e2e/lib/onboard/scenario.sh @@ -261,7 +261,7 @@ run_case_local_basic() { sessions_dir="$OPENCLAW_STATE_DIR/agents/main/sessions" openclaw_e2e_assert_dir "$sessions_dir" - for file in AGENTS.md BOOTSTRAP.md IDENTITY.md SOUL.md TOOLS.md USER.md; do + for file in AGENTS.md BOOTSTRAP.md IDENTITY.md SOUL.md USER.md; do openclaw_e2e_assert_file "$workspace_dir/$file" done diff --git a/scripts/lib/workspace-bootstrap-smoke.mjs b/scripts/lib/workspace-bootstrap-smoke.mjs index 33c2ae17868f..4afb049889e6 100644 --- a/scripts/lib/workspace-bootstrap-smoke.mjs +++ b/scripts/lib/workspace-bootstrap-smoke.mjs @@ -10,7 +10,6 @@ import { dirname, join } from "node:path"; export const WORKSPACE_TEMPLATE_PACK_PATHS = [ "docs/reference/templates/AGENTS.md", "docs/reference/templates/SOUL.md", - "docs/reference/templates/TOOLS.md", "docs/reference/templates/IDENTITY.md", "docs/reference/templates/USER.md", "src/agents/templates/HEARTBEAT.md", @@ -22,7 +21,6 @@ export const WORKSPACE_TEMPLATE_PACK_PATHS = [ const REQUIRED_BOOTSTRAP_WORKSPACE_FILES = [ "AGENTS.md", "SOUL.md", - "TOOLS.md", "IDENTITY.md", "USER.md", "BOOTSTRAP.md", diff --git a/src/agents/bootstrap-files.test.ts b/src/agents/bootstrap-files.test.ts index 7a75c1c84c76..8d922ad40dab 100644 --- a/src/agents/bootstrap-files.test.ts +++ b/src/agents/bootstrap-files.test.ts @@ -187,7 +187,6 @@ describe("resolveBootstrapFilesForRun", () => { expect(files.map((file) => path.relative(workspaceDir, file.path))).toEqual([ "AGENTS.md", "SOUL.md", - "TOOLS.md", "IDENTITY.md", "USER.md", "BOOTSTRAP.md", @@ -302,12 +301,11 @@ describe("resolveBootstrapFilesForRun", () => { expect(files.map((file) => file.path)).not.toContain(path.join(workspaceDir, "BOOTSTRAP.md")); }); - it("keeps subagent sessions to project and tool bootstrap files", async () => { + it("keeps subagent sessions to AGENTS.md", async () => { const workspaceDir = await makeTempWorkspace("openclaw-bootstrap-subagent-"); await Promise.all( [ ["AGENTS.md", "project rules"], - ["TOOLS.md", "tool rules"], ["SOUL.md", "persona"], ["IDENTITY.md", "identity"], ["USER.md", "user profile"], @@ -328,7 +326,7 @@ describe("resolveBootstrapFilesForRun", () => { sessionKey: "agent:main:subagent:worker", }); - expect(files.map((file) => file.name)).toStrictEqual(["AGENTS.md", "TOOLS.md"]); + expect(files.map((file) => file.name)).toStrictEqual(["AGENTS.md"]); }); it("keeps cron sessions on their existing minimal bootstrap files", async () => { @@ -336,7 +334,6 @@ describe("resolveBootstrapFilesForRun", () => { await Promise.all( [ ["AGENTS.md", "project rules"], - ["TOOLS.md", "tool rules"], ["SOUL.md", "persona"], ["IDENTITY.md", "identity"], ["USER.md", "user profile"], @@ -360,7 +357,6 @@ describe("resolveBootstrapFilesForRun", () => { expect(files.map((file) => file.name)).toStrictEqual([ "AGENTS.md", "SOUL.md", - "TOOLS.md", "IDENTITY.md", "USER.md", ]); diff --git a/src/agents/embedded-agent-helpers.buildbootstrapcontextfiles.test.ts b/src/agents/embedded-agent-helpers.buildbootstrapcontextfiles.test.ts index d95dbf8e909a..722d536ad9fb 100644 --- a/src/agents/embedded-agent-helpers.buildbootstrapcontextfiles.test.ts +++ b/src/agents/embedded-agent-helpers.buildbootstrapcontextfiles.test.ts @@ -67,7 +67,7 @@ describe("buildBootstrapContextFiles", () => { const head = `HEAD-${"a".repeat(600)}`; const tail = `${"b".repeat(300)}-TAIL`; const long = `${head}${tail}`; - const files = [makeFile({ name: "TOOLS.md", content: long })]; + const files = [makeFile({ name: "SOUL.md", path: "/tmp/SOUL.md", content: long })]; const warnings: string[] = []; const maxChars = 200; const [result] = buildBootstrapContextFiles(files, { @@ -75,10 +75,10 @@ describe("buildBootstrapContextFiles", () => { warn: (message) => warnings.push(message), }); const kept = result?.content.match(/kept (\d+)\+(\d+) chars/); - expect(kept?.slice(0, 3)).toStrictEqual(["kept 74+24 chars", "74", "24"]); + expect(kept?.slice(0, 3)).toStrictEqual(["kept 75+25 chars", "75", "25"]); const headChars = Number(kept?.[1]); const tailChars = Number(kept?.[2]); - expect(result?.content).toContain("[...truncated, read TOOLS.md for full content...]"); + expect(result?.content).toContain("[...truncated, read SOUL.md for full content...]"); expect(result?.content.length).toBe(199); expect(result?.content.length).toBeLessThan(long.length); expect(result?.content.length).toBeLessThanOrEqual(maxChars); @@ -87,15 +87,15 @@ describe("buildBootstrapContextFiles", () => { expect(result?.content.endsWith(long.slice(-tailChars))).toBe(true); } expect(warnings).toHaveLength(1); - expect(warnings[0]).toContain("TOOLS.md"); + expect(warnings[0]).toContain("SOUL.md"); expect(warnings[0]).toContain("limit 200"); }); it("keeps generic and AGENTS.md truncation valid at UTF-16 boundaries", () => { const cases = [ { file: makeFile({ - name: "TOOLS.md", - path: "/tmp/TOOLS.md", + name: "SOUL.md", + path: "/tmp/SOUL.md", content: `${"h".repeat(73)}😀${"m".repeat(200)}😀${"t".repeat(23)}`, }), maxChars: 200, diff --git a/src/agents/embedded-agent-runner/run/attempt.bootstrap-context.test.ts b/src/agents/embedded-agent-runner/run/attempt.bootstrap-context.test.ts index 89f4a49af861..115da8ac187b 100644 --- a/src/agents/embedded-agent-runner/run/attempt.bootstrap-context.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.bootstrap-context.test.ts @@ -14,8 +14,8 @@ describe("remapInjectedContextFilesToWorkspace", () => { content: "agents", }, { - path: "/real/workspace/nested/TOOLS.md", - content: "tools", + path: "/real/workspace/nested/SOUL.md", + content: "soul", }, { path: "/real/workspace/..context/USER.md", @@ -35,8 +35,8 @@ describe("remapInjectedContextFilesToWorkspace", () => { content: "agents", }, { - path: "/sandbox/workspace/nested/TOOLS.md", - content: "tools", + path: "/sandbox/workspace/nested/SOUL.md", + content: "soul", }, { path: "/sandbox/workspace/..context/USER.md", diff --git a/src/agents/sandbox/workspace.test.ts b/src/agents/sandbox/workspace.test.ts index 1f94da9918a5..c2dd21cfbc67 100644 --- a/src/agents/sandbox/workspace.test.ts +++ b/src/agents/sandbox/workspace.test.ts @@ -5,7 +5,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES } from "../workspace-bootstrap-read.js"; -import { DEFAULT_AGENTS_FILENAME, DEFAULT_TOOLS_FILENAME } from "../workspace.js"; +import { DEFAULT_AGENTS_FILENAME, DEFAULT_SOUL_FILENAME } from "../workspace.js"; import { ensureSandboxWorkspace } from "./workspace.js"; const tempRoots: string[] = []; @@ -90,15 +90,15 @@ describe("ensureSandboxWorkspace", () => { `## Startup\n\n` + "x".repeat(MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES), "utf-8", ); - await fs.writeFile(path.join(seed, DEFAULT_TOOLS_FILENAME), "seeded-tools", "utf-8"); + await fs.writeFile(path.join(seed, DEFAULT_SOUL_FILENAME), "seeded-soul", "utf-8"); await ensureSandboxWorkspace(sandbox, seed, true); await expect(fs.readFile(path.join(sandbox, DEFAULT_AGENTS_FILENAME), "utf-8")).rejects.toThrow( "no such file", ); - await expect(fs.readFile(path.join(sandbox, DEFAULT_TOOLS_FILENAME), "utf-8")).resolves.toBe( - "seeded-tools", + await expect(fs.readFile(path.join(sandbox, DEFAULT_SOUL_FILENAME), "utf-8")).resolves.toBe( + "seeded-soul", ); }); diff --git a/src/agents/sandbox/workspace.ts b/src/agents/sandbox/workspace.ts index 03f5417826c1..00806cd85e53 100644 --- a/src/agents/sandbox/workspace.ts +++ b/src/agents/sandbox/workspace.ts @@ -19,7 +19,6 @@ import { DEFAULT_BOOTSTRAP_FILENAME, DEFAULT_IDENTITY_FILENAME, DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_USER_FILENAME, ensureAgentWorkspace, } from "../workspace.js"; @@ -38,7 +37,6 @@ export async function ensureSandboxWorkspace( const files = [ DEFAULT_AGENTS_FILENAME, DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_IDENTITY_FILENAME, DEFAULT_USER_FILENAME, DEFAULT_BOOTSTRAP_FILENAME, diff --git a/src/agents/system-prompt-stability.test.ts b/src/agents/system-prompt-stability.test.ts index fad469bc367e..1afc049c5655 100644 --- a/src/agents/system-prompt-stability.test.ts +++ b/src/agents/system-prompt-stability.test.ts @@ -7,7 +7,6 @@ import { makeTempWorkspace, writeWorkspaceFile } from "../test-helpers/workspace import { loadWorkspaceBootstrapFiles, DEFAULT_AGENTS_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_SOUL_FILENAME, } from "./workspace.js"; @@ -20,7 +19,6 @@ describe("system prompt stability for cache hits", () => { it("returns identical results for same inputs across multiple calls", async () => { const agentsContent = "# AGENTS.md - Your Workspace\n\nTest agents file."; - const toolsContent = "# TOOLS.md - Local Notes\n\nTest tools file."; const soulContent = "# SOUL.md - Who You Are\n\nTest soul file."; // Write workspace files @@ -29,11 +27,6 @@ describe("system prompt stability for cache hits", () => { name: DEFAULT_AGENTS_FILENAME, content: agentsContent, }); - await writeWorkspaceFile({ - dir: workspaceDir, - name: DEFAULT_TOOLS_FILENAME, - content: toolsContent, - }); await writeWorkspaceFile({ dir: workspaceDir, name: DEFAULT_SOUL_FILENAME, @@ -58,21 +51,16 @@ describe("system prompt stability for cache hits", () => { const agentsFiles = results.map((result) => result.find((f) => f.name === DEFAULT_AGENTS_FILENAME), ); - const toolsFiles = results.map((result) => - result.find((f) => f.name === DEFAULT_TOOLS_FILENAME), - ); const soulFiles = results.map((result) => result.find((f) => f.name === DEFAULT_SOUL_FILENAME)); // All instances should have identical content for (let i = 1; i < agentsFiles.length; i++) { expect(agentsFiles[i]?.content).toBe(agentsFiles[0]?.content); - expect(toolsFiles[i]?.content).toBe(toolsFiles[0]?.content); expect(soulFiles[i]?.content).toBe(soulFiles[0]?.content); } // Verify the actual content matches what we wrote expect(agentsFiles[0]?.content).toBe(agentsContent); - expect(toolsFiles[0]?.content).toBe(toolsContent); expect(soulFiles[0]?.content).toBe(soulContent); }); @@ -81,7 +69,6 @@ describe("system prompt stability for cache hits", () => { // the canonical bootstrap ordering independent of filesystem timing. const testFiles = [ { name: DEFAULT_AGENTS_FILENAME, content: "# Agents content" }, - { name: DEFAULT_TOOLS_FILENAME, content: "# Tools content" }, { name: DEFAULT_SOUL_FILENAME, content: "# Soul content" }, ]; @@ -128,12 +115,8 @@ describe("system prompt stability for cache hits", () => { // Verify missing files are consistently marked as missing for (const result of results) { const agentsFile = result.find((f) => f.name === DEFAULT_AGENTS_FILENAME); - const toolsFile = result.find((f) => f.name === DEFAULT_TOOLS_FILENAME); - expect(agentsFile?.missing).toBe(false); expect(agentsFile?.content).toBe("# Agents only"); - expect(toolsFile?.missing).toBe(true); - expect(toolsFile?.content).toBeUndefined(); } }); diff --git a/src/agents/system-prompt.ts b/src/agents/system-prompt.ts index 0c7b0f527167..3e8a366c368d 100644 --- a/src/agents/system-prompt.ts +++ b/src/agents/system-prompt.ts @@ -1108,7 +1108,7 @@ export function buildAgentSystemPrompt(params: { ...(toolSchemaDirectoryPrompt ? ["", "### Deferred Tool Schemas", toolSchemaDirectoryPrompt] : []), - "TOOLS.md guides usage; never grants availability.", + "The AGENTS.md Tools section guides usage; it never grants availability.", ...(renderOpenClawToolWorkflowHints ? [ `Long wait: no rapid poll. Use ${execToolName} yieldMs or ${processToolName}(poll, timeout=).`, diff --git a/src/agents/workspace.attestation-survival.test.ts b/src/agents/workspace.attestation-survival.test.ts new file mode 100644 index 000000000000..e95b7a253510 --- /dev/null +++ b/src/agents/workspace.attestation-survival.test.ts @@ -0,0 +1,129 @@ +// Workspace attestation survival tests cover the canonical AGENTS.md hash and +// ignore retired generated-file evidence. +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { + createOpenClawTestState, + type OpenClawTestState, +} from "../test-utils/openclaw-test-state.js"; +import { resetLegacyWorkspaceStateCheckForTest } from "./workspace-legacy-state.test-support.js"; +import { + readWorkspaceStateSnapshot, + replaceWorkspaceAttestation, +} from "./workspace-state-store.js"; +import { + DEFAULT_AGENTS_FILENAME, + DEFAULT_BOOTSTRAP_FILENAME, + ensureAgentWorkspace, + WORKSPACE_VANISHED_ERROR_CODE, +} from "./workspace.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +let testState: OpenClawTestState | undefined; + +beforeEach(async () => { + resetLegacyWorkspaceStateCheckForTest(); + testState = await createOpenClawTestState({ + layout: "state-only", + prefix: "openclaw-workspace-attestation-survival-", + }); +}); + +afterEach(async () => { + closeOpenClawStateDatabaseForTest(); + resetLegacyWorkspaceStateCheckForTest(); + await testState?.cleanup(); + testState = undefined; +}); + +async function makeWorkspace(): Promise { + return fs.realpath(tempDirs.make("openclaw-workspace-attestation-survival-")); +} + +async function expectWorkspaceVanished(action: Promise): Promise { + await expect(action).rejects.toMatchObject({ + code: WORKSPACE_VANISHED_ERROR_CODE, + name: "WorkspaceVanishedError", + }); +} + +describe("workspace attestation survival", () => { + it("ignores retired generated-file hashes", async () => { + const tempDir = await makeWorkspace(); + await ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }); + await fs.rm(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); + await ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }); + + const snapshot = readWorkspaceStateSnapshot(tempDir); + replaceWorkspaceAttestation({ + workspaceDir: tempDir, + attestedAtMs: Date.now(), + generatedHashes: new Map([ + ...snapshot.attestation!.generatedHashes, + ["RETIRED.md", "a".repeat(64)], + ]), + }); + + await expect( + ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }), + ).resolves.toMatchObject({ dir: tempDir }); + }); + + it("requires an AGENTS.md hash before trusting generated-file evidence", async () => { + const tempDir = await makeWorkspace(); + await ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }); + await fs.rm(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); + const snapshot = readWorkspaceStateSnapshot(tempDir); + const generatedHashes = new Map(snapshot.attestation!.generatedHashes); + generatedHashes.delete(DEFAULT_AGENTS_FILENAME); + replaceWorkspaceAttestation({ + workspaceDir: tempDir, + attestedAtMs: Date.now(), + generatedHashes, + }); + + await expectWorkspaceVanished( + ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }), + ); + }); + + it("accepts customized AGENTS.md when its attestation hash is missing", async () => { + const tempDir = await makeWorkspace(); + await ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }); + await fs.rm(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); + await fs.writeFile(path.join(tempDir, DEFAULT_AGENTS_FILENAME), "custom instructions\n"); + const snapshot = readWorkspaceStateSnapshot(tempDir); + const generatedHashes = new Map(snapshot.attestation!.generatedHashes); + generatedHashes.delete(DEFAULT_AGENTS_FILENAME); + replaceWorkspaceAttestation({ + workspaceDir: tempDir, + attestedAtMs: Date.now(), + generatedHashes, + }); + + await expect( + ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }), + ).resolves.toMatchObject({ dir: tempDir }); + }); + + it("rejects a corrupted AGENTS.md attestation hash", async () => { + const tempDir = await makeWorkspace(); + await ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }); + await fs.rm(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); + const snapshot = readWorkspaceStateSnapshot(tempDir); + const generatedHashes = new Map(snapshot.attestation!.generatedHashes); + generatedHashes.set(DEFAULT_AGENTS_FILENAME, "0".repeat(64)); + replaceWorkspaceAttestation({ + workspaceDir: tempDir, + attestedAtMs: Date.now(), + generatedHashes, + }); + + await expectWorkspaceVanished( + ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }), + ); + }); +}); diff --git a/src/agents/workspace.load-extra-bootstrap-files.test.ts b/src/agents/workspace.load-extra-bootstrap-files.test.ts index a445771c2afc..aedd401ddf63 100644 --- a/src/agents/workspace.load-extra-bootstrap-files.test.ts +++ b/src/agents/workspace.load-extra-bootstrap-files.test.ts @@ -4,7 +4,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { loadExtraBootstrapFilesWithDiagnostics } from "./workspace.js"; +import { + loadExtraBootstrapFilesWithDiagnostics, + loadWorkspacePatternFilesWithDiagnostics, +} from "./workspace.js"; describe("loadExtraBootstrapFilesWithDiagnostics", () => { let fixtureRoot = ""; @@ -35,16 +38,16 @@ describe("loadExtraBootstrapFilesWithDiagnostics", () => { const workspaceDir = await createWorkspaceDir("glob"); const packageDir = path.join(workspaceDir, "packages", "core"); await fs.mkdir(packageDir, { recursive: true }); - await fs.writeFile(path.join(packageDir, "TOOLS.md"), "tools", "utf-8"); + await fs.writeFile(path.join(packageDir, "SOUL.md"), "soul", "utf-8"); await fs.writeFile(path.join(packageDir, "README.md"), "not bootstrap", "utf-8"); const files = await loadExtraBootstrapFileList(workspaceDir, ["packages/*/*"]); expect(files).toStrictEqual([ { - name: "TOOLS.md", - path: path.join(packageDir, "TOOLS.md"), - content: "tools", + name: "SOUL.md", + path: path.join(packageDir, "SOUL.md"), + content: "soul", missing: false, }, ]); @@ -86,6 +89,24 @@ describe("loadExtraBootstrapFilesWithDiagnostics", () => { ]); }); + it("loads bootstrap files from valid child directories beginning with two dots", async () => { + const workspaceDir = await createWorkspaceDir("dotdot-name"); + const packageDir = path.join(workspaceDir, "..notes"); + await fs.mkdir(packageDir); + await fs.writeFile(path.join(packageDir, "AGENTS.md"), "agents", "utf-8"); + + const files = await loadExtraBootstrapFileList(workspaceDir, ["..notes/AGENTS.md"]); + + expect(files).toStrictEqual([ + { + name: "AGENTS.md", + path: path.join(packageDir, "AGENTS.md"), + content: "agents", + missing: false, + }, + ]); + }); + it("keeps path-traversal attempts outside workspace excluded", async () => { const rootDir = await createWorkspaceDir("root"); const workspaceDir = path.join(rootDir, "workspace"); @@ -163,4 +184,67 @@ describe("loadExtraBootstrapFilesWithDiagnostics", () => { expect(files).toHaveLength(0); expect(diagnostics.map((diagnostic) => diagnostic.reason)).toContain("security"); }); + + it.runIf(process.platform !== "win32")( + "reports unreadable glob branches during strict doctor discovery", + async () => { + const workspaceDir = await createWorkspaceDir("strict-unreadable"); + const blockedDir = path.join(workspaceDir, "packages", "blocked"); + const readableDir = path.join(workspaceDir, "packages", "readable"); + await fs.mkdir(blockedDir, { recursive: true }); + await fs.mkdir(readableDir, { recursive: true }); + await fs.writeFile(path.join(blockedDir, "TOOLS.md"), "blocked", "utf-8"); + await fs.writeFile(path.join(readableDir, "TOOLS.md"), "readable", "utf-8"); + await fs.chmod(blockedDir, 0o000); + try { + const result = await loadWorkspacePatternFilesWithDiagnostics( + workspaceDir, + ["packages/*/TOOLS.md"], + { + acceptedBasenames: new Set(["TOOLS.md"]), + strictPatternRead: true, + }, + ); + expect(result.files).toEqual([]); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ + reason: "io", + path: path.join(workspaceDir, "packages", "*", "TOOLS.md"), + }), + ]); + } finally { + await fs.chmod(blockedDir, 0o700); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "does not descend into unreadable branches that cannot satisfy a shallow pattern", + async () => { + const workspaceDir = await createWorkspaceDir("strict-pruned"); + const privateDir = path.join(workspaceDir, "packages", "blocked", "node_modules", "private"); + const readableDir = path.join(workspaceDir, "packages", "readable"); + await fs.mkdir(privateDir, { recursive: true }); + await fs.mkdir(readableDir, { recursive: true }); + await fs.writeFile(path.join(privateDir, "TOOLS.md"), "irrelevant", "utf-8"); + await fs.writeFile(path.join(readableDir, "TOOLS.md"), "readable", "utf-8"); + await fs.chmod(privateDir, 0o000); + try { + const result = await loadWorkspacePatternFilesWithDiagnostics( + workspaceDir, + ["packages/*/TOOLS.md"], + { + acceptedBasenames: new Set(["TOOLS.md"]), + strictPatternRead: true, + }, + ); + expect(result.diagnostics).toEqual([]); + expect(result.files).toEqual([ + expect.objectContaining({ path: path.join(readableDir, "TOOLS.md") }), + ]); + } finally { + await fs.chmod(privateDir, 0o700); + } + }, + ); }); diff --git a/src/agents/workspace.test.ts b/src/agents/workspace.test.ts index 7f20386cbd05..32bea7988661 100644 --- a/src/agents/workspace.test.ts +++ b/src/agents/workspace.test.ts @@ -31,7 +31,6 @@ import { DEFAULT_IDENTITY_FILENAME, DEFAULT_MEMORY_FILENAME, DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_USER_FILENAME, ensureAgentWorkspace, filterBootstrapFilesForSession, @@ -143,12 +142,12 @@ async function expectCompletedWithoutBootstrap(dir: string) { function expectSubagentAllowedBootstrapNames(files: WorkspaceBootstrapFile[]) { const names = files.map((file) => file.name); - expect(names).toStrictEqual(["AGENTS.md", "TOOLS.md"]); + expect(names).toStrictEqual(["AGENTS.md"]); } function expectCronAllowedBootstrapNames(files: WorkspaceBootstrapFile[]) { const names = files.map((file) => file.name); - expect(names).toStrictEqual(["AGENTS.md", "SOUL.md", "TOOLS.md", "IDENTITY.md", "USER.md"]); + expect(names).toStrictEqual(["AGENTS.md", "SOUL.md", "IDENTITY.md", "USER.md"]); } describe("ensureAgentWorkspace", () => { @@ -290,7 +289,7 @@ describe("ensureAgentWorkspace", () => { await expectPathMissing(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); }); - it("refuses to accept old generated bootstrap files recorded by SQLite attestation", async () => { + it("accepts an intact historical AGENTS.md recorded by SQLite attestation", async () => { const tempDir = await makeTempWorkspace("openclaw-workspace-"); const oldGeneratedAgents = "old generated agents\n"; await fs.writeFile(path.join(tempDir, DEFAULT_AGENTS_FILENAME), oldGeneratedAgents); @@ -306,9 +305,9 @@ describe("ensureAgentWorkspace", () => { ]), }); - await expectWorkspaceVanished( + await expect( ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }), - ); + ).resolves.toMatchObject({ dir: tempDir }); await expectPathMissing(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); }); @@ -566,12 +565,10 @@ describe("ensureAgentWorkspace", () => { await writeWorkspaceFile({ dir: tempDir, name: DEFAULT_IDENTITY_FILENAME, content: "custom" }); await writeWorkspaceFile({ dir: tempDir, name: DEFAULT_USER_FILENAME, content: "custom" }); await fs.unlink(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); - await fs.unlink(path.join(tempDir, DEFAULT_TOOLS_FILENAME)); await ensureAgentWorkspace({ dir: tempDir, ensureBootstrapFiles: true }); await expectPathMissing(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)); - await expect(fs.access(path.join(tempDir, DEFAULT_TOOLS_FILENAME))).resolves.toBeUndefined(); const state = await readWorkspaceState(tempDir); expect(state.setupCompletedAt).toMatch(/\d{4}-\d{2}-\d{2}T/); }); @@ -630,7 +627,6 @@ describe("ensureAgentWorkspace", () => { }); await expect(fs.access(path.join(tempDir, DEFAULT_AGENTS_FILENAME))).resolves.toBeUndefined(); - await expect(fs.access(path.join(tempDir, DEFAULT_TOOLS_FILENAME))).resolves.toBeUndefined(); await expect( fs.access(path.join(tempDir, DEFAULT_BOOTSTRAP_FILENAME)), ).resolves.toBeUndefined(); @@ -879,9 +875,8 @@ describe("ensureAgentWorkspace", () => { await expectPathMissing(path.join(tempDir, DEFAULT_USER_FILENAME)); await expectPathMissing(path.join(tempDir, DEFAULT_HEARTBEAT_FILENAME)); - // Verify required files (AGENTS.md, TOOLS.md) still exist. + // Verify the required AGENTS.md file still exists. await expect(fs.access(path.join(tempDir, DEFAULT_AGENTS_FILENAME))).resolves.toBeUndefined(); - await expect(fs.access(path.join(tempDir, DEFAULT_TOOLS_FILENAME))).resolves.toBeUndefined(); }); it("observes setup completed concurrently before writing optional bootstrap files", async () => { @@ -1138,7 +1133,6 @@ describe("filterBootstrapFilesForSession", () => { const mockFiles: WorkspaceBootstrapFile[] = [ { name: "AGENTS.md", path: "/w/AGENTS.md", content: "", missing: false }, { name: "SOUL.md", path: "/w/SOUL.md", content: "", missing: false }, - { name: "TOOLS.md", path: "/w/TOOLS.md", content: "", missing: false }, { name: "IDENTITY.md", path: "/w/IDENTITY.md", content: "", missing: false }, { name: "USER.md", path: "/w/USER.md", content: "", missing: false }, { name: "BOOTSTRAP.md", path: "/w/BOOTSTRAP.md", content: "", missing: false }, diff --git a/src/agents/workspace.ts b/src/agents/workspace.ts index aa7d672fe418..e4158ccd75e8 100644 --- a/src/agents/workspace.ts +++ b/src/agents/workspace.ts @@ -7,9 +7,11 @@ import { createHash } from "node:crypto"; import syncFs from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import { Minimatch } from "minimatch"; import { extractFrontmatterBlock } from "../../packages/markdown-core/src/frontmatter.js"; import { openRootFile } from "../infra/boundary-file-read.js"; import { pathExists } from "../infra/fs-safe.js"; +import { isPathInside } from "../infra/path-guards.js"; import { retryAsync } from "../infra/retry.js"; import { CANONICAL_ROOT_MEMORY_FILENAME, @@ -51,6 +53,15 @@ export const DEFAULT_USER_FILENAME = "USER.md"; export const DEFAULT_HEARTBEAT_FILENAME = "HEARTBEAT.md"; export const DEFAULT_BOOTSTRAP_FILENAME = "BOOTSTRAP.md"; export const DEFAULT_MEMORY_FILENAME = CANONICAL_ROOT_MEMORY_FILENAME; +export const GENERATED_WORKSPACE_BOOTSTRAP_FILENAMES = [ + DEFAULT_AGENTS_FILENAME, + DEFAULT_SOUL_FILENAME, + DEFAULT_IDENTITY_FILENAME, + DEFAULT_USER_FILENAME, +] as const; +const GENERATED_WORKSPACE_BOOTSTRAP_FILENAME_SET: ReadonlySet = new Set( + GENERATED_WORKSPACE_BOOTSTRAP_FILENAMES, +); const WORKSPACE_ONBOARDING_PROFILE_FILENAMES = [ DEFAULT_SOUL_FILENAME, DEFAULT_IDENTITY_FILENAME, @@ -178,7 +189,6 @@ async function loadTemplate(name: string): Promise { export type WorkspaceBootstrapFileName = | typeof DEFAULT_AGENTS_FILENAME | typeof DEFAULT_SOUL_FILENAME - | typeof DEFAULT_TOOLS_FILENAME | typeof DEFAULT_IDENTITY_FILENAME | typeof DEFAULT_USER_FILENAME | typeof DEFAULT_BOOTSTRAP_FILENAME @@ -203,11 +213,16 @@ export type ExtraBootstrapLoadDiagnostic = { detail: string; }; +export type WorkspacePatternFile = { + name: string; + path: string; + content: string; +}; + /** Set of recognized bootstrap filenames for runtime validation */ const VALID_BOOTSTRAP_NAMES: ReadonlySet = new Set([ DEFAULT_AGENTS_FILENAME, DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_IDENTITY_FILENAME, DEFAULT_USER_FILENAME, DEFAULT_BOOTSTRAP_FILENAME, @@ -374,7 +389,7 @@ async function workspaceRequiredBootstrapLooksCustomized( dir: string, opts?: { generatedHashes?: ReadonlyMap }, ): Promise { - const fileNames = [DEFAULT_AGENTS_FILENAME, DEFAULT_TOOLS_FILENAME]; + const fileNames = [DEFAULT_AGENTS_FILENAME]; const generatedHashes = opts?.generatedHashes; if (generatedHashes && generatedHashes.size > 0) { for (const fileName of fileNames) { @@ -383,7 +398,7 @@ async function workspaceRequiredBootstrapLooksCustomized( try { const content = await fs.readFile(filePath, "utf-8"); const contentHash = createHash("sha256").update(content).digest("hex"); - if (!generatedHash || contentHash !== generatedHash) { + if (contentHash !== generatedHash && content !== (await loadTemplate(fileName))) { return true; } } catch { @@ -404,13 +419,15 @@ async function workspaceAttestedGeneratedFilesIntact( dir: string, generatedHashes: ReadonlyMap, ): Promise { - if ( - !generatedHashes.has(DEFAULT_AGENTS_FILENAME) || - !generatedHashes.has(DEFAULT_TOOLS_FILENAME) - ) { + if (!generatedHashes.has(DEFAULT_AGENTS_FILENAME)) { return false; } for (const [fileName, generatedHash] of generatedHashes) { + // Retiring a generated bootstrap file must not make an attested workspace + // look vanished merely because its historical hash row remains. + if (!GENERATED_WORKSPACE_BOOTSTRAP_FILENAME_SET.has(fileName)) { + continue; + } try { const content = await fs.readFile(path.join(dir, fileName), "utf-8"); const contentHash = createHash("sha256").update(content).digest("hex"); @@ -484,14 +501,7 @@ async function reconcileWorkspaceBootstrapCompletionState(params: { async function collectGeneratedBootstrapHashes(dir: string): Promise> { const hashes = new Map(); - const fileNames = [ - DEFAULT_AGENTS_FILENAME, - DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, - DEFAULT_IDENTITY_FILENAME, - DEFAULT_USER_FILENAME, - ]; - for (const fileName of fileNames) { + for (const fileName of GENERATED_WORKSPACE_BOOTSTRAP_FILENAMES) { try { const content = await fs.readFile(path.join(dir, fileName), "utf-8"); if (content === (await loadTemplate(fileName))) { @@ -599,7 +609,6 @@ async function workspaceSetupStateHasSurvivalEvidence(params: { return [ DEFAULT_AGENTS_FILENAME, DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_IDENTITY_FILENAME, DEFAULT_USER_FILENAME, ].every((fileName) => generatedHashes.has(fileName)); @@ -687,14 +696,13 @@ export async function ensureAgentWorkspace(params?: { /** * List of optional bootstrap filenames to skip writing. * Applies only to SOUL.md, USER.md, IDENTITY.md. - * Required workspace setup such as AGENTS.md and TOOLS.md still runs. + * Required workspace setup such as AGENTS.md still runs. */ skipOptionalBootstrapFiles?: string[]; }): Promise<{ dir: string; agentsPath?: string; soulPath?: string; - toolsPath?: string; identityPath?: string; userPath?: string; bootstrapPath?: string; @@ -750,12 +758,11 @@ export async function ensureAgentWorkspace(params?: { const agentsPath = path.join(dir, DEFAULT_AGENTS_FILENAME); const soulPath = path.join(dir, DEFAULT_SOUL_FILENAME); - const toolsPath = path.join(dir, DEFAULT_TOOLS_FILENAME); const identityPath = path.join(dir, DEFAULT_IDENTITY_FILENAME); const userPath = path.join(dir, DEFAULT_USER_FILENAME); const isBrandNewWorkspace = await (async () => { - const templatePaths = [agentsPath, soulPath, toolsPath, identityPath, userPath]; + const templatePaths = [agentsPath, soulPath, identityPath, userPath]; const paths = [...templatePaths, path.join(dir, "memory")]; const existing = await Promise.all( paths.map(async (p) => { @@ -819,7 +826,6 @@ export async function ensureAgentWorkspace(params?: { const agentsTemplate = await loadTemplate(DEFAULT_AGENTS_FILENAME); const soulTemplate = await loadTemplate(DEFAULT_SOUL_FILENAME); - const toolsTemplate = await loadTemplate(DEFAULT_TOOLS_FILENAME); const identityTemplate = await loadTemplate(DEFAULT_IDENTITY_FILENAME); const userTemplate = await loadTemplate(DEFAULT_USER_FILENAME); // Template and filesystem checks above are async. Another process may have @@ -842,7 +848,6 @@ export async function ensureAgentWorkspace(params?: { if (shouldWriteBootstrapFile(DEFAULT_SOUL_FILENAME)) { await writeFileIfMissing(soulPath, soulTemplate); } - await writeFileIfMissing(toolsPath, toolsTemplate); const identityPathCreated = shouldWriteBootstrapFile(DEFAULT_IDENTITY_FILENAME) ? await writeFileIfMissing(identityPath, identityTemplate) : false; @@ -919,7 +924,6 @@ export async function ensureAgentWorkspace(params?: { dir, agentsPath, soulPath, - toolsPath, identityPath, userPath, bootstrapPath, @@ -943,10 +947,6 @@ export async function loadWorkspaceBootstrapFiles(dir: string): Promise { const stack = [initialRelativeDir === "." ? "" : initialRelativeDir]; while (stack.length > 0) { const currentRelativeDir = stack.pop() ?? ""; const currentDir = path.resolve(workspaceDir, currentRelativeDir); - const relativeToWorkspace = path.relative(workspaceDir, currentDir); - if (relativeToWorkspace.startsWith("..") || path.isAbsolute(relativeToWorkspace)) { + if (!isPathInside(workspaceDir, currentDir)) { continue; } let entries: syncFs.Dirent[]; try { entries = await fs.readdir(currentDir, { withFileTypes: true }); - } catch { + } catch (error) { + if (strictRead && (error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } continue; } @@ -1063,12 +1066,15 @@ async function* walkWorkspaceFiles( const childRelativePath = currentRelativeDir ? path.join(currentRelativeDir, entry.name) : entry.name; + const normalizedChildPath = normalizeWorkspacePatternPath(childRelativePath); if (entry.isDirectory()) { - stack.push(childRelativePath); + if (matcher.match(normalizedChildPath, true)) { + stack.push(childRelativePath); + } continue; } - if (entry.isFile() || entry.isSymbolicLink()) { - yield normalizeWorkspacePatternPath(childRelativePath); + if ((entry.isFile() || entry.isSymbolicLink()) && matcher.match(normalizedChildPath)) { + yield normalizedChildPath; } } } @@ -1077,8 +1083,9 @@ async function* walkWorkspaceFiles( async function resolveExtraBootstrapPatternPaths( workspaceDir: string, pattern: string, + strictRead: boolean, ): Promise { - if (typeof fs.glob === "function") { + if (!strictRead && typeof fs.glob === "function") { try { const matches: string[] = []; for await (const match of fs.glob(pattern, { cwd: workspaceDir })) { @@ -1095,55 +1102,93 @@ async function resolveExtraBootstrapPatternPaths( } const normalizedPattern = normalizeWorkspacePatternPath(pattern); + const matcher = new Minimatch(normalizedPattern, { + nocomment: true, + nonegate: true, + windowsPathsNoEscape: true, + }); const matches: string[] = []; for await (const candidate of walkWorkspaceFiles( workspaceDir, resolveGlobWalkRoot(normalizedPattern), + strictRead, + matcher, )) { - if (path.matchesGlob(candidate, normalizedPattern)) { - matches.push(candidate); - } + matches.push(candidate); } return matches.length > 0 ? matches : [pattern]; } -export async function loadExtraBootstrapFilesWithDiagnostics( +function patternWalkRootStaysInWorkspace(workspaceDir: string, pattern: string): boolean { + const walkRoot = path.resolve(workspaceDir, resolveGlobWalkRoot(pattern)); + return isPathInside(workspaceDir, walkRoot); +} + +export async function loadWorkspacePatternFilesWithDiagnostics( dir: string, extraPatterns: string[], + options: { + acceptedBasenames: ReadonlySet; + acceptedBasenamePrefixes?: readonly string[]; + reportUnsupportedBasenames?: boolean; + strictPatternRead?: boolean; + }, ): Promise<{ - files: WorkspaceBootstrapFile[]; + files: WorkspacePatternFile[]; diagnostics: ExtraBootstrapLoadDiagnostic[]; }> { if (!extraPatterns.length) { return { files: [], diagnostics: [] }; } const resolvedDir = resolveUserPath(dir); - - // Resolve glob patterns into concrete file paths + const diagnostics: ExtraBootstrapLoadDiagnostic[] = []; const resolvedPaths = new Set(); for (const pattern of extraPatterns) { - if (hasGlobPattern(pattern)) { - const matches = await resolveExtraBootstrapPatternPaths(resolvedDir, pattern); - for (const match of matches) { - resolvedPaths.add(match); + if (!patternWalkRootStaysInWorkspace(resolvedDir, pattern)) { + diagnostics.push({ + path: path.resolve(resolvedDir, pattern), + reason: "security", + detail: "pattern resolves outside the workspace", + }); + continue; + } + try { + if (hasGlobPattern(pattern)) { + const matches = await resolveExtraBootstrapPatternPaths( + resolvedDir, + pattern, + options.strictPatternRead === true, + ); + for (const match of matches) { + resolvedPaths.add(match); + } + } else { + resolvedPaths.add(pattern); } - } else { - resolvedPaths.add(pattern); + } catch (error) { + diagnostics.push({ + path: path.resolve(resolvedDir, pattern), + reason: "io", + detail: error instanceof Error ? error.message : String(error), + }); } } - const files: WorkspaceBootstrapFile[] = []; - const diagnostics: ExtraBootstrapLoadDiagnostic[] = []; + const files: WorkspacePatternFile[] = []; for (const relPath of resolvedPaths) { const filePath = path.resolve(resolvedDir, relPath); - // Only load files whose basename is a recognized bootstrap filename const baseName = path.basename(relPath); - if (!VALID_BOOTSTRAP_NAMES.has(baseName)) { - diagnostics.push({ - path: filePath, - reason: "invalid-bootstrap-filename", - detail: `unsupported bootstrap basename: ${baseName}`, - }); + const accepted = + options.acceptedBasenames.has(baseName) || + options.acceptedBasenamePrefixes?.some((prefix) => baseName.startsWith(prefix)) === true; + if (!accepted) { + if (options.reportUnsupportedBasenames !== false) { + diagnostics.push({ + path: filePath, + reason: "invalid-bootstrap-filename", + detail: `unsupported bootstrap basename: ${baseName}`, + }); + } continue; } const loaded = await readWorkspaceFileWithGuards({ @@ -1151,17 +1196,18 @@ export async function loadExtraBootstrapFilesWithDiagnostics( workspaceDir: resolvedDir, }); if (loaded.ok) { - files.push({ - name: baseName as WorkspaceBootstrapFileName, - path: filePath, - content: loaded.content, - missing: false, - }); + files.push({ name: baseName, path: filePath, content: loaded.content }); continue; } + const missing = (loaded.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT"; const reason: ExtraBootstrapLoadDiagnosticCode = - loaded.reason === "path" ? "missing" : loaded.reason === "validation" ? "security" : "io"; + loaded.reason === "validation" || + (options.strictPatternRead === true && loaded.reason === "path" && !missing) + ? "security" + : loaded.reason === "path" + ? "missing" + : "io"; diagnostics.push({ path: filePath, reason, @@ -1175,4 +1221,25 @@ export async function loadExtraBootstrapFilesWithDiagnostics( } return { files, diagnostics }; } + +export async function loadExtraBootstrapFilesWithDiagnostics( + dir: string, + extraPatterns: string[], +): Promise<{ + files: WorkspaceBootstrapFile[]; + diagnostics: ExtraBootstrapLoadDiagnostic[]; +}> { + const loaded = await loadWorkspacePatternFilesWithDiagnostics(dir, extraPatterns, { + acceptedBasenames: VALID_BOOTSTRAP_NAMES, + }); + return { + files: loaded.files.map((file) => ({ + name: file.name as WorkspaceBootstrapFileName, + path: file.path, + content: file.content, + missing: false, + })), + diagnostics: loaded.diagnostics, + }; +} /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/cli/gateway-cli/dev.ts b/src/cli/gateway-cli/dev.ts index ef20ad09231f..db7e970f5f8f 100644 --- a/src/cli/gateway-cli/dev.ts +++ b/src/cli/gateway-cli/dev.ts @@ -65,7 +65,7 @@ async function ensureDevWorkspace(dir: string) { const resolvedDir = resolveUserPath(dir); await fs.promises.mkdir(resolvedDir, { recursive: true }); - const [agents, soul, tools, identity, user] = await Promise.all([ + const [agents, soul, identity, user] = await Promise.all([ loadDevTemplate( "AGENTS.dev.md", `# AGENTS.md - OpenClaw Dev Workspace\n\nDefault dev workspace for openclaw gateway --dev.\n`, @@ -74,10 +74,6 @@ async function ensureDevWorkspace(dir: string) { "SOUL.dev.md", `# SOUL.md - Dev Persona\n\nProtocol droid for debugging and operations.\n`, ), - loadDevTemplate( - "TOOLS.dev.md", - `# TOOLS.md - User Tool Notes (editable)\n\nAdd your local tool notes here.\n`, - ), loadDevTemplate( "IDENTITY.dev.md", `# IDENTITY.md - Agent Identity\n\n- Name: ${DEV_IDENTITY_NAME}\n- Creature: protocol droid\n- Vibe: ${DEV_IDENTITY_THEME}\n- Emoji: ${DEV_IDENTITY_EMOJI}\n`, @@ -90,7 +86,6 @@ async function ensureDevWorkspace(dir: string) { await writeFileIfMissing(path.join(resolvedDir, "AGENTS.md"), agents); await writeFileIfMissing(path.join(resolvedDir, "SOUL.md"), soul); - await writeFileIfMissing(path.join(resolvedDir, "TOOLS.md"), tools); await writeFileIfMissing(path.join(resolvedDir, "IDENTITY.md"), identity); await writeFileIfMissing(path.join(resolvedDir, "USER.md"), user); } diff --git a/src/commands/doctor-tools-md-migration-budget.ts b/src/commands/doctor-tools-md-migration-budget.ts new file mode 100644 index 000000000000..334a2042b2c0 --- /dev/null +++ b/src/commands/doctor-tools-md-migration-budget.ts @@ -0,0 +1,47 @@ +import path from "node:path"; +import { listAgentIds, resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { resolveBootstrapMaxChars } from "../agents/embedded-agent-helpers.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; + +type ToolsMdMigrationWorkspaceTarget = { + primaryAgentId: string; + agentIds: string[]; + workspaceDir: string; +}; + +export function resolveToolsMdMigrationWorkspaceTargets( + cfg: OpenClawConfig, +): ToolsMdMigrationWorkspaceTarget[] { + const targets = new Map(); + for (const agentId of listAgentIds(cfg)) { + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const key = path.resolve(workspaceDir); + const existing = targets.get(key); + if (existing !== undefined) { + existing.agentIds.push(agentId); + } else { + targets.set(key, { primaryAgentId: agentId, agentIds: [agentId], workspaceDir }); + } + } + return [...targets.values()]; +} + +/** One finding per agent whose own bootstrap budget cannot hold the merged file. */ +export function describeToolsMdMergedBootstrapLimits(params: { + cfg: OpenClawConfig; + agentIds: readonly string[]; + mergedChars: number; +}): Array<{ agentId: string; message: string }> { + return params.agentIds.flatMap((agentId) => { + const bootstrapMaxChars = resolveBootstrapMaxChars(params.cfg, agentId); + if (params.mergedChars <= bootstrapMaxChars) { + return []; + } + return [ + { + agentId, + message: `Agent "${agentId}" TOOLS.md migration will produce a ${params.mergedChars}-character AGENTS.md, exceeding its configured bootstrapMaxChars limit of ${bootstrapMaxChars}. Raise \`agents.entries.*.bootstrapMaxChars\` for this agent, or \`agents.defaults.bootstrapMaxChars\` as fallback, to preserve all migrated instructions.`, + }, + ]; + }); +} diff --git a/src/commands/doctor-tools-md-migration-content.ts b/src/commands/doctor-tools-md-migration-content.ts new file mode 100644 index 000000000000..42ca014bd628 --- /dev/null +++ b/src/commands/doctor-tools-md-migration-content.ts @@ -0,0 +1,21 @@ +const LEGACY_TOOLS_MD_TEMPLATE = + "# TOOLS.md - Local Notes\n\nSkills define _how_ tools work. This file is for _your_ specifics — the stuff that's unique to your setup: camera names and locations, SSH hosts and aliases, preferred TTS voices, speaker/room names, device nicknames, anything environment-specific.\n\n## Examples\n\n```markdown\n### Cameras\n\n" + + "- living-room → Main area, 180° wide angle\n- front-door → Entrance, motion-triggered\n\n### SSH\n\n- home-server → 192.168.1.100, user: admin\n\n### TTS\n\n" + + '- Preferred voice: "Nova" (warm, slightly British)\n- Default speaker: Kitchen HomePod\n```\n\n## Why Separate?\n\n' + + "Skills are shared. Your setup is yours. Keeping them apart means you can update skills without losing your notes, and share skills without leaking your infrastructure.\n\n---\n\nAdd whatever helps you do your job. This is your cheat sheet.\n\n## Related\n\n- [Agent workspace](/concepts/agent-workspace)\n"; + +const LEGACY_TOOLS_DEV_MD_TEMPLATE = + "# TOOLS.md - User Tool Notes (editable)\n\nThis file is for _your_ notes about external tools and conventions. It does not define which tools exist; OpenClaw provides built-in tools internally, and skills add the rest.\n\n## Examples\n\n### imsg\n\n" + + "- Send an iMessage/SMS: describe who/what, confirm before sending.\n- Prefer short messages; avoid sending secrets.\n\n### sag\n\n" + + "- Text-to-speech: specify voice, target speaker/room, and whether to stream.\n\nAdd whatever else you want the assistant to know about your local toolchain.\n\n## Related\n\n- [TOOLS.md template](/reference/templates/TOOLS)\n"; +const LEGACY_TOOLS_DEV_FALLBACK = + "# TOOLS.md - User Tool Notes (editable)\n\nAdd your local tool notes here.\n"; + +export function shouldMergeToolsMd(content: string): boolean { + return ( + content.trim().length > 0 && + content !== LEGACY_TOOLS_MD_TEMPLATE && + content !== LEGACY_TOOLS_DEV_MD_TEMPLATE && + content !== LEGACY_TOOLS_DEV_FALLBACK + ); +} diff --git a/src/commands/doctor-tools-md-migration-guidance.ts b/src/commands/doctor-tools-md-migration-guidance.ts new file mode 100644 index 000000000000..188ff401530c --- /dev/null +++ b/src/commands/doctor-tools-md-migration-guidance.ts @@ -0,0 +1,61 @@ +const CURRENT_AGENTS_TOOLS_GUIDANCE = + "Skills define how tools work. Keep environment-specific local notes in this section."; + +const LEGACY_AGENTS_TOOLS_GUIDANCE_REWRITES = [ + [ + "Skills provide your tools. When you need one, check its `SKILL.md`. Keep local notes (camera names, SSH details, voice preferences) in `TOOLS.md`.", + CURRENT_AGENTS_TOOLS_GUIDANCE, + ], + [ + "- Keep environment-specific notes in `TOOLS.md` (notes for skills).", + "- Keep environment-specific notes in this file's `## Tools` section.", + ], + [ + "- Keep environment-specific notes in `TOOLS.md` (Notes for Skills).", + "- Keep environment-specific notes in this file's `## Tools` section.", + ], + [ + "- You learn a lesson -> update `AGENTS.md`, `TOOLS.md`, or the relevant skill.", + "- You learn a lesson -> update `AGENTS.md` or the relevant skill.", + ], + [ + "- When you learn a lesson → update AGENTS.md, TOOLS.md, or the relevant skill", + "- When you learn a lesson → update AGENTS.md or the relevant skill", + ], + [ + "Skills 提供你的工具。当你需要某个工具时，查看它的 `SKILL.md`。在 `TOOLS.md` 中保存本地笔记（摄像头名称、SSH 详情、语音偏好等）。", + "Skills 定义工具的使用方式。请将环境特定的本地笔记保存在本节中。", + ], + [ + "技能提供你的工具。当你需要某个工具时，查看它的 `SKILL.md`。在 `TOOLS.md` 中保存本地笔记（摄像头名称、SSH 详情、语音偏好等）。", + "技能定义工具的使用方式。请将环境特定的本地笔记保存在本节中。", + ], + [ + "- 在 `TOOLS.md` 中保存环境特定的笔记（Skills 注意事项）。", + "- 将环境特定的笔记保存在本文件的 `## Tools` 部分。", + ], + [ + "- 将环境相关的备注保存在 `TOOLS.md`（Skills 备注）中。", + "- 将环境相关的备注保存在本文件的 `## Tools` 部分。", + ], + [ + "- 将环境相关的备注保存在 `TOOLS.md`（技能备注）中。", + "- 将环境相关的备注保存在本文件的 `## Tools` 部分。", + ], + [ + "- 当你学到教训 → 更新 AGENTS.md、TOOLS.md 或相关 Skills 文件", + "- 当你学到教训 → 更新 AGENTS.md 或相关 Skills 文件", + ], + [ + "- 当你学到教训 → 更新 AGENTS.md、TOOLS.md 或相关技能文件", + "- 当你学到教训 → 更新 AGENTS.md 或相关技能文件", + ], +] as const; + +export function rewriteLegacyAgentsToolsGuidance(content: string): string { + let rewritten = content; + for (const [legacy, current] of LEGACY_AGENTS_TOOLS_GUIDANCE_REWRITES) { + rewritten = rewritten.replaceAll(legacy, current); + } + return rewritten; +} diff --git a/src/commands/doctor-tools-md-migration.test.ts b/src/commands/doctor-tools-md-migration.test.ts new file mode 100644 index 000000000000..7f6a1af34056 --- /dev/null +++ b/src/commands/doctor-tools-md-migration.test.ts @@ -0,0 +1,491 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; + +vi.mock("../../packages/terminal-core/src/note.js", () => ({ note: vi.fn() })); + +import { note } from "../../packages/terminal-core/src/note.js"; +import { + collectToolsMdMigrationFindings, + maybeMigrateToolsMd, +} from "./doctor-tools-md-migration.js"; + +const noteMock = vi.mocked(note); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const LEGACY_TOOLS_MD_TEMPLATE_FIXTURE = + [ + "# TOOLS.md - Local Notes", + "", + "Skills define _how_ tools work. This file is for _your_ specifics — the stuff that's unique to your setup: camera names and locations, SSH hosts and aliases, preferred TTS voices, speaker/room names, device nicknames, anything environment-specific.", + "", + "## Examples", + "", + "```markdown", + "### Cameras", + "", + "- living-room → Main area, 180° wide angle", + "- front-door → Entrance, motion-triggered", + "", + "### SSH", + "", + "- home-server → 192.168.1.100, user: admin", + "", + "### TTS", + "", + '- Preferred voice: "Nova" (warm, slightly British)', + "- Default speaker: Kitchen HomePod", + "```", + "", + "## Why Separate?", + "", + "Skills are shared. Your setup is yours. Keeping them apart means you can update skills without losing your notes, and share skills without leaking your infrastructure.", + "", + "---", + "", + "Add whatever helps you do your job. This is your cheat sheet.", + "", + "## Related", + "", + "- [Agent workspace](/concepts/agent-workspace)", + ].join("\n") + "\n"; +const LEGACY_AGENTS_GUIDANCE_REWRITES = [ + [ + "generic workspace template", + "Skills provide your tools. When you need one, check its `SKILL.md`. Keep local notes (camera names, SSH details, voice preferences) in `TOOLS.md`.", + "Skills define how tools work. Keep environment-specific local notes in this section.", + ], + [ + "default template lowercase parenthetical", + "- Keep environment-specific notes in `TOOLS.md` (notes for skills).", + "- Keep environment-specific notes in this file's `## Tools` section.", + ], + [ + "default template title-case parenthetical", + "- Keep environment-specific notes in `TOOLS.md` (Notes for Skills).", + "- Keep environment-specific notes in this file's `## Tools` section.", + ], + [ + "current lesson reminder", + "- You learn a lesson -> update `AGENTS.md`, `TOOLS.md`, or the relevant skill.", + "- You learn a lesson -> update `AGENTS.md` or the relevant skill.", + ], + [ + "original lesson reminder", + "- When you learn a lesson → update AGENTS.md, TOOLS.md, or the relevant skill", + "- When you learn a lesson → update AGENTS.md or the relevant skill", + ], + [ + "Chinese generic template with English Skills", + "Skills 提供你的工具。当你需要某个工具时，查看它的 `SKILL.md`。在 `TOOLS.md` 中保存本地笔记（摄像头名称、SSH 详情、语音偏好等）。", + "Skills 定义工具的使用方式。请将环境特定的本地笔记保存在本节中。", + ], + [ + "Chinese generic template", + "技能提供你的工具。当你需要某个工具时，查看它的 `SKILL.md`。在 `TOOLS.md` 中保存本地笔记（摄像头名称、SSH 详情、语音偏好等）。", + "技能定义工具的使用方式。请将环境特定的本地笔记保存在本节中。", + ], + [ + "Chinese default template", + "- 在 `TOOLS.md` 中保存环境特定的笔记（Skills 注意事项）。", + "- 将环境特定的笔记保存在本文件的 `## Tools` 部分。", + ], + [ + "Chinese default template with Skills", + "- 将环境相关的备注保存在 `TOOLS.md`（Skills 备注）中。", + "- 将环境相关的备注保存在本文件的 `## Tools` 部分。", + ], + [ + "Chinese default template with translated skills", + "- 将环境相关的备注保存在 `TOOLS.md`（技能备注）中。", + "- 将环境相关的备注保存在本文件的 `## Tools` 部分。", + ], + [ + "Chinese lesson reminder with Skills", + "- 当你学到教训 → 更新 AGENTS.md、TOOLS.md 或相关 Skills 文件", + "- 当你学到教训 → 更新 AGENTS.md 或相关 Skills 文件", + ], + [ + "Chinese lesson reminder with translated skills", + "- 当你学到教训 → 更新 AGENTS.md、TOOLS.md 或相关技能文件", + "- 当你学到教训 → 更新 AGENTS.md 或相关技能文件", + ], +] as const; + +afterEach(() => { + noteMock.mockReset(); +}); + +async function createFixture() { + const root = await fs.realpath(tempDirs.make("openclaw-tools-md-migration-")); + const stateDir = path.join(root, "state"); + const workspace = path.join(root, "workspace"); + await fs.mkdir(workspace, { recursive: true }); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace }] }, + } as OpenClawConfig; + return { + root, + stateDir, + workspace, + cfg, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + agentsPath: path.join(workspace, "AGENTS.md"), + toolsPath: path.join(workspace, "TOOLS.md"), + }; +} + +async function readOnlyArchive(stateDir: string): Promise { + const archiveDir = path.join(stateDir, "backups", "tools-md-migration"); + const archives = await fs.readdir(archiveDir); + expect(archives).toHaveLength(1); + return fs.readFile(path.join(archiveDir, archives[0]!)); +} + +async function expectMissing(filePath: string): Promise { + await expect(fs.access(filePath)).rejects.toMatchObject({ code: "ENOENT" }); +} + +describe("TOOLS.md migration", () => { + it("previews the migration without mutating or archiving workspace files", async () => { + const fixture = await createFixture(); + const agents = "# Agent\n\n## Tools\n\nExisting notes.\n"; + const tools = "### Cameras\n\n- kitchen → wide angle\n"; + await fs.writeFile(fixture.agentsPath, agents); + await fs.writeFile(fixture.toolsPath, tools); + + await expect(collectToolsMdMigrationFindings(fixture.cfg)).resolves.toEqual([ + expect.objectContaining({ + checkId: "core/doctor/tools-md-migration", + requirement: "legacy-tools-md", + }), + ]); + + await expect( + maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: false, + env: fixture.env, + }), + ).resolves.toEqual({ changes: [], warnings: [] }); + + await expect(fs.readFile(fixture.agentsPath, "utf8")).resolves.toBe(agents); + await expect(fs.readFile(fixture.toolsPath, "utf8")).resolves.toBe(tools); + await expectMissing(fixture.stateDir); + expect(noteMock).toHaveBeenCalledWith( + expect.stringContaining("will be archived and merged into AGENTS.md when customized"), + "TOOLS.md migration preview", + ); + }); + + it("appends customized content verbatim under the existing Tools section and is idempotent", async () => { + const fixture = await createFixture(); + const agents = "# Agent\n\n## Tools\n\nExisting notes.\n\n## Safety\n\nBe careful.\n"; + const tools = "### Cameras\n\n- kitchen → wide angle\n\nKeep trailing spaces. \n"; + const expected = + "# Agent\n\n## Tools\n\nExisting notes.\n\n" + + "### Local notes (migrated from TOOLS.md)\n\n" + + tools + + "\n## Safety\n\nBe careful.\n"; + await fs.writeFile(fixture.agentsPath, agents); + await fs.writeFile(fixture.toolsPath, tools); + + const result = await maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toHaveLength(1); + await expect(fs.readFile(fixture.agentsPath, "utf8")).resolves.toBe(expected); + await expect(readOnlyArchive(fixture.stateDir)).resolves.toEqual(Buffer.from(tools)); + await expectMissing(fixture.toolsPath); + + await expect( + maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }), + ).resolves.toEqual({ changes: [], warnings: [] }); + const rerunAgents = await fs.readFile(fixture.agentsPath, "utf8"); + expect(rerunAgents).toBe(expected); + expect(rerunAgents.match(/migrated from TOOLS\.md/gu)).toHaveLength(1); + }); + + it("warns when the merged AGENTS.md will exceed the agent bootstrap file limit", async () => { + const fixture = await createFixture(); + const agents = `# Agent\n\n## Tools\n\n${"a".repeat(15_000)}\n`; + const tools = `${"b".repeat(15_000)}\n`; + const merged = `${agents}\n### Local notes (migrated from TOOLS.md)\n\n${tools}`; + fixture.cfg.agents!.list![0]!.bootstrapMaxChars = 20_000; + await fs.writeFile(fixture.agentsPath, agents); + await fs.writeFile(fixture.toolsPath, tools); + + const findings = await collectToolsMdMigrationFindings(fixture.cfg); + + expect(findings).toContainEqual( + expect.objectContaining({ + checkId: "core/doctor/tools-md-migration", + requirement: "tools-md-merged-bootstrap-limit", + message: `Agent "main" TOOLS.md migration will produce a ${merged.length}-character AGENTS.md, exceeding its configured bootstrapMaxChars limit of 20000. Raise \`agents.entries.*.bootstrapMaxChars\` for this agent, or \`agents.defaults.bootstrapMaxChars\` as fallback, to preserve all migrated instructions.`, + }), + ); + }); + + it("checks every agent budget while migrating a shared workspace once", async () => { + const fixture = await createFixture(); + const agents = `# Agent\n\n## Tools\n\n${"a".repeat(15_000)}\n`; + const tools = `${"b".repeat(15_000)}\n`; + const merged = `${agents}\n### Local notes (migrated from TOOLS.md)\n\n${tools}`; + fixture.cfg.agents!.list = [ + { id: "main", default: true, workspace: fixture.workspace, bootstrapMaxChars: 40_000 }, + { id: "limited", workspace: fixture.workspace, bootstrapMaxChars: 20_000 }, + ]; + await fs.writeFile(fixture.agentsPath, agents); + await fs.writeFile(fixture.toolsPath, tools); + + const findings = await collectToolsMdMigrationFindings(fixture.cfg); + + expect( + findings.filter((finding) => finding.requirement === "tools-md-merged-bootstrap-limit"), + ).toEqual([ + expect.objectContaining({ + target: "limited", + message: `Agent "limited" TOOLS.md migration will produce a ${merged.length}-character AGENTS.md, exceeding its configured bootstrapMaxChars limit of 20000. Raise \`agents.entries.*.bootstrapMaxChars\` for this agent, or \`agents.defaults.bootstrapMaxChars\` as fallback, to preserve all migrated instructions.`, + }), + ]); + + const result = await maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }); + + expect(result).toMatchObject({ changes: [expect.any(String)], warnings: [] }); + await expect(fs.readFile(fixture.agentsPath, "utf8")).resolves.toBe(merged); + await expect(readOnlyArchive(fixture.stateDir)).resolves.toEqual(Buffer.from(tools)); + await expectMissing(fixture.toolsPath); + }); + + it("does not emit a bootstrap limit finding for a normal-sized merge", async () => { + const fixture = await createFixture(); + await fs.writeFile(fixture.agentsPath, "# Agent\n\n## Tools\n\nExisting notes.\n"); + await fs.writeFile(fixture.toolsPath, "Local camera: kitchen\n"); + + const findings = await collectToolsMdMigrationFindings(fixture.cfg); + + expect(findings).not.toContainEqual( + expect.objectContaining({ requirement: "tools-md-merged-bootstrap-limit" }), + ); + }); + + it.each(LEGACY_AGENTS_GUIDANCE_REWRITES)( + "rewrites the %s without leaving a TOOLS.md directive", + async (_label, legacy, current) => { + const fixture = await createFixture(); + await fs.writeFile(fixture.agentsPath, `# Agent\n\n## Tools\n\n${legacy}\n`); + await fs.writeFile(fixture.toolsPath, LEGACY_TOOLS_MD_TEMPLATE_FIXTURE); + + const result = await maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }); + + expect(result.warnings).toEqual([]); + const migratedAgents = await fs.readFile(fixture.agentsPath, "utf8"); + expect(migratedAgents).toContain(current); + expect(migratedAgents).not.toContain(legacy); + expect(migratedAgents).not.toContain("TOOLS.md"); + await expectMissing(fixture.toolsPath); + }, + ); + + it.runIf(process.platform !== "win32")( + "refuses symlinked AGENTS.md files and interrupted claims", + async () => { + const linkedFixture = await createFixture(); + const linkedTarget = path.join(linkedFixture.root, "outside-agents.md"); + await fs.writeFile(linkedTarget, "Private external instructions.\n"); + await fs.writeFile(linkedFixture.toolsPath, "Local tool notes.\n"); + await fs.symlink(linkedTarget, linkedFixture.agentsPath); + + const linkedResult = await maybeMigrateToolsMd({ + cfg: linkedFixture.cfg, + shouldRepair: true, + env: linkedFixture.env, + }); + + expect(linkedResult.changes).toEqual([]); + expect(linkedResult.warnings).toEqual([ + expect.stringContaining("AGENTS.md must be an unlinked regular file"), + ]); + await expect(fs.readFile(linkedFixture.toolsPath, "utf8")).resolves.toBe( + "Local tool notes.\n", + ); + await expect(fs.readFile(linkedTarget, "utf8")).resolves.toBe( + "Private external instructions.\n", + ); + + const claimFixture = await createFixture(); + const claimTarget = path.join(claimFixture.root, "outside-claim.md"); + const claimPath = `${claimFixture.agentsPath}.doctor-backup-999999-${Date.now() - 60_000}`; + await fs.writeFile(claimTarget, "Untrusted claim content.\n"); + await fs.writeFile(claimFixture.toolsPath, "Local tool notes.\n"); + await fs.symlink(claimTarget, claimPath); + + const claimResult = await maybeMigrateToolsMd({ + cfg: claimFixture.cfg, + shouldRepair: true, + env: claimFixture.env, + }); + + expect(claimResult.changes).toEqual([]); + expect(claimResult.warnings).toEqual([ + expect.stringContaining("AGENTS.md migration claim must be an unlinked regular file"), + ]); + await expect(fs.readFile(claimFixture.toolsPath, "utf8")).resolves.toBe( + "Local tool notes.\n", + ); + await expect(fs.readFile(claimTarget, "utf8")).resolves.toBe("Untrusted claim content.\n"); + await expectMissing(claimFixture.agentsPath); + }, + ); + + it("keeps live claims created from old source files fresh", async () => { + const ownerPid = process.ppid; + const oldTimestamp = new Date("2000-01-01T00:00:00.000Z"); + + const toolsFixture = await createFixture(); + await fs.writeFile(toolsFixture.toolsPath, "old tool notes\n"); + await fs.utimes(toolsFixture.toolsPath, oldTimestamp, oldTimestamp); + const toolsClaimPath = `${toolsFixture.toolsPath}.doctor-importing-${ownerPid}-${Date.now()}-claim`; + await fs.rename(toolsFixture.toolsPath, toolsClaimPath); + + const toolsResult = await maybeMigrateToolsMd({ + cfg: toolsFixture.cfg, + shouldRepair: true, + env: toolsFixture.env, + }); + + expect(toolsResult.warnings).toEqual([ + expect.stringContaining(`TOOLS.md migration claim is held by running process ${ownerPid}`), + ]); + await expect(fs.stat(toolsClaimPath)).resolves.toMatchObject({ + mtimeMs: oldTimestamp.getTime(), + }); + + const agentsFixture = await createFixture(); + await fs.writeFile(agentsFixture.toolsPath, "old tool notes\n"); + await fs.writeFile(agentsFixture.agentsPath, "# Agent\n"); + await fs.utimes(agentsFixture.agentsPath, oldTimestamp, oldTimestamp); + const agentsClaimPath = `${agentsFixture.agentsPath}.doctor-backup-${ownerPid}-${Date.now()}`; + await fs.rename(agentsFixture.agentsPath, agentsClaimPath); + + const agentsResult = await maybeMigrateToolsMd({ + cfg: agentsFixture.cfg, + shouldRepair: true, + env: agentsFixture.env, + }); + + expect(agentsResult.warnings).toEqual([ + expect.stringContaining(`AGENTS.md migration claim is held by running process ${ownerPid}`), + ]); + await expect(fs.readFile(agentsFixture.toolsPath, "utf8")).resolves.toBe("old tool notes\n"); + await expect(fs.stat(agentsClaimPath)).resolves.toMatchObject({ + mtimeMs: oldTimestamp.getTime(), + }); + }); + + it("recovers an AGENTS.md publish interrupted after the hard link", async () => { + const fixture = await createFixture(); + const agents = "# Agent\n\n## Tools\n\nExisting notes.\n"; + const tools = "### Cameras\n\n- kitchen → wide angle\n"; + const merged = `${agents}\n### Local notes (migrated from TOOLS.md)\n\n${tools}`; + const interruptedAt = Date.now() - 60_000; + const toolsClaimPath = `${fixture.toolsPath}.doctor-importing-999999-${interruptedAt}-claim`; + const agentsClaimPath = `${fixture.agentsPath}.doctor-backup-999999-${interruptedAt}`; + const tempPath = `${fixture.agentsPath}.doctor-writing-999999-${interruptedAt}`; + await fs.writeFile(toolsClaimPath, tools); + await fs.writeFile(agentsClaimPath, agents); + await fs.writeFile(tempPath, merged); + await fs.link(tempPath, fixture.agentsPath); + await expect(fs.stat(fixture.agentsPath)).resolves.toMatchObject({ nlink: 2 }); + + const result = await maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toHaveLength(1); + await expect(fs.readFile(fixture.agentsPath, "utf8")).resolves.toBe(merged); + await expect(fs.stat(fixture.agentsPath)).resolves.toMatchObject({ nlink: 1 }); + await expect(fs.readdir(fixture.workspace)).resolves.toEqual(["AGENTS.md"]); + }); + + it("appends a Tools section when AGENTS.md has no Tools heading", async () => { + const fixture = await createFixture(); + const agents = "# Agent\n\nKeep safe."; + const tools = "Local camera: kitchen → wide angle\n"; + await fs.writeFile(fixture.agentsPath, agents); + await fs.writeFile(fixture.toolsPath, tools); + + await maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }); + + await expect(fs.readFile(fixture.agentsPath, "utf8")).resolves.toBe( + `${agents}\n\n## Tools\n\n### Local notes (migrated from TOOLS.md)\n\n${tools}`, + ); + await expectMissing(fixture.toolsPath); + }); + + it.each([ + ["untouched template", LEGACY_TOOLS_MD_TEMPLATE_FIXTURE], + ["empty file", ""], + ["whitespace-only file", " \n\t"], + ])("deletes the %s without appending content", async (_label, tools) => { + const fixture = await createFixture(); + const agents = "# Agent\n\n## Tools\n\nLocal details stay unchanged.\n"; + await fs.writeFile(fixture.agentsPath, agents); + await fs.writeFile(fixture.toolsPath, tools); + + const result = await maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }); + + expect(result.warnings).toEqual([]); + await expect(fs.readFile(fixture.agentsPath, "utf8")).resolves.toBe(agents); + await expect(readOnlyArchive(fixture.stateDir)).resolves.toEqual(Buffer.from(tools)); + await expectMissing(fixture.toolsPath); + }); + + it("keeps TOOLS.md untouched when the original cannot be archived", async () => { + const fixture = await createFixture(); + const agents = "# Agent\n\n## Tools\n\nExisting notes.\n"; + const tools = "Local camera: kitchen → wide angle\n"; + await fs.writeFile(fixture.agentsPath, agents); + await fs.writeFile(fixture.toolsPath, tools); + await fs.writeFile(fixture.stateDir, "not a directory"); + + const result = await maybeMigrateToolsMd({ + cfg: fixture.cfg, + shouldRepair: true, + env: fixture.env, + }); + + expect(result.changes).toEqual([]); + expect(result.warnings).toHaveLength(1); + await expect(fs.readFile(fixture.agentsPath, "utf8")).resolves.toBe(agents); + await expect(fs.readFile(fixture.toolsPath, "utf8")).resolves.toBe(tools); + }); +}); diff --git a/src/commands/doctor-tools-md-migration.ts b/src/commands/doctor-tools-md-migration.ts new file mode 100644 index 000000000000..32883c7f62c8 --- /dev/null +++ b/src/commands/doctor-tools-md-migration.ts @@ -0,0 +1,721 @@ +/** Doctor-owned migration from workspace TOOLS.md into the AGENTS.md Tools section. */ +import { createHash } from "node:crypto"; +import syncFs from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { note } from "../../packages/terminal-core/src/note.js"; +import { DEFAULT_AGENTS_FILENAME, DEFAULT_TOOLS_FILENAME } from "../agents/workspace.js"; +import { formatCliCommand } from "../cli/command-format.js"; +import { resolveStateDir } from "../config/paths.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { HealthFinding } from "../flows/health-checks.js"; +import { formatErrorMessage as errorMessage } from "../infra/errors.js"; +import { shortenHomePath } from "../utils.js"; +import { + describeToolsMdMergedBootstrapLimits, + resolveToolsMdMigrationWorkspaceTargets, +} from "./doctor-tools-md-migration-budget.js"; +import { shouldMergeToolsMd } from "./doctor-tools-md-migration-content.js"; +import { rewriteLegacyAgentsToolsGuidance as rewriteLegacyToolsGuidance } from "./doctor-tools-md-migration-guidance.js"; + +const TOOLS_MD_MIGRATION_CHECK_ID = "core/doctor/tools-md-migration"; +const MIGRATED_SUBSECTION_HEADING = "### Local notes (migrated from TOOLS.md)"; +const TOOLS_CLAIM_INFIX = ".doctor-importing-"; +const ACTIVE_CLAIM_MAX_AGE_MS = 10 * 60 * 1000; +const HARD_LINK_UNSUPPORTED_CODES = new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "EXDEV"]); + +type ToolsMdMigrationResult = { + changes: string[]; + warnings: string[]; +}; + +type ToolsMdSource = { + path: string; + content: string; + sha256: string; +}; + +type MigrationClaimIdentity = { + ownerPid: number; + createdAtMs: number; +}; + +type MigrationFileSnapshot = { + content: string; + stat?: syncFs.Stats; +}; + +function sha256(content: string): string { + return createHash("sha256").update(content).digest("hex"); +} + +async function readMigrationFileSnapshot(params: { + filePath: string; + label: string; + allowMissing?: boolean; +}): Promise { + let stat: syncFs.Stats; + try { + stat = await fs.lstat(params.filePath); + } catch (error) { + if (params.allowMissing && (error as NodeJS.ErrnoException).code === "ENOENT") { + return { content: "" }; + } + throw error; + } + if (!stat.isFile() || stat.nlink > 1) { + throw new Error(`${params.label} must be an unlinked regular file for automatic migration`); + } + const noFollow = syncFs.constants.O_NOFOLLOW ?? 0; + const handle = await fs.open(params.filePath, syncFs.constants.O_RDONLY | noFollow); + try { + const openedStat = await handle.stat(); + if ( + !openedStat.isFile() || + openedStat.nlink !== 1 || + openedStat.dev !== stat.dev || + openedStat.ino !== stat.ino + ) { + throw new Error(`${params.label} changed while opening it for migration`); + } + const content = await handle.readFile("utf8"); + const currentStat = await fs.lstat(params.filePath); + if (currentStat.dev !== openedStat.dev || currentStat.ino !== openedStat.ino) { + throw new Error(`${params.label} changed while opening it for migration`); + } + return { content, stat: openedStat }; + } finally { + await handle.close(); + } +} + +function parseMigrationClaimIdentity( + claimName: string, + prefix: string, +): MigrationClaimIdentity | undefined { + const [ownerPidText, createdAtMsText] = claimName.slice(prefix.length).split("-"); + const ownerPid = Number(ownerPidText); + const createdAtMs = Number(createdAtMsText); + if (!Number.isSafeInteger(ownerPid) || !Number.isSafeInteger(createdAtMs) || createdAtMs <= 0) { + return undefined; + } + return { ownerPid, createdAtMs }; +} + +async function readToolsMd( + workspaceDir: string, + options?: { recoverClaims?: boolean }, +): Promise { + const toolsPath = path.join(workspaceDir, DEFAULT_TOOLS_FILENAME); + let stat; + try { + stat = await fs.lstat(toolsPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + const entries = await fs.readdir(workspaceDir).catch(() => [] as string[]); + const claims = entries.filter((entry) => + entry.startsWith(`${DEFAULT_TOOLS_FILENAME}${TOOLS_CLAIM_INFIX}`), + ); + if (claims.length === 0) { + return undefined; + } + if (claims.length > 1) { + throw new Error("multiple interrupted TOOLS.md migration claims require manual recovery", { + cause: error, + }); + } + const claimPath = path.join(workspaceDir, claims[0]!); + const claimIdentity = parseMigrationClaimIdentity( + claims[0]!, + `${DEFAULT_TOOLS_FILENAME}${TOOLS_CLAIM_INFIX}`, + ); + if ( + claimIdentity && + claimIdentity.ownerPid !== process.pid && + Date.now() - claimIdentity.createdAtMs < ACTIVE_CLAIM_MAX_AGE_MS && + isProcessAlive(claimIdentity.ownerPid) + ) { + throw new Error( + `TOOLS.md migration claim is held by running process ${claimIdentity.ownerPid}`, + { cause: error }, + ); + } + if (!options?.recoverClaims) { + throw new Error("an interrupted TOOLS.md migration claim requires doctor --fix", { + cause: error, + }); + } + await restoreClaimNoClobber(claimPath, toolsPath); + stat = await fs.lstat(toolsPath); + } else { + throw error; + } + } + if (!stat.isFile()) { + throw new Error("TOOLS.md must be a regular file"); + } + if (stat.nlink > 1) { + if (!options?.recoverClaims) { + throw new Error("an interrupted TOOLS.md migration restoration requires doctor --fix"); + } + const entries = await fs.readdir(workspaceDir); + const claims = entries.filter((entry) => + entry.startsWith(`${DEFAULT_TOOLS_FILENAME}${TOOLS_CLAIM_INFIX}`), + ); + if (claims.length === 1) { + const claimPath = path.join(workspaceDir, claims[0]!); + const claimStat = await fs.lstat(claimPath); + if (claimStat.dev === stat.dev && claimStat.ino === stat.ino && stat.nlink === 2) { + await fs.rm(claimPath); + stat = await fs.lstat(toolsPath); + } + } + if (stat.nlink > 1) { + throw new Error("TOOLS.md has multiple hard links; refusing automatic removal"); + } + } + const noFollow = syncFs.constants.O_NOFOLLOW ?? 0; + const handle = await fs.open(toolsPath, syncFs.constants.O_RDONLY | noFollow); + let content: string; + try { + const openedStat = await handle.stat(); + if (!openedStat.isFile() || openedStat.nlink !== stat.nlink) { + throw new Error("TOOLS.md changed while opening it for migration"); + } + content = await handle.readFile("utf8"); + const currentStat = await fs.lstat(toolsPath); + if (currentStat.dev !== openedStat.dev || currentStat.ino !== openedStat.ino) { + throw new Error("TOOLS.md changed while opening it for migration"); + } + } finally { + await handle.close(); + } + return { path: toolsPath, content, sha256: sha256(content) }; +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function migratedBlock(content: string): string { + return `${MIGRATED_SUBSECTION_HEADING}\n\n${content}`; +} + +function appendWithSpacing(before: string, addition: string, after = ""): string { + const prefix = + before.length === 0 ? "" : before.endsWith("\n\n") ? "" : before.endsWith("\n") ? "\n" : "\n\n"; + const suffix = + after.length === 0 + ? "" + : addition.endsWith("\n\n") + ? "" + : addition.endsWith("\n") + ? "\n" + : "\n\n"; + return `${before}${prefix}${addition}${suffix}${after}`; +} + +function mergeToolsMdIntoAgentsMd(agentsContent: string, toolsContent: string): string { + const mergedAgentsContent = rewriteLegacyAgentsToolsGuidance(agentsContent); + if (mergedAgentsContent.includes(MIGRATED_SUBSECTION_HEADING)) { + if (mergedAgentsContent.includes(toolsContent)) { + return mergedAgentsContent; + } + const headingIndex = mergedAgentsContent.indexOf(MIGRATED_SUBSECTION_HEADING); + const insertAt = headingIndex + MIGRATED_SUBSECTION_HEADING.length; + return appendWithSpacing( + mergedAgentsContent.slice(0, insertAt), + toolsContent, + mergedAgentsContent.slice(insertAt), + ); + } + const block = migratedBlock(toolsContent); + const toolsSection = findToolsSection(mergedAgentsContent); + if (!toolsSection) { + return appendWithSpacing(mergedAgentsContent, `## Tools\n\n${block}`); + } + const insertAt = toolsSection.insertAt; + return appendWithSpacing( + mergedAgentsContent.slice(0, insertAt), + block, + mergedAgentsContent.slice(insertAt), + ); +} + +function rewriteLegacyAgentsToolsGuidance(content: string): string { + const rewritten = rewriteLegacyToolsGuidance(content); + return rewritten === content ? content : ensureLocalNotesHeading(rewritten); +} + +function findToolsSection(content: string): { headingEnd: number; insertAt: number } | undefined { + let offset = 0; + let insideTools = false; + let headingEnd = 0; + let fence: { marker: "`" | "~"; length: number } | undefined; + for (const lineWithEnding of content.match(/.*(?:\n|$)/gu) ?? []) { + if (lineWithEnding === "") { + continue; + } + const line = lineWithEnding.replace(/\n$/u, ""); + const fenceRun = /^\s*(`{3,}|~{3,})/u.exec(line)?.[1]; + const closingFenceRun = /^\s*(`{3,}|~{3,})\s*$/u.exec(line)?.[1]; + const marker = fenceRun?.[0] as "`" | "~" | undefined; + if (marker && !fence) { + fence = { marker, length: fenceRun!.length }; + } else if ( + closingFenceRun && + fence && + closingFenceRun[0] === fence.marker && + closingFenceRun.length >= fence.length + ) { + fence = undefined; + } else if (!fence) { + const heading = /^(#{1,6})\s+(.+?)\s*#*\s*$/u.exec(line); + if (heading) { + const depth = heading[1]!.length; + if (insideTools && depth <= 2) { + return { headingEnd, insertAt: offset }; + } + if (depth === 2 && heading[2]!.trim().toLowerCase() === "tools") { + insideTools = true; + headingEnd = offset + lineWithEnding.length; + } + } + } + offset += lineWithEnding.length; + } + return insideTools ? { headingEnd, insertAt: content.length } : undefined; +} + +function ensureLocalNotesHeading(content: string): string { + const section = findToolsSection(content); + if (!section) { + return content; + } + const body = content.slice(section.headingEnd, section.insertAt); + if (/^###\s+Local notes(?:\s|$)/imu.test(body)) { + return content; + } + return `${content.slice(0, section.headingEnd)}\n### Local notes\n${content.slice(section.headingEnd)}`; +} + +async function writeAgentsAtomically(params: { + agentsPath: string; + expected: string; + content: string; +}): Promise { + const snapshot = await readMigrationFileSnapshot({ + filePath: params.agentsPath, + label: "AGENTS.md", + allowMissing: true, + }); + if (snapshot.content !== params.expected) { + throw new Error("AGENTS.md changed during TOOLS.md migration"); + } + const stat = snapshot.stat; + const mode = stat?.mode ?? 0o600; + const tempPath = `${params.agentsPath}.doctor-writing-${process.pid}-${Date.now()}`; + const handle = await fs.open(tempPath, "wx", mode); + try { + await handle.writeFile(params.content, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + const backupPath = `${params.agentsPath}.doctor-backup-${process.pid}-${Date.now()}`; + let claimed = false; + try { + if (stat) { + const currentStat = await fs.lstat(params.agentsPath); + if ( + currentStat.dev !== stat.dev || + currentStat.ino !== stat.ino || + (await readMigrationFileSnapshot({ filePath: params.agentsPath, label: "AGENTS.md" })) + .content !== params.expected + ) { + throw new Error("AGENTS.md changed during TOOLS.md migration"); + } + syncFs.renameSync(params.agentsPath, backupPath); + claimed = true; + publishNoClobberSync(tempPath, params.agentsPath); + syncFs.unlinkSync(tempPath); + if ( + (await readMigrationFileSnapshot({ filePath: backupPath, label: "AGENTS.md backup" })) + .content !== params.expected + ) { + syncFs.renameSync(backupPath, params.agentsPath); + claimed = false; + throw new Error("AGENTS.md changed during TOOLS.md migration"); + } + } else { + publishNoClobberSync(tempPath, params.agentsPath); + syncFs.unlinkSync(tempPath); + } + await syncDirectory(path.dirname(params.agentsPath)); + if (stat) { + await fs.rm(backupPath); + claimed = false; + await syncDirectory(path.dirname(params.agentsPath)); + } + } catch (error) { + await fs.rm(tempPath, { force: true }); + if (claimed) { + try { + await fs.lstat(params.agentsPath); + } catch (pathError) { + if ((pathError as NodeJS.ErrnoException).code === "ENOENT") { + await restoreClaimNoClobber(backupPath, params.agentsPath); + } + } + } + throw error; + } +} + +async function recoverInterruptedAgentsClaim(params: { + agentsPath: string; + toolsContent: string; + shouldMerge: boolean; +}): Promise { + const { agentsPath } = params; + await recoverInterruptedAgentsPublish(agentsPath); + const entries = await fs.readdir(path.dirname(agentsPath)).catch(() => [] as string[]); + const prefix = `${path.basename(agentsPath)}.doctor-backup-`; + const claims = entries.filter((entry) => entry.startsWith(prefix)); + if (claims.length === 0) { + return; + } + if (claims.length > 1) { + throw new Error("multiple interrupted AGENTS.md migration claims require manual recovery"); + } + const claimPath = path.join(path.dirname(agentsPath), claims[0]!); + const claimSnapshot = await readMigrationFileSnapshot({ + filePath: claimPath, + label: "AGENTS.md migration claim", + }); + const claimIdentity = parseMigrationClaimIdentity(claims[0]!, prefix); + if ( + claimIdentity && + claimIdentity.ownerPid !== process.pid && + Date.now() - claimIdentity.createdAtMs < ACTIVE_CLAIM_MAX_AGE_MS && + isProcessAlive(claimIdentity.ownerPid) + ) { + throw new Error( + `AGENTS.md migration claim is held by running process ${claimIdentity.ownerPid}`, + ); + } + try { + const agentsSnapshot = await readMigrationFileSnapshot({ + filePath: agentsPath, + label: "AGENTS.md", + }); + const claimedContent = claimSnapshot.content; + const expected = params.shouldMerge + ? mergeToolsMdIntoAgentsMd(claimedContent, params.toolsContent) + : rewriteLegacyAgentsToolsGuidance(claimedContent); + if (agentsSnapshot.content === expected || agentsSnapshot.content === claimedContent) { + await fs.rm(claimPath); + return; + } + throw new Error(`interrupted AGENTS.md claim is preserved at ${claimPath}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + await publishNoClobber(claimPath, agentsPath); + await fs.rm(claimPath); +} + +async function recoverInterruptedAgentsPublish(agentsPath: string): Promise { + const dir = path.dirname(agentsPath); + const prefix = `${path.basename(agentsPath)}.doctor-writing-`; + let agentsStat: syncFs.Stats; + try { + agentsStat = syncFs.lstatSync(agentsPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + const linkedTemps = syncFs.readdirSync(dir).filter((entry) => { + if (!entry.startsWith(prefix)) { + return false; + } + const tempStat = syncFs.lstatSync(path.join(dir, entry)); + return tempStat.isFile() && tempStat.dev === agentsStat.dev && tempStat.ino === agentsStat.ino; + }); + if (!agentsStat.isFile() || agentsStat.nlink !== 2 || linkedTemps.length !== 1) { + return; + } + // The active TOOLS.md claim excludes concurrent doctor writers here; this + // same-inode link can only be the completed half of an interrupted publish. + syncFs.unlinkSync(path.join(dir, linkedTemps[0]!)); + await syncDirectory(dir); +} + +async function syncDirectory(dir: string): Promise { + if (process.platform === "win32") { + return; + } + const handle = await fs.open(dir, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function restoreClaimNoClobber(claimPath: string, destinationPath: string): Promise { + try { + await publishNoClobber(claimPath, destinationPath); + await fs.rm(claimPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new Error(`migration claim is preserved at ${claimPath}`, { cause: error }); + } + throw error; + } +} + +async function publishNoClobber(sourcePath: string, destinationPath: string): Promise { + try { + await fs.link(sourcePath, destinationPath); + } catch (error) { + if (!HARD_LINK_UNSUPPORTED_CODES.has((error as NodeJS.ErrnoException).code ?? "")) { + throw error; + } + await fs.copyFile(sourcePath, destinationPath, syncFs.constants.COPYFILE_EXCL); + } +} + +function publishNoClobberSync(sourcePath: string, destinationPath: string): void { + try { + syncFs.linkSync(sourcePath, destinationPath); + } catch (error) { + if (!HARD_LINK_UNSUPPORTED_CODES.has((error as NodeJS.ErrnoException).code ?? "")) { + throw error; + } + syncFs.copyFileSync(sourcePath, destinationPath, syncFs.constants.COPYFILE_EXCL); + } +} + +function archivePathForSource( + agentId: string, + source: ToolsMdSource, + env: NodeJS.ProcessEnv, +): string { + const safeAgentId = agentId.replace(/[^A-Za-z0-9._-]+/g, "-"); + return path.join( + resolveStateDir(env), + "backups", + "tools-md-migration", + `${safeAgentId}-${source.sha256}.md`, + ); +} + +async function archiveSource(params: { + agentId: string; + source: ToolsMdSource; + env: NodeJS.ProcessEnv; +}): Promise { + const archivePath = archivePathForSource(params.agentId, params.source, params.env); + const archiveDir = path.dirname(archivePath); + await fs.mkdir(archiveDir, { recursive: true, mode: 0o700 }); + const tempPath = `${archivePath}.doctor-writing-${process.pid}-${Date.now()}`; + try { + const handle = await fs.open(tempPath, "wx", 0o600); + try { + await handle.writeFile(params.source.content, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + await publishNoClobber(tempPath, archivePath); + await fs.rm(tempPath); + await syncDirectory(archiveDir); + } catch (error) { + await fs.rm(tempPath, { force: true }); + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + throw error; + } + if (sha256(await fs.readFile(archivePath, "utf8")) !== params.source.sha256) { + throw new Error(`TOOLS.md migration archive collision at ${archivePath}`, { cause: error }); + } + } + return archivePath; +} + +function migrationFinding(params: { + agentId: string; + path: string; + message: string; + severity?: HealthFinding["severity"]; + requirement: string; +}): HealthFinding { + return { + checkId: TOOLS_MD_MIGRATION_CHECK_ID, + severity: params.severity ?? "warning", + message: params.message, + path: params.path, + target: params.agentId, + requirement: params.requirement, + fixHint: `Run ${formatCliCommand("openclaw doctor --fix")} to merge TOOLS.md into AGENTS.md.`, + }; +} + +export async function collectToolsMdMigrationFindings( + cfg: OpenClawConfig, +): Promise { + const findings: HealthFinding[] = []; + for (const target of resolveToolsMdMigrationWorkspaceTargets(cfg)) { + try { + const source = await readToolsMd(target.workspaceDir); + if (source) { + findings.push( + migrationFinding({ + agentId: target.primaryAgentId, + path: source.path, + message: `Agent "${target.primaryAgentId}" still stores local tool notes in TOOLS.md.`, + requirement: "legacy-tools-md", + }), + ); + if (shouldMergeToolsMd(source.content)) { + const agentsPath = path.join(target.workspaceDir, DEFAULT_AGENTS_FILENAME); + const agentsContent = ( + await readMigrationFileSnapshot({ + filePath: agentsPath, + label: "AGENTS.md", + allowMissing: true, + }) + ).content; + const mergedChars = mergeToolsMdIntoAgentsMd(agentsContent, source.content).length; + for (const budget of describeToolsMdMergedBootstrapLimits({ + cfg, + agentIds: target.agentIds, + mergedChars, + })) { + findings.push( + migrationFinding({ + agentId: budget.agentId, + path: agentsPath, + message: budget.message, + requirement: "tools-md-merged-bootstrap-limit", + }), + ); + } + } + } + } catch (error) { + findings.push( + migrationFinding({ + agentId: target.primaryAgentId, + path: path.join(target.workspaceDir, DEFAULT_TOOLS_FILENAME), + message: `Agent "${target.primaryAgentId}" TOOLS.md cannot be migrated: ${errorMessage(error)}`, + severity: "error", + requirement: "tools-md-migration-blocked", + }), + ); + } + } + return findings; +} + +export async function maybeMigrateToolsMd(params: { + cfg: OpenClawConfig; + shouldRepair: boolean; + env?: NodeJS.ProcessEnv; +}): Promise { + const env = params.env ?? process.env; + const changes: string[] = []; + const warnings: string[] = []; + for (const target of resolveToolsMdMigrationWorkspaceTargets(params.cfg)) { + try { + const source = await readToolsMd(target.workspaceDir, { + recoverClaims: params.shouldRepair, + }); + if (!source) { + continue; + } + if (!params.shouldRepair) { + note( + `${shortenHomePath(source.path)} will be archived and merged into AGENTS.md when customized.`, + "TOOLS.md migration preview", + ); + continue; + } + + const shouldMerge = shouldMergeToolsMd(source.content); + await archiveSource({ agentId: target.primaryAgentId, source, env }); + const claimPath = `${source.path}${TOOLS_CLAIM_INFIX}${process.pid}-${Date.now()}-${source.sha256.slice(0, 12)}`; + await fs.rename(source.path, claimPath); + try { + if (sha256(await fs.readFile(claimPath, "utf8")) !== source.sha256) { + throw new Error("TOOLS.md changed before the migration claim was acquired"); + } + const agentsPath = path.join(target.workspaceDir, DEFAULT_AGENTS_FILENAME); + await recoverInterruptedAgentsClaim({ + agentsPath, + toolsContent: source.content, + shouldMerge, + }); + const agentsContent = ( + await readMigrationFileSnapshot({ + filePath: agentsPath, + label: "AGENTS.md", + allowMissing: true, + }) + ).content; + const merged = shouldMerge + ? mergeToolsMdIntoAgentsMd(agentsContent, source.content) + : rewriteLegacyAgentsToolsGuidance(agentsContent); + if (merged !== agentsContent) { + await writeAgentsAtomically({ agentsPath, expected: agentsContent, content: merged }); + } + if (sha256(await fs.readFile(claimPath, "utf8")) !== source.sha256) { + throw new Error("TOOLS.md changed while the migration claim was held"); + } + if ( + merged !== agentsContent && + (await readMigrationFileSnapshot({ filePath: agentsPath, label: "AGENTS.md" })) + .content !== merged + ) { + throw new Error("AGENTS.md changed after TOOLS.md migration was written"); + } + await fs.rm(claimPath); + await syncDirectory(target.workspaceDir); + } catch (error) { + try { + await restoreClaimNoClobber(claimPath, source.path); + } catch (restoreError) { + throw new Error(`TOOLS.md migration claim is preserved at ${claimPath}`, { + cause: restoreError, + }); + } + throw error; + } + changes.push( + shouldMerge + ? `Merged ${shortenHomePath(source.path)} into AGENTS.md and archived the original.` + : `Removed untouched ${shortenHomePath(source.path)} after archiving it.`, + ); + } catch (error) { + warnings.push( + `Agent "${target.primaryAgentId}" TOOLS.md was not migrated: ${errorMessage(error)}`, + ); + } + } + if (changes.length > 0) { + note(changes.join("\n"), "TOOLS.md migration"); + } + if (warnings.length > 0) { + note(warnings.join("\n"), "Doctor warnings"); + } + return { changes, warnings }; +} diff --git a/src/config/types.agent-defaults.ts b/src/config/types.agent-defaults.ts index 5c443a41a027..c583b8311dae 100644 --- a/src/config/types.agent-defaults.ts +++ b/src/config/types.agent-defaults.ts @@ -166,7 +166,7 @@ export type AgentDefaultsConfig = { /** * List of optional bootstrap filenames to skip writing to the workspace root. * Applies to: SOUL.md, USER.md, IDENTITY.md ("HEARTBEAT.md" is accepted but a no-op). - * Required workspace setup such as AGENTS.md and TOOLS.md still runs. + * Required workspace setup such as AGENTS.md still runs. * Example: ["SOUL.md", "USER.md", "IDENTITY.md"] */ skipOptionalBootstrapFiles?: OptionalBootstrapFileName[]; diff --git a/src/flows/doctor-health-contribution-runners.workspace.ts b/src/flows/doctor-health-contribution-runners.workspace.ts index 161c00a61b39..c61592a3d858 100644 --- a/src/flows/doctor-health-contribution-runners.workspace.ts +++ b/src/flows/doctor-health-contribution-runners.workspace.ts @@ -122,6 +122,15 @@ export async function runHeartbeatScratchMigrationHealth( }); } +export async function runToolsMdMigrationHealth(ctx: DoctorHealthFlowContext): Promise { + const { maybeMigrateToolsMd } = await import("../commands/doctor-tools-md-migration.js"); + await maybeMigrateToolsMd({ + cfg: ctx.cfg, + shouldRepair: ctx.prompter.shouldRepair, + env: ctx.env, + }); +} + export async function runHeartbeatTaskMigrationHealth(ctx: DoctorHealthFlowContext): Promise { const { maybeMigrateHeartbeatTasksToCron } = await import("../commands/doctor-heartbeat-task-migration.js"); diff --git a/src/flows/doctor-health-contributions-final.ts b/src/flows/doctor-health-contributions-final.ts index e543ca17125e..ad1d3244f199 100644 --- a/src/flows/doctor-health-contributions-final.ts +++ b/src/flows/doctor-health-contributions-final.ts @@ -27,6 +27,7 @@ import { runHooksModelHealth, runMemorySearchHealthContribution, runSkillsHealth, + runToolsMdMigrationHealth, runWorkspaceStatusHealth, runWorkspaceSuggestionsHealth, } from "./doctor-health-contribution-runners.workspace.js"; @@ -263,6 +264,20 @@ export function resolveFinalDoctorHealthContributions(params: { }, run: runHeartbeatScratchMigrationHealth, }), + createDoctorHealthContribution({ + id: "doctor:tools-md-migration", + label: "TOOLS.md migration", + healthChecks: { + description: "Workspace TOOLS.md notes must migrate into the AGENTS.md Tools section.", + defaultEnabled: true, + async detect(ctx) { + const { collectToolsMdMigrationFindings } = + await import("../commands/doctor-tools-md-migration.js"); + return collectToolsMdMigrationFindings(ctx.cfg); + }, + }, + run: runToolsMdMigrationHealth, + }), createDoctorHealthContribution({ id: "doctor:heartbeat-task-cron-migration", label: "Heartbeat task cron migration", diff --git a/src/flows/doctor-health-contributions.test.ts b/src/flows/doctor-health-contributions.test.ts index 5e3bc74f3fb8..1f340b18b72e 100644 --- a/src/flows/doctor-health-contributions.test.ts +++ b/src/flows/doctor-health-contributions.test.ts @@ -111,7 +111,9 @@ const mocks = vi.hoisted(() => ({ collectHeartbeatCadenceMigrationFindings: vi.fn(async () => [] as unknown[]), maybeMigrateHeartbeatCadenceToCron: vi.fn().mockResolvedValue({ changes: [], warnings: [] }), collectHeartbeatScratchMigrationFindings: vi.fn(async () => [] as unknown[]), + collectToolsMdMigrationFindings: vi.fn(async () => [] as unknown[]), maybeMigrateHeartbeatFilesToScratch: vi.fn().mockResolvedValue({ changes: [], warnings: [] }), + maybeMigrateToolsMd: vi.fn().mockResolvedValue({ changes: [], warnings: [] }), collectHeartbeatTaskMigrationFindings: vi.fn(async () => [] as unknown[]), maybeMigrateHeartbeatTasksToCron: vi.fn().mockResolvedValue({ changes: [], warnings: [] }), collectWhatsappResponsivenessHealthFindings: vi.fn((): readonly HealthFinding[] => []), @@ -420,6 +422,11 @@ vi.mock("../commands/doctor-heartbeat-scratch-migration.js", () => ({ maybeMigrateHeartbeatFilesToScratch: mocks.maybeMigrateHeartbeatFilesToScratch, })); +vi.mock("../commands/doctor-tools-md-migration.js", () => ({ + collectToolsMdMigrationFindings: mocks.collectToolsMdMigrationFindings, + maybeMigrateToolsMd: mocks.maybeMigrateToolsMd, +})); + vi.mock("../commands/doctor-heartbeat-task-migration.js", () => ({ collectHeartbeatTaskMigrationFindings: mocks.collectHeartbeatTaskMigrationFindings, maybeMigrateHeartbeatTasksToCron: mocks.maybeMigrateHeartbeatTasksToCron, @@ -700,6 +707,10 @@ describe("doctor health contributions", () => { mocks.collectHeartbeatScratchMigrationFindings.mockResolvedValue([]); mocks.maybeMigrateHeartbeatFilesToScratch.mockReset(); mocks.maybeMigrateHeartbeatFilesToScratch.mockResolvedValue({ changes: [], warnings: [] }); + mocks.collectToolsMdMigrationFindings.mockReset(); + mocks.collectToolsMdMigrationFindings.mockResolvedValue([]); + mocks.maybeMigrateToolsMd.mockReset(); + mocks.maybeMigrateToolsMd.mockResolvedValue({ changes: [], warnings: [] }); mocks.collectHeartbeatTaskMigrationFindings.mockReset(); mocks.collectHeartbeatTaskMigrationFindings.mockResolvedValue([]); mocks.maybeMigrateHeartbeatTasksToCron.mockReset(); diff --git a/src/gateway/server-methods/agents.ts b/src/gateway/server-methods/agents.ts index 9053a100e67f..8dd1ebb3857f 100644 --- a/src/gateway/server-methods/agents.ts +++ b/src/gateway/server-methods/agents.ts @@ -61,7 +61,6 @@ import { DEFAULT_IDENTITY_FILENAME, DEFAULT_MEMORY_FILENAME, DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_USER_FILENAME, ensureAgentWorkspace, isWorkspaceSetupCompleted, @@ -106,7 +105,6 @@ import type { GatewayRequestHandlers, RespondFn } from "./types.js"; const BOOTSTRAP_FILE_NAMES = [ DEFAULT_AGENTS_FILENAME, DEFAULT_SOUL_FILENAME, - DEFAULT_TOOLS_FILENAME, DEFAULT_IDENTITY_FILENAME, DEFAULT_USER_FILENAME, DEFAULT_BOOTSTRAP_FILENAME, diff --git a/src/hooks/bundled/README.md b/src/hooks/bundled/README.md index 7eff826a2264..a553fbb1d34e 100644 --- a/src/hooks/bundled/README.md +++ b/src/hooks/bundled/README.md @@ -20,7 +20,7 @@ openclaw hooks enable session-memory ### 📎 bootstrap-extra-files -Injects extra bootstrap files (for example monorepo `AGENTS.md`/`TOOLS.md`) during prompt assembly. +Injects extra bootstrap files (for example monorepo `AGENTS.md`) during prompt assembly. **Events**: `agent:bootstrap` **What it does**: Expands configured workspace glob/path patterns and appends matching bootstrap files to injected context. diff --git a/src/hooks/bundled/bootstrap-extra-files/HOOK.md b/src/hooks/bundled/bootstrap-extra-files/HOOK.md index 5f56baf923a0..873cd9b6ed8f 100644 --- a/src/hooks/bundled/bootstrap-extra-files/HOOK.md +++ b/src/hooks/bundled/bootstrap-extra-files/HOOK.md @@ -21,7 +21,7 @@ Loads additional bootstrap files into `Project Context` during `agent:bootstrap` ## Why Use this when your workspace has multiple context roots (for example monorepos) and -you want to include extra `AGENTS.md`/`TOOLS.md`-class files without changing the +you want to include extra `AGENTS.md`-class files without changing the workspace root. ## Configuration @@ -34,7 +34,7 @@ workspace root. "entries": { "bootstrap-extra-files": { "enabled": true, - "paths": ["packages/*/AGENTS.md", "packages/*/TOOLS.md"] + "paths": ["packages/*/AGENTS.md"] } } } @@ -49,5 +49,5 @@ workspace root. - `files` (string[]): alias of `paths`. All paths are resolved from the workspace and must stay inside it (including realpath checks). -Only recognized bootstrap basenames are loaded (`AGENTS.md`, `SOUL.md`, `TOOLS.md`, +Only recognized bootstrap basenames are loaded (`AGENTS.md`, `SOUL.md`, `IDENTITY.md`, `USER.md`, `BOOTSTRAP.md`, `MEMORY.md`). diff --git a/src/hooks/bundled/bootstrap-extra-files/handler.test.ts b/src/hooks/bundled/bootstrap-extra-files/handler.test.ts index f8cdd3e7b750..bf8380655f44 100644 --- a/src/hooks/bundled/bootstrap-extra-files/handler.test.ts +++ b/src/hooks/bundled/bootstrap-extra-files/handler.test.ts @@ -87,14 +87,11 @@ describe("bootstrap-extra-files hook", () => { workspaceDir: tempDir, cfg, sessionKey: "agent:main:subagent:abc", - rootFiles: [ - { name: "AGENTS.md", content: "root agents" }, - { name: "TOOLS.md", content: "root tools" }, - ], + rootFiles: [{ name: "AGENTS.md", content: "root agents" }], }); const event = createHookEvent("agent", "bootstrap", "agent:main:subagent:abc", context); await handler(event); - expect(context.bootstrapFiles.map((f) => f.name).toSorted()).toEqual(["AGENTS.md", "TOOLS.md"]); + expect(context.bootstrapFiles.map((f) => f.name).toSorted()).toEqual(["AGENTS.md"]); }); }); diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md index 8dba76ef7183..a1ba04c4f1d1 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/discord-group-codex-message-tool.md @@ -7,7 +7,7 @@ - Default happy path: the same Codex agent is mentioned in a Discord group/channel while Telegram can remain the user's primary direct interface. - Group-visible output must be explicit through the message tool; the model is also told to mostly lurk unless directly addressed or clearly useful. - This captures the OpenClaw-owned Codex app-server inputs and reconstructs the stable Codex model/permission layers from committed Codex prompt fixtures. -- This also simulates Codex workspace bootstrap routing: `TOOLS.md` as inherited developer instructions, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer. +- This also simulates Codex workspace bootstrap routing: `AGENTS.md` through native project-doc discovery, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer. ## Scenario Metadata @@ -22,7 +22,6 @@ "runtime": "codex_app_server", "simulatedHeartbeatWorkspaceFile": "/tmp/openclaw-happy-path/workspace/HEARTBEAT.md", "simulatedWorkspaceBootstrapFiles": ["/tmp/openclaw-happy-path/workspace/MEMORY.md"], - "simulatedWorkspaceDeveloperInstructionFiles": ["/tmp/openclaw-happy-path/workspace/TOOLS.md"], "simulatedWorkspaceTurnScopedDeveloperInstructionFiles": [ "/tmp/openclaw-happy-path/workspace/IDENTITY.md", "/tmp/openclaw-happy-path/workspace/SOUL.md", @@ -225,20 +224,20 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the "roughTokens": 14906 }, "openClawDeveloperInstructions": { - "chars": 3715, - "roughTokens": 929 + "chars": 3471, + "roughTokens": 868 }, "totalTextOnly": { - "chars": 28162, - "roughTokens": 7041 + "chars": 27854, + "roughTokens": 6964 }, "totalWithDynamicToolsJson": { - "chars": 87788, - "roughTokens": 21947 + "chars": 87480, + "roughTokens": 21870 }, "userInputText": { - "chars": 1364, - "roughTokens": 341 + "chars": 1300, + "roughTokens": 325 } } ``` @@ -448,14 +447,6 @@ When explicitly_mentioned_bot is true, the incoming message mentions your channe You are in a Discord group chat. Normal final replies are private and are not automatically sent to this group chat. To post visible output here, use the message tool with action=send; the target defaults to this group chat. Be a good group participant: mostly lurk and follow the conversation; reply only when directly addressed or you can add clear value. Emoji reactions are welcome when available. Write like a human. Avoid Markdown tables. Minimize empty lines and use normal chat conventions, not document-style spacing. Don't type literal \n sequences; use real line breaks sparingly. If addressed to someone else, stay silent unless invited or correcting key facts. Discord: wrap bare URLs like to suppress embeds. When subagent or session-spawn tools are available and a directly requested group-chat task will require several tool calls, prefer delegating bounded side investigations early so the channel gets a responsive path forward. Keep the critical path local, avoid subagents for simple one-step work, and only surface concise group-visible updates when they add value. If no visible group response is needed, do not call message(action=send). Your normal final answer stays private and will not be posted to this group chat. Be extremely selective: reply only when directly addressed or clearly helpful. Activation: trigger-only (you are invoked only when explicitly mentioned; recent context may be included). Address the specific sender noted in the message context. - -## OpenClaw Workspace Instructions - -OpenClaw loaded these workspace instruction files from the active agent workspace. Internalize and follow them accordingly. - -### /tmp/openclaw-happy-path/workspace/TOOLS.md - - ```` ### Developer: Codex Collaboration Mode Instructions @@ -498,7 +489,7 @@ Treat this OpenClaw-provided context as supporting project/user reference for th ## OpenClaw Workspace Context -OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. TOOLS.md is provided as inherited Codex developer instructions. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here. +OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here. # Project Context diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md index 6008f9a14ccc..e700fed8eb54 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md @@ -7,7 +7,7 @@ - Default happy path: OpenAI model through the Codex harness/runtime, Telegram direct conversation, and message-tool-only visible replies. - A quiet turn is represented by not calling `message(action=send)`; the normal final assistant text is private to OpenClaw/Codex. - This captures the OpenClaw-owned Codex app-server inputs and reconstructs the stable Codex model/permission layers from committed Codex prompt fixtures. -- This also simulates Codex workspace bootstrap routing: `TOOLS.md` as inherited developer instructions, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer. +- This also simulates Codex workspace bootstrap routing: `AGENTS.md` through native project-doc discovery, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer. ## Scenario Metadata @@ -22,7 +22,6 @@ "runtime": "codex_app_server", "simulatedHeartbeatWorkspaceFile": "/tmp/openclaw-happy-path/workspace/HEARTBEAT.md", "simulatedWorkspaceBootstrapFiles": ["/tmp/openclaw-happy-path/workspace/MEMORY.md"], - "simulatedWorkspaceDeveloperInstructionFiles": ["/tmp/openclaw-happy-path/workspace/TOOLS.md"], "simulatedWorkspaceTurnScopedDeveloperInstructionFiles": [ "/tmp/openclaw-happy-path/workspace/IDENTITY.md", "/tmp/openclaw-happy-path/workspace/SOUL.md", @@ -225,20 +224,20 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the "roughTokens": 14829 }, "openClawDeveloperInstructions": { - "chars": 2606, - "roughTokens": 652 + "chars": 2362, + "roughTokens": 591 }, "totalTextOnly": { - "chars": 26682, - "roughTokens": 6671 + "chars": 26374, + "roughTokens": 6594 }, "totalWithDynamicToolsJson": { - "chars": 86000, - "roughTokens": 21500 + "chars": 85692, + "roughTokens": 21423 }, "userInputText": { - "chars": 993, - "roughTokens": 249 + "chars": 929, + "roughTokens": 233 } } ``` @@ -446,14 +445,6 @@ When explicitly_mentioned_bot is true, the incoming message mentions your channe You are in a Telegram direct conversation. Normal final replies are private and are not automatically sent to this conversation. To post visible output here, use the message tool with action=send; the target defaults to this conversation. If no visible direct response is needed, do not call message(action=send). Your normal final answer stays private and will not be posted to the conversation. - -## OpenClaw Workspace Instructions - -OpenClaw loaded these workspace instruction files from the active agent workspace. Internalize and follow them accordingly. - -### /tmp/openclaw-happy-path/workspace/TOOLS.md - - ```` ### Developer: Codex Collaboration Mode Instructions @@ -496,7 +487,7 @@ Treat this OpenClaw-provided context as supporting project/user reference for th ## OpenClaw Workspace Context -OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. TOOLS.md is provided as inherited Codex developer instructions. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here. +OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here. # Project Context diff --git a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md index 25ab4c1c977c..f5cbe884aa71 100644 --- a/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md +++ b/test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-heartbeat-codex-tool.md @@ -7,7 +7,7 @@ - Heartbeat happy path: Codex receives the structured `heartbeat_respond` dynamic tool in the searchable catalog instead of the initial tool context. - The heartbeat tool still carries the notify/no-notify decision, outcome, summary, and optional notification text instead of relying only on final-text parsing. - This captures the OpenClaw-owned Codex app-server inputs and reconstructs the stable Codex model/permission layers from committed Codex prompt fixtures. -- This also simulates Codex workspace bootstrap routing: `TOOLS.md` as inherited developer instructions, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer. +- This also simulates Codex workspace bootstrap routing: `AGENTS.md` through native project-doc discovery, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer. ## Scenario Metadata @@ -22,7 +22,6 @@ "runtime": "codex_app_server", "simulatedHeartbeatWorkspaceFile": "/tmp/openclaw-happy-path/workspace/HEARTBEAT.md", "simulatedWorkspaceBootstrapFiles": ["/tmp/openclaw-happy-path/workspace/MEMORY.md"], - "simulatedWorkspaceDeveloperInstructionFiles": ["/tmp/openclaw-happy-path/workspace/TOOLS.md"], "simulatedWorkspaceTurnScopedDeveloperInstructionFiles": [ "/tmp/openclaw-happy-path/workspace/IDENTITY.md", "/tmp/openclaw-happy-path/workspace/SOUL.md", @@ -226,20 +225,20 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the "roughTokens": 15220 }, "openClawDeveloperInstructions": { - "chars": 2625, - "roughTokens": 657 + "chars": 2381, + "roughTokens": 596 }, "totalTextOnly": { - "chars": 27130, - "roughTokens": 6783 + "chars": 26822, + "roughTokens": 6706 }, "totalWithDynamicToolsJson": { - "chars": 88010, - "roughTokens": 22003 + "chars": 87702, + "roughTokens": 21926 }, "userInputText": { - "chars": 1348, - "roughTokens": 337 + "chars": 1284, + "roughTokens": 321 } } ``` @@ -447,14 +446,6 @@ When explicitly_mentioned_bot is true, the incoming message mentions your channe You are in a Telegram direct conversation. Normal final replies are private and are not automatically sent to this conversation. To post visible output here, use the message tool with action=send; the target defaults to this conversation. If no visible direct response is needed, do not call message(action=send). Your normal final answer stays private and will not be posted to the conversation. - -## OpenClaw Workspace Instructions - -OpenClaw loaded these workspace instruction files from the active agent workspace. Internalize and follow them accordingly. - -### /tmp/openclaw-happy-path/workspace/TOOLS.md - - ```` ### Developer: Codex Collaboration Mode Instructions @@ -497,7 +488,7 @@ Treat this OpenClaw-provided context as supporting project/user reference for th ## OpenClaw Workspace Context -OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. TOOLS.md is provided as inherited Codex developer instructions. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here. +OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here. # Project Context diff --git a/test/helpers/agents/happy-path-prompt-snapshots.ts b/test/helpers/agents/happy-path-prompt-snapshots.ts index 33c17cc8d4ab..6029cbdfb780 100644 --- a/test/helpers/agents/happy-path-prompt-snapshots.ts +++ b/test/helpers/agents/happy-path-prompt-snapshots.ts @@ -240,13 +240,6 @@ const CODEX_WORKSPACE_BOOTSTRAP_CONTEXT_FILES = [ }, ] as const; -const CODEX_WORKSPACE_THREAD_DEVELOPER_CONTEXT_FILES = [ - { - path: path.join(WORKSPACE_DIR, "TOOLS.md"), - content: "", - }, -] as const; - const CODEX_WORKSPACE_TURN_SCOPED_DEVELOPER_CONTEXT_FILES = [ { path: path.join(WORKSPACE_DIR, "IDENTITY.md"), @@ -268,7 +261,7 @@ const CODEX_HEARTBEAT_CONTEXT_FILE = { } as const; const CODEX_WORKSPACE_BOOTSTRAP_PROMPT_CONTEXT = [ - "OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. TOOLS.md is provided as inherited Codex developer instructions. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here.", + "OpenClaw loaded these user-editable workspace files for the current turn. Codex loads AGENTS.md natively. SOUL.md, IDENTITY.md, and USER.md are provided as turn-scoped collaboration instructions so native Codex subagents do not inherit them. HEARTBEAT.md is handled by heartbeat collaboration-mode guidance. Those files are not repeated here.", "", "# Project Context", "", @@ -284,21 +277,6 @@ const CODEX_WORKSPACE_BOOTSTRAP_PROMPT_CONTEXT = [ .join("\n") .trim(); -const CODEX_WORKSPACE_THREAD_DEVELOPER_INSTRUCTIONS = [ - "## OpenClaw Workspace Instructions", - "", - "OpenClaw loaded these workspace instruction files from the active agent workspace. Internalize and follow them accordingly.", - "", - ...CODEX_WORKSPACE_THREAD_DEVELOPER_CONTEXT_FILES.flatMap((file) => [ - `### ${file.path}`, - "", - file.content, - "", - ]), -] - .join("\n") - .trim(); - const CODEX_WORKSPACE_TURN_SCOPED_DEVELOPER_INSTRUCTIONS = [ "## OpenClaw Agent Soul", "", @@ -903,7 +881,6 @@ function renderScenarioSnapshot( appServer, config: CODEX_PROMPT_SNAPSHOT_THREAD_CONFIG, promptText: codexTurnPromptText, - developerInstructionAdditions: CODEX_WORKSPACE_THREAD_DEVELOPER_INSTRUCTIONS, turnScopedDeveloperInstructions: CODEX_WORKSPACE_TURN_SCOPED_DEVELOPER_INSTRUCTIONS, heartbeatCollaborationInstructions: scenario.trigger === "heartbeat" ? CODEX_HEARTBEAT_COLLABORATION_INSTRUCTIONS : undefined, @@ -922,7 +899,7 @@ function renderScenarioSnapshot( "", ...scenario.notes.map((note) => `- ${note}`), "- This captures the OpenClaw-owned Codex app-server inputs and reconstructs the stable Codex model/permission layers from committed Codex prompt fixtures.", - "- This also simulates Codex workspace bootstrap routing: `TOOLS.md` as inherited developer instructions, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer.", + "- This also simulates Codex workspace bootstrap routing: `AGENTS.md` through native project-doc discovery, `SOUL.md`, `IDENTITY.md`, and `USER.md` as turn-scoped collaboration instructions, `MEMORY.md` in turn input, and `HEARTBEAT.md` as a heartbeat-only file pointer.", "", "## Scenario Metadata", "", @@ -942,8 +919,6 @@ function renderScenarioSnapshot( simulatedWorkspaceBootstrapFiles: CODEX_WORKSPACE_BOOTSTRAP_CONTEXT_FILES.map( (file) => file.path, ), - simulatedWorkspaceDeveloperInstructionFiles: - CODEX_WORKSPACE_THREAD_DEVELOPER_CONTEXT_FILES.map((file) => file.path), simulatedWorkspaceTurnScopedDeveloperInstructionFiles: CODEX_WORKSPACE_TURN_SCOPED_DEVELOPER_CONTEXT_FILES.map((file) => file.path), simulatedHeartbeatWorkspaceFile: CODEX_HEARTBEAT_CONTEXT_FILE.path, diff --git a/test/helpers/agents/prompt-composition-scenarios.ts b/test/helpers/agents/prompt-composition-scenarios.ts index 4869d21050d7..3f8e0587a0bc 100644 --- a/test/helpers/agents/prompt-composition-scenarios.ts +++ b/test/helpers/agents/prompt-composition-scenarios.ts @@ -504,7 +504,7 @@ async function createToolRichScenario(workspaceDir: string): PromisereleaseRelease OpenClaw safely./skills/release/SKILL.md", "", ].join("\n"); - const contextFiles = await readContextFiles(workspaceDir, ["AGENTS.md", "TOOLS.md", "SOUL.md"]); + const contextFiles = await readContextFiles(workspaceDir, ["AGENTS.md", "SOUL.md"]); const systemPrompt = buildToolRichSystemPrompt({ workspaceDir, skillsPrompt, @@ -573,9 +573,9 @@ async function createBootstrapWarningScenario(workspaceDir: string): Promise { "# AGENTS.md", "", "## Session Startup", - "Read AGENTS.md and TOOLS.md before making changes.", + "Read AGENTS.md before making changes.", + "", + "## Tools", + "Use rg before grep.", "", "## Red Lines", "Do not rewrite user commits.", ].join("\n"), }); - await writeWorkspaceFile({ - dir: workspaceDir, - name: "TOOLS.md", - content: "# TOOLS.md\n\nUse rg before grep.\n", - }); await writeWorkspaceFile({ dir: workspaceDir, name: "SOUL.md", diff --git a/test/scripts/prompt-snapshots.test.ts b/test/scripts/prompt-snapshots.test.ts index 2499dd49995a..646c18a552f2 100644 --- a/test/scripts/prompt-snapshots.test.ts +++ b/test/scripts/prompt-snapshots.test.ts @@ -188,7 +188,6 @@ describe("happy path prompt snapshots", () => { expect(telegram).toContain("OpenClaw runtime context for this turn:"); expect(telegram).toContain(""); expect(telegram).toContain(""); - expect(telegram).toContain(""); expect(telegram).toContain(""); expect(telegram).toContain(""); expect(telegram).not.toContain("");